diff --git a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes index a16c54ab841bb..4306966b7fcc3 100644 --- a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes +++ b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes @@ -2,33 +2,36 @@ What: /sys/class/reboot-mode//reboot_modes Date: March 2026(TBD) KernelVersion: TBD Contact: linux-pm@vger.kernel.org - Description: +Description: This interface exposes the reboot-mode arguments registered with the reboot-mode framework. It is a read-only interface and provides a space separated list of reboot-mode arguments supported on the current platform. Example: + recovery fastboot bootloader The exact sysfs path may vary depending on the name of the driver that registers the arguments. - Example: + Example:: + /sys/class/reboot-mode/nvmem-reboot-mode/reboot_modes /sys/class/reboot-mode/syscon-reboot-mode/reboot_modes /sys/class/reboot-mode/qcom-pon/reboot_modes The supported arguments can be used by userspace to invoke device reset using the standard reboot() system - call interface, with the "argument" as string to "*arg" - parameter along with LINUX_REBOOT_CMD_RESTART2. + call interface, with the "argument" as string to ``*arg`` + parameter along with ``LINUX_REBOOT_CMD_RESTART2``. A driver can expose the supported arguments by registering them with the reboot-mode framework using the property names that follow the mode- format. Example: - mode-bootloader, mode-recovery. + + mode-bootloader, mode-recovery This attribute is useful for scripts or initramfs logic that need to programmatically determine diff --git a/Documentation/ABI/testing/sysfs-platform-ayn-ec b/Documentation/ABI/testing/sysfs-platform-ayn-ec new file mode 100644 index 0000000000000..32cb6f7ca2fc5 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-platform-ayn-ec @@ -0,0 +1,59 @@ +What: /sys/class/hwmon/hwmon[0-9]/pwm1_enable +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This sets the PWM fan mode of operation. Valid values are [0-3]. + Values [0-2] conform with standard hwmon operating modes. Value 3 + enables user defined fan curve settings. + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/hwmon/hwmon[0-9]/pwm1_auto_point[1-5]_pwm +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This sets the PWM fan duty cycle for the given index of the fan curve. + When the temperature reaches the corresponding pwm1_auto_point[1-5]_temp, + the EC will automatically increase the fan duty cycle to the given value. + + Values are [0-255] + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/hwmon/hwmon[0-9]/pwm1_auto_point[1-5]_temp +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This sets the activation temperature for the given index of the fan curve. + When the temperature reaches the given value, the EC will automatically + increase the fan duty cycle to the corresponding pwm1_auto_point[1-5]_pwm + value. + + Values are [0-100] + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/leds/ayn:rgb:joystick_rings/effect +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This controls the display effect of the RGB interface. + + Values are monocolor or breathe. + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/leds/ayn:rgb:joystick_rings/effect_index +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This displays the available options for the effect attribute. + + Values are monocolor or breathe. + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. diff --git a/Documentation/admin-guide/pm/cpufreq.rst b/Documentation/admin-guide/pm/cpufreq.rst index dbe6d23a5d671..8831cface5855 100644 --- a/Documentation/admin-guide/pm/cpufreq.rst +++ b/Documentation/admin-guide/pm/cpufreq.rst @@ -516,7 +516,7 @@ This governor exposes the following tunables: of those tasks above 0 and set this attribute to 1. ``sampling_down_factor`` - Temporary multiplier, between 1 (default) and 100 inclusive, to apply to + Temporary multiplier, between 1 (default) and 100000 inclusive, to apply to the ``sampling_rate`` value if the CPU load goes above ``up_threshold``. This causes the next execution of the governor's worker routine (after @@ -586,8 +586,8 @@ This governor exposes the following tunables: 100 (5 by default). This is how much the frequency is allowed to change in one go. Setting - it to 0 will cause the default frequency step (5 percent) to be used - and setting it to 100 effectively causes the governor to periodically + it to 0 disables frequency changes by the governor entirely and setting + it to 100 effectively causes the governor to periodically switch the frequency between the ``scaling_min_freq`` and ``scaling_max_freq`` policy limits. diff --git a/Documentation/arch/arm64/cpu-hotplug.rst b/Documentation/arch/arm64/cpu-hotplug.rst index 8fb438bf77816..7c3379b704aaf 100644 --- a/Documentation/arch/arm64/cpu-hotplug.rst +++ b/Documentation/arch/arm64/cpu-hotplug.rst @@ -47,11 +47,12 @@ ever have can be described at boot. There are no power-domain considerations as such devices are emulated. CPU Hotplug on virtual systems is supported. It is distinct from physical -CPU Hotplug as all resources are described as ``present``, but CPUs may be -marked as disabled by firmware. Only the CPU's online/offline behaviour is -influenced by firmware. An example is where a virtual machine boots with a -single CPU, and additional CPUs are added once a cloud orchestrator deploys -the workload. +CPU Hotplug as all vCPU resources are statically described in the firmware +configuration tables (e.g. MADT), meaning their maximum possible count is +known at boot. However, vCPUs that are not enabled at boot are not marked +as ``present`` by the kernel until they are hotplugged. An example is where +a virtual machine boots with a single CPU, and additional CPUs are added +once a cloud orchestrator deploys the workload. For a virtual machine, the VMM (e.g. Qemu) plays the part of firmware. @@ -60,16 +61,19 @@ brought online. Firmware can enforce its policy via PSCI's return codes. e.g. ``DENIED``. The ACPI tables must describe all the resources of the virtual machine. CPUs -that firmware wishes to disable either from boot (or later) should not be -``enabled`` in the MADT GICC structures, but should have the ``online capable`` -bit set, to indicate they can be enabled later. The boot CPU must be marked as -``enabled``. The 'always on' GICR structure must be used to describe the -redistributors. +that are hot-pluggable must have the ``online capable`` bit set and the +``enabled`` bit cleared in the MADT GICC structures to indicate they can be +enabled later. The boot CPU must be marked as ``enabled`` with its +``online capable`` bit cleared. The 'always on' GICR structure must be used +to describe the redistributors. CPUs described as ``online capable`` but not ``enabled`` can be set to enabled by the DSDT's Processor object's _STA method. On virtual systems the _STA method -must always report the CPU as ``present``. Changes to the firmware policy can -be notified to the OS via device-check or eject-request. +must always set the ``ACPI_STA_DEVICE_PRESENT`` bit, while toggling the +``ACPI_STA_DEVICE_ENABLED`` bit to reflect its plug status. The kernel will +then dynamically mark the vCPU as ``present`` within the OS when the +``ACPI_STA_DEVICE_ENABLED`` bit becomes set during hot-add. Changes to the +firmware policy can be notified to the OS via device-check or eject-request. CPUs described as ``enabled`` in the static table, should not have their _STA modified dynamically by firmware. Soft-restart features such as kexec will diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 211119ce7adc7..ad04d1cdc0f05 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -128,16 +128,28 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76AE | #4193801 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1491015 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #3324348 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A77 | #4193798 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78AE | #4193793 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78C | #3324346,3324347| ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #4193794 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2054223 | ARM64_ERRATUM_2054223 | @@ -146,6 +158,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #3456084 | ARM64_ERRATUM_3194386 | @@ -158,20 +172,32 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1C | #3324346 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1C | #4193792 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X2 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X3 | #3324335 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X3 | #4193786 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X4 | #4118414 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X925 | #4193781 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1349291 | N/A | @@ -182,6 +208,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N1 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | @@ -190,20 +218,34 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N3 | #3456111 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #1619801 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #3324341 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V1 | #4193790 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V2 | #3324336 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V2 | #4193787 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3 | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| | | | #562869,1047329 | | +----------------+-----------------+-----------------+-----------------------------+ @@ -256,6 +298,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | @@ -323,3 +367,5 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Microsoft | Azure Cobalt 100| #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| Microsoft | Azure Cobalt 100| #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst index 021759198fe77..d220e266b8deb 100644 --- a/Documentation/crypto/userspace-if.rst +++ b/Documentation/crypto/userspace-if.rst @@ -327,33 +327,10 @@ CRYPTO_USER_API_RNG_CAVP option: Zero-Copy Interface ------------------- -In addition to the send/write/read/recv system call family, the AF_ALG -interface can be accessed with the zero-copy interface of -splice/vmsplice. As the name indicates, the kernel tries to avoid a copy -operation into kernel space. - -The zero-copy operation requires data to be aligned at the page -boundary. Non-aligned data can be used as well, but may require more -operations of the kernel which would defeat the speed gains obtained -from the zero-copy interface. - -The system-inherent limit for the size of one zero-copy operation is 16 -pages. If more data is to be sent to AF_ALG, user space must slice the -input into segments with a maximum size of 16 pages. - -Zero-copy can be used with the following code example (a complete -working example is provided with libkcapi): - -:: - - int pipes[2]; - - pipe(pipes); - /* input data in iov */ - vmsplice(pipes[1], iov, iovlen, SPLICE_F_GIFT); - /* opfd is the file descriptor returned from accept() system call */ - splice(pipes[0], NULL, opfd, NULL, ret, 0); - read(opfd, out, outlen); +AF_ALG used to have zero-copy support, but it was removed due to it being a +frequent source of vulnerabilities. For backwards compatibility the splice() +and sendfile() system calls are still supported, but the kernel will make an +internal copy of the data before passing it to the crypto code. Setsockopt Interface diff --git a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml index ef2b1e2044309..a177a1934b19f 100644 --- a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml @@ -42,12 +42,6 @@ properties: - const: cfg_ahb_clk - const: gcc_disp_gpll0_div_clk_src - '#clock-cells': - const: 1 - - '#power-domain-cells': - const: 1 - power-domains: description: A phandle and PM domain specifier for the CX power domain. @@ -58,18 +52,16 @@ properties: A phandle to an OPP node describing the power domain's performance point. maxItems: 1 - reg: - maxItems: 1 - required: - compatible - - reg - clocks - clock-names - - '#clock-cells' - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | @@ -101,6 +93,7 @@ examples: power-domains = <&rpmpd SM6125_VDDCX>; #clock-cells = <1>; + #reset-cells = <1>; #power-domain-cells = <1>; }; ... diff --git a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml index 938a2f1ff3fca..b28614186cc09 100644 --- a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml @@ -23,6 +23,7 @@ properties: compatible: enum: - qcom,x1e80100-camcc + - qcom,x1p42100-camcc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml index 0dabe9bbb219b..64f1e9d9896df 100644 --- a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml +++ b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml @@ -15,16 +15,14 @@ maintainers: - Jon Hunter - Rajesh Gumasta -allOf: - - $ref: dma-controller.yaml# - properties: compatible: oneOf: - - const: nvidia,tegra186-gpcdma + - enum: + - nvidia,tegra264-gpcdma + - nvidia,tegra186-gpcdma - items: - enum: - - nvidia,tegra264-gpcdma - nvidia,tegra234-gpcdma - nvidia,tegra194-gpcdma - const: nvidia,tegra186-gpcdma @@ -60,12 +58,23 @@ required: - compatible - reg - interrupts - - resets - - reset-names - "#dma-cells" - iommus - dma-channel-mask +allOf: + - $ref: dma-controller.yaml# + - if: + properties: + compatible: + contains: + enum: + - nvidia,tegra186-gpcdma + then: + required: + - resets + - reset-names + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml index 804514732dbe6..0a30a455b0ee5 100644 --- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml @@ -21,11 +21,12 @@ properties: - enum: - snps,axi-dma-1.01a - intel,kmb-axi-dma - - sophgo,cv1800b-axi-dma - starfive,jh7110-axi-dma - starfive,jh8100-axi-dma - items: - - const: altr,agilex5-axi-dma + - enum: + - altr,agilex5-axi-dma + - sophgo,cv1800b-axi-dma - const: snps,axi-dma-1.01a reg: diff --git a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml index 541325f900a1d..01f2afa023f04 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml @@ -63,6 +63,16 @@ properties: CMA pool to use for buffers allocation instead of the default CMA pool. + # FIXME: This should be made required eventually once every SoC will + # have the MBUS declared. + interconnects: + maxItems: 1 + + # FIXME: This should be made required eventually once every SoC will + # have the MBUS declared. + interconnect-names: + const: dma-mem + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml b/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml index 45630067d3c8e..0beda26ae8bb4 100644 --- a/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml +++ b/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml @@ -50,9 +50,6 @@ properties: description: VDD_RFA_1P7 supply regulator handle deprecated: true - vddrfa1p8-supply: - description: VDD_RFA_1P8 supply regulator handle - vddrfacmn-supply: description: VDD_RFA_CMN supply regulator handle @@ -68,7 +65,7 @@ required: - vddbtcmx-supply - vddrfa0p8-supply - vddrfa1p2-supply - - vddrfa1p8-supply + - vddrfa1p7-supply - vddrfacmn-supply - vddwlcx-supply - vddwlmx-supply @@ -91,7 +88,7 @@ examples: vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>; vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; - vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>; vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; diff --git a/Documentation/devicetree/bindings/net/renesas,ether.yaml b/Documentation/devicetree/bindings/net/renesas,ether.yaml index f0a52f47f95a0..dd7187f12a675 100644 --- a/Documentation/devicetree/bindings/net/renesas,ether.yaml +++ b/Documentation/devicetree/bindings/net/renesas,ether.yaml @@ -121,8 +121,7 @@ examples: #size-cells = <0>; phy1: ethernet-phy@1 { - compatible = "ethernet-phy-id0022.1537", - "ethernet-phy-ieee802.3-c22"; + compatible = "ethernet-phy-id0022.1537"; reg = <1>; interrupt-parent = <&irqc0>; interrupts = <0 IRQ_TYPE_LEVEL_LOW>; diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml index 3a35120a77ec0..431e8cb5df84e 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml @@ -136,13 +136,22 @@ allOf: items: - description: port a - description: port b - required: - - qcom,4ln-config-sel else: properties: reg: maxItems: 1 + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8280xp-qmp-gen3x4-pcie-phy + - qcom,x1e80100-qmp-gen4x4-pcie-phy + then: + required: + - qcom,4ln-config-sel + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml index db8224dfba2c1..56fb9cf763ef7 100644 --- a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml +++ b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml @@ -58,6 +58,10 @@ patternProperties: drive_soc_gpio27_pee6, drive_ao_retention_n_pee2, drive_vcomp_alert_pee1, drive_hdmi_cec_pgg0 ] +required: + - compatible + - reg + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml index f5a3a881dec4f..bd305a34eee2b 100644 --- a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml +++ b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml @@ -115,6 +115,10 @@ patternProperties: drive_sdmmc1_dat2_pj4, drive_sdmmc1_dat1_pj3, drive_sdmmc1_dat0_pj2 ] +required: + - compatible + - reg + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/pinctrl/realtek,rtd1625-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/realtek,rtd1625-pinctrl.yaml index 9562a043707ee..adc5955a2047c 100644 --- a/Documentation/devicetree/bindings/pinctrl/realtek,rtd1625-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/realtek,rtd1625-pinctrl.yaml @@ -110,7 +110,7 @@ patternProperties: input-schmitt-disable: true - input-voltage-microvolt: + input-threshold-voltage-microvolt: description: | Select the input receiver voltage domain for the pin. Valid arguments are: diff --git a/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml b/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml index fe4ac9350ba03..ac925334ae833 100644 --- a/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml @@ -18,84 +18,41 @@ patternProperties: "^buck_v(s1|gpu11|modem|pu|core|s2|pa|proc2|proc1|core_sshub)$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^v(s1|gpu11|modem|pu|core|s2|pa|proc2|proc1|core_sshub)$" - unevaluatedProperties: false "^ldo_v(ibr|rf12|usb|camio|efuse|xo22)$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^v(ibr|rf12|usb|camio|efuse|xo22)$" - unevaluatedProperties: false "^ldo_v(rfck|emc|a12|a09|ufs|bbck)$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^v(rfck|emc|a12|a09|ufs|bbck)$" - unevaluatedProperties: false "^ldo_vcn(18|13|33_1_bt|13_1_wifi|33_2_bt|33_2_wifi)$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^vcn(18|13|33_1_bt|13_1_wifi|33_2_bt|33_2_wifi)$" - unevaluatedProperties: false "^ldo_vsram_(proc2|others|md|proc1|others_sshub)$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^vsram_(proc2|others|md|proc1|others_sshub)$" - unevaluatedProperties: false "^ldo_v(fe|bif|io)28$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^v(fe|bif|io)28$" - unevaluatedProperties: false "^ldo_v(aud|io|aux|rf|m)18$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^v(aud|io|aux|rf|m)18$" - unevaluatedProperties: false "^ldo_vsim[12]$": type: object $ref: regulator.yaml# - - properties: - regulator-name: - pattern: "^vsim[12]$" - - required: - - regulator-name - unevaluatedProperties: false additionalProperties: false diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml index 3c16b260db040..051edb1da0d73 100644 --- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml +++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml @@ -72,22 +72,6 @@ properties: minItems: 1 maxItems: 4095 - sifive,fine-ctr-bits: - maximum: 15 - description: The width in bits of the fine counter. - -if: - properties: - compatible: - contains: - const: sifive,clint2 -then: - required: - - sifive,fine-ctr-bits -else: - properties: - sifive,fine-ctr-bits: false - additionalProperties: false required: diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 28784d66ae7ba..ac83310259a91 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -437,6 +437,8 @@ patternProperties: description: Diodes, Inc. "^dioo,.*": description: Dioo Microcircuit Co., Ltd + "^displaytech,.*": + description: Displaytech Ltd. "^djn,.*": description: Shenzhen DJN Optronics Technology Co., Ltd "^dlc,.*": @@ -1783,6 +1785,8 @@ patternProperties: description: Variscite Ltd. "^vdl,.*": description: Van der Laan b.v. + "^verbatim,.*": + description: Verbatim Corporation "^verisilicon,.*": description: VeriSilicon Microelectronics (Shanghai) Co., Ltd. "^vertexcom,.*": diff --git a/Documentation/driver-api/nvdimm/btt.rst b/Documentation/driver-api/nvdimm/btt.rst index 2d8269f834bd6..d29fab95f1494 100644 --- a/Documentation/driver-api/nvdimm/btt.rst +++ b/Documentation/driver-api/nvdimm/btt.rst @@ -161,9 +161,8 @@ process:: nlanes = min(nfree, num_cpus) A lane number is obtained at the start of any IO, and is used for indexing into -all the on-disk and in-memory data structures for the duration of the IO. If -there are more CPUs than the max number of available lanes, than lanes are -protected by spinlocks. +all the on-disk and in-memory data structures for the duration of the IO. Lanes +are protected by mutexes. d. In-memory data structure: Read Tracking Table (RTT) diff --git a/Documentation/driver-api/uio-howto.rst b/Documentation/driver-api/uio-howto.rst index 907ffa3b38f5b..c08472dfbcfeb 100644 --- a/Documentation/driver-api/uio-howto.rst +++ b/Documentation/driver-api/uio-howto.rst @@ -246,10 +246,10 @@ the members are required, others are optional. hardware interrupt number. The flags given here will be used in the call to :c:func:`request_irq()`. -- ``int (*mmap)(struct uio_info *info, struct vm_area_struct *vma)``: +- ``int (*mmap_prepare)(struct uio_info *info, struct vm_area_desc *desc)``: Optional. If you need a special :c:func:`mmap()` function, you can set it here. If this pointer is not NULL, your - :c:func:`mmap()` will be called instead of the built-in one. + ``mmap_prepare`` will be called instead of the built-in one. - ``int (*open)(struct uio_info *info, struct inode *inode)``: Optional. You might want to have your own :c:func:`open()`, diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index db6167befb7b2..3c6e4f52f3e4e 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -23,13 +23,13 @@ fixes/update part 1.1 Stefani Seibold June 9 2009 1 Collecting System Information 1.1 Process-Specific Subdirectories 1.2 Kernel data - 1.3 IDE devices in /proc/ide - 1.4 Networking info in /proc/net - 1.5 SCSI info - 1.6 Parallel port info in /proc/parport - 1.7 TTY info in /proc/tty - 1.8 Miscellaneous kernel statistics in /proc/stat - 1.9 Ext4 file system parameters + 1.3 Networking info in /proc/net + 1.4 SCSI info + 1.5 Parallel port info in /proc/parport + 1.6 TTY info in /proc/tty + 1.7 Miscellaneous kernel statistics in /proc/stat + 1.8 Ext4 file system parameters + 1.9 /proc/consoles - Shows registered system consoles 2 Modifying System Parameters diff --git a/Documentation/mm/memfd_preservation.rst b/Documentation/mm/memfd_preservation.rst index a8a5b476afd30..c908a12dffa7c 100644 --- a/Documentation/mm/memfd_preservation.rst +++ b/Documentation/mm/memfd_preservation.rst @@ -11,7 +11,7 @@ Memfd Preservation ABI ====================== .. kernel-doc:: include/linux/kho/abi/memfd.h - :doc: DOC: memfd Live Update ABI + :doc: memfd Live Update ABI .. kernel-doc:: include/linux/kho/abi/memfd.h :internal: diff --git a/Documentation/netlink/specs/handshake.yaml b/Documentation/netlink/specs/handshake.yaml index 1024297b38513..ffec12b467597 100644 --- a/Documentation/netlink/specs/handshake.yaml +++ b/Documentation/netlink/specs/handshake.yaml @@ -125,6 +125,7 @@ operations: name: done doc: Handler reports handshake completion attribute-set: done + flags: [admin-perm] do: request: attributes: diff --git a/Documentation/netlink/specs/rt-link.yaml b/Documentation/netlink/specs/rt-link.yaml index f23aa5f229c50..644a8bd7b93c3 100644 --- a/Documentation/netlink/specs/rt-link.yaml +++ b/Documentation/netlink/specs/rt-link.yaml @@ -1351,6 +1351,12 @@ attribute-sets: - name: coupled-control type: u8 + - + name: broadcast-neigh + type: u8 + - + name: lacp-strict + type: u8 - name: bond-ad-info-attrs name-prefix: ifla-bond-ad-info- diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst index e700bf1d095c3..33ca5afafdf6d 100644 --- a/Documentation/networking/bonding.rst +++ b/Documentation/networking/bonding.rst @@ -619,6 +619,29 @@ min_links aggregator cannot be active without at least one available link, setting this option to 0 or to 1 has the exact same effect. +lacp_strict + + Specifies the fallback behavior of a bonding when LACP negotiation + fails on all slave links, i.e. when no slave is in the + Collecting_Distributing state, while at least `min_links` link still + reports carrier up. + + This option is only applicable to 802.3ad mode (mode 4). + + Valid values are: + + off or 0 + One interface of the bond is selected to be active, in order to + facilitate communication with peer devices that do not implement + LACP. + + on or 1 + Interfaces are only permitted to be made active if they have an + active LACP partner and have successfully reached + Collecting_Distributing state. + + The default value is 0 (off). + mode Specifies one of the bonding policies. The default is diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst index 9a99037270ff2..a4db8f7b3afb7 100644 --- a/Documentation/process/changes.rst +++ b/Documentation/process/changes.rst @@ -53,7 +53,7 @@ mcelog 0.6 mcelog --version mkimage (optional) 2017.01 mkimage --version nfs-utils 1.0.5 showmount --version openssl & libcrypto 1.0.0 openssl version -pahole 1.22 pahole --version +pahole 1.26 pahole --version pcmciautils 004 pccardctl -V PPP 2.4.0 pppd --version procps 3.2.0 ps --version @@ -147,6 +147,11 @@ Since Linux 5.2, if CONFIG_DEBUG_INFO_BTF is selected, the build system generates BTF (BPF Type Format) from DWARF in vmlinux, a bit later from kernel modules as well. This requires pahole v1.22 or later. +Since Linux 7.0, kfuncs annotated with KF_IMPLICIT_ARGS require pahole v1.26 +or later. Without it, such kfuncs will have incorrect BTF prototypes in +vmlinux, causing BPF programs to fail to load with a "func_proto incompatible +with vmlinux" error. Many sched_ext kfuncs are affected. + It is found in the 'dwarves' or 'pahole' distro packages or from https://fedorapeople.org/~acme/dwarves/. diff --git a/Documentation/process/threat-model.rst b/Documentation/process/threat-model.rst index f177b8d3c1caf..9dd8011dde828 100644 --- a/Documentation/process/threat-model.rst +++ b/Documentation/process/threat-model.rst @@ -176,7 +176,7 @@ regular bug: * problems seen only under development simulators, emulators, or combinations that do not exist on real systems at the time of reporting (issues involving tens of millions of threads, tens of thousands of CPUs, - unrealistic CPU frequencies, RAM sizes or disk capacities, network speeds. + unrealistic CPU frequencies, RAM sizes or disk capacities, network speeds). * issues whose reproduction requires hardware modification or emulation, including fake USB devices that pretend to be another one. diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst index f75f087639415..4b30cd63c5a5a 100644 --- a/Documentation/sound/alsa-configuration.rst +++ b/Documentation/sound/alsa-configuration.rst @@ -2383,6 +2383,12 @@ quirk_flags ``V(x) = k * x``; ``dB(x) = 20 * log10(x)``. Overrides bit 24 * bit 28: ``mixer_capture_linear_vol`` Similar to bit 27 but for capture streams. Overrides bit 25 + * bit 29: ``ifb_silence_on_empty`` + In implicit feedback mode, when an entire capture URB returns with + all iso_frame_desc[i].status != 0 (bytes==0), do not silently return + from snd_usb_handle_sync_urb. Instead fall through and enqueue a + packet_info containing only size-0 packets, so the OUT ring keeps + moving (emits silence). Needed by Behringer Flow 8 (1397:050c). This module supports multiple devices, autoprobe and hotplugging. diff --git a/Documentation/trace/eprobetrace.rst b/Documentation/trace/eprobetrace.rst index 89b5157cfab86..fe3602540569a 100644 --- a/Documentation/trace/eprobetrace.rst +++ b/Documentation/trace/eprobetrace.rst @@ -46,6 +46,10 @@ Synopsis of eprobe_events (x8/x16/x32/x64), VFS layer common type(%pd/%pD), "char", "string", "ustring", "symbol", "symstr" and "bitfield" are supported. + (STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to + a pointer to STRUCT and then derference the pointer defined by + ->MEMBER. Note that when this is used, the FIELD name does not + need to be prefixed with a '$'. Types ----- diff --git a/Documentation/trace/rv/monitor_sched.rst b/Documentation/trace/rv/monitor_sched.rst index 0b96d6e147c65..d3ba7edc202f6 100644 --- a/Documentation/trace/rv/monitor_sched.rst +++ b/Documentation/trace/rv/monitor_sched.rst @@ -36,7 +36,7 @@ Specifications -------------- The specifications included in sched are currently a work in progress, adapting the ones -defined in by Daniel Bristot in [1]. +defined by Daniel Bristot in [1]_. Currently we included the following: @@ -365,4 +365,7 @@ constraints when processing the events:: References ---------- -[1] - https://bristot.me/linux-task-model +.. [1] Daniel Bristot de Oliveira et al.: + `A thread synchronization model for the PREEMPT_RT Linux kernel + `_, + J. Syst. Archit., 2020. diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 331223761ffff..29a08bc059dd7 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -229,7 +229,6 @@ Code Seq# Include File Comments 'a' all linux/atm*.h, linux/sonet.h ATM on linux -'a' 00-0F drivers/crypto/qat/qat_common/adf_cfg_common.h conflict! qat driver 'b' 00-FF conflict! bit3 vme host bridge 'b' 00-0F linux/dma-buf.h conflict! diff --git a/MAINTAINERS b/MAINTAINERS index c8d4b913f26c1..53b0ed43cd395 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4373,6 +4373,13 @@ F: Documentation/devicetree/bindings/spi/axiado,ax3000-spi.yaml F: drivers/spi/spi-axiado.c F: drivers/spi/spi-axiado.h +AYN PLATFORM EC DRIVER +M: Derek J. Clark +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/sysfs-platform-ayn-ec +F: drivers/platform/x86/ayn-ec.c + AYANEO PLATFORM EC DRIVER M: Antheas Kapenekakis L: platform-driver-x86@vger.kernel.org @@ -15087,7 +15094,6 @@ M: Qunqin Zhao L: linux-crypto@vger.kernel.org S: Maintained F: drivers/char/tpm/tpm_loongson.c -F: drivers/crypto/loongson/ F: drivers/mfd/loongson-se.c F: include/linux/mfd/loongson-se.h diff --git a/Makefile b/Makefile index 408f070d49b8e..58f3b3e2dc1cf 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 7 PATCHLEVEL = 1 -SUBLEVEL = 0 +SUBLEVEL = 5 EXTRAVERSION = NAME = Baby Opossum Posse @@ -827,12 +827,6 @@ endif # KBUILD_EXTMOD # Defaults to vmlinux, but the arch makefile usually adds further targets all: vmlinux -CFLAGS_GCOV := -fprofile-arcs -ftest-coverage -ifdef CONFIG_CC_IS_GCC -CFLAGS_GCOV += -fno-tree-loop-im -endif -export CFLAGS_GCOV - # The arch Makefiles can override CC_FLAGS_FTRACE. We may also append it later. ifdef CONFIG_FUNCTION_TRACER CC_FLAGS_FTRACE := -pg @@ -965,6 +959,9 @@ KBUILD_CFLAGS += $(stackp-flags-y) ifdef CONFIG_FRAME_POINTER KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls KBUILD_RUSTFLAGS += -Cforce-frame-pointers=y +# Work around rustc bug on compilers without +# https://github.com/rust-lang/rust/pull/156980. +KBUILD_RUSTFLAGS += $(if $(call rustc-min-version,109800),,-Zllvm_module_flag=frame-pointer:u32:2:max) else # Some targets (ARM with Thumb2, for example), can't be built with frame # pointers. For those, we don't have FUNCTION_TRACER automatically @@ -1150,6 +1147,27 @@ endif # Ensure compilers do not transform certain loops into calls to wcslen() KBUILD_CFLAGS += -fno-builtin-wcslen +CFLAGS_GCOV := -fprofile-arcs -ftest-coverage +ifdef CONFIG_CC_IS_GCC +CFLAGS_GCOV += -fno-tree-loop-im +# Use atomic counter updates to avoid concurrent-access crashes in GCOV. +# Only enable if -fprofile-update=prefer-atomic does not introduce new +# undefined symbols (e.g. libatomic calls that the kernel cannot link). +CFLAGS_GCOV += $(call try-run,\ + echo 'long long x; void f(void){x++;}' | \ + $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -w -fprofile-arcs \ + -ftest-coverage -x c - -c -o "$$TMP.base" && \ + echo 'long long x; void f(void){x++;}' | \ + $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -w -fprofile-arcs \ + -ftest-coverage -fprofile-update=prefer-atomic \ + -x c - -c -o "$$TMP" && \ + $(NM) "$$TMP.base" | grep ' U ' > "$$TMP.ubase" || true ; \ + $(NM) "$$TMP" | grep ' U ' > "$$TMP.utest" || true ; \ + cmp -s "$$TMP.ubase" "$$TMP.utest",\ + -fprofile-update=prefer-atomic) +endif +export CFLAGS_GCOV + # change __FILE__ to the relative path to the source directory ifdef building_out_of_srctree KBUILD_CPPFLAGS += -fmacro-prefix-map=$(srcroot)/= diff --git a/arch/alpha/kernel/pci-sysfs.c b/arch/alpha/kernel/pci-sysfs.c index 3048758304b57..21833d1c867d7 100644 --- a/arch/alpha/kernel/pci-sysfs.c +++ b/arch/alpha/kernel/pci-sysfs.c @@ -11,6 +11,7 @@ */ #include +#include #include #include #include @@ -36,12 +37,16 @@ static int hose_mmap_page_range(struct pci_controller *hose, static int __pci_mmap_fits(struct pci_dev *pdev, int num, struct vm_area_struct *vma, int sparse) { + resource_size_t len = pci_resource_len(pdev, num); unsigned long nr, start, size; int shift = sparse ? 5 : 0; + if (!len) + return 0; + nr = vma_pages(vma); start = vma->vm_pgoff; - size = ((pci_resource_len(pdev, num) - 1) >> (PAGE_SHIFT - shift)) + 1; + size = ((len - 1) >> (PAGE_SHIFT - shift)) + 1; if (start < size && size - start >= nr) return 1; @@ -71,7 +76,11 @@ static int pci_mmap_resource(struct kobject *kobj, struct resource *res = attr->private; enum pci_mmap_state mmap_type; struct pci_bus_region bar; - int i; + int i, ret; + + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; for (i = 0; i < PCI_STD_NUM_BARS; i++) if (res == &pdev->resource[i]) diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index 1e8809ea000a3..6df2209541ac0 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -628,8 +628,7 @@ static inline unsigned long __clear_user(void __user *to, unsigned long n) return res; } -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER #define __clear_user __clear_user diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index a159120d1e42e..e3f550d628578 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -157,4 +157,4 @@ $(obj)/piggy_data: $(obj)/../Image FORCE $(obj)/piggy.o: $(obj)/piggy_data -CFLAGS_font.o := -Dstatic= +CFLAGS_font.o := -DBOOTLOADER diff --git a/arch/arm/boot/dts/broadcom/bcm2711-rpi-4-b.dts b/arch/arm/boot/dts/broadcom/bcm2711-rpi-4-b.dts index 353bb50ce5425..5469fa663526f 100644 --- a/arch/arm/boot/dts/broadcom/bcm2711-rpi-4-b.dts +++ b/arch/arm/boot/dts/broadcom/bcm2711-rpi-4-b.dts @@ -110,7 +110,7 @@ "GPIO26", "GPIO27", "RGMII_MDIO", - "RGMIO_MDC", + "RGMII_MDC", /* Used by BT module */ "CTS0", /* 30 */ "RTS0", diff --git a/arch/arm/boot/dts/broadcom/bcm2711-rpi-cm4-io.dts b/arch/arm/boot/dts/broadcom/bcm2711-rpi-cm4-io.dts index 6bc77dd48c0d9..1f4ebec68370c 100644 --- a/arch/arm/boot/dts/broadcom/bcm2711-rpi-cm4-io.dts +++ b/arch/arm/boot/dts/broadcom/bcm2711-rpi-cm4-io.dts @@ -56,7 +56,7 @@ "GPIO26", "GPIO27", "RGMII_MDIO", - "RGMIO_MDC", + "RGMII_MDC", /* Used by BT module */ "CTS0", "RTS0", diff --git a/arch/arm/boot/dts/nvidia/tegra124-nyan.dtsi b/arch/arm/boot/dts/nvidia/tegra124-nyan.dtsi index 974c76f007db4..89a749cb89338 100644 --- a/arch/arm/boot/dts/nvidia/tegra124-nyan.dtsi +++ b/arch/arm/boot/dts/nvidia/tegra124-nyan.dtsi @@ -14,6 +14,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <1>; + #size-cells = <1>; + }; + /* * Note that recent version of the device tree compiler (starting with * version 1.4.2) warn about this node containing a reg property, but diff --git a/arch/arm/boot/dts/nvidia/tegra124-venice2.dts b/arch/arm/boot/dts/nvidia/tegra124-venice2.dts index df98dc2a67b85..059ee6c5b13cd 100644 --- a/arch/arm/boot/dts/nvidia/tegra124-venice2.dts +++ b/arch/arm/boot/dts/nvidia/tegra124-venice2.dts @@ -18,6 +18,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <1>; + #size-cells = <1>; + }; + memory@80000000 { reg = <0x0 0x80000000 0x0 0x80000000>; }; diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi index 4e536e0252def..3fb5e2fb68777 100644 --- a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi +++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi @@ -45,10 +45,6 @@ assigned-clock-rates = <786432000>; }; -&cpu0 { - dc-supply = <®_gpio_dvfs>; -}; - &fec1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_enet1>, <&pinctrl_enet1_gpio>, <&pinctrl_enet1_mdio>; diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi b/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi index 2d6cf08d00f90..ca8e8e7350781 100644 --- a/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi +++ b/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi @@ -18,6 +18,11 @@ stdout-path = "serial2:115200n8"; }; + firmware { + #address-cells = <1>; + #size-cells = <1>; + }; + /* * The default coreboot on veyron devices ignores memory@0 nodes * and would instead create another memory node. diff --git a/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts b/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts index a5ea1431c3991..862782d20d10e 100644 --- a/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts +++ b/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts @@ -36,6 +36,56 @@ }; }; +&adc1 { + channel@0 { + reg = <0>; + st,min-sample-time-ns = <20000>; + label = "ain_aux0"; + }; + + channel@1 { + reg = <1>; + st,min-sample-time-ns = <20000>; + label = "ain_aux1"; + }; + + channel@3 { + reg = <3>; + st,min-sample-time-ns = <20000>; + label = "hpdcm1_i2"; + }; + + channel@5 { + reg = <5>; + st,min-sample-time-ns = <20000>; + label = "pout1_i"; + }; + + channel@9 { + reg = <9>; + st,min-sample-time-ns = <20000>; + label = "pout0_i"; + }; + + channel@13 { + reg = <13>; + st,min-sample-time-ns = <200000>; + label = "phint1_ain"; + }; + + channel@15 { + reg = <15>; + st,min-sample-time-ns = <20000>; + label = "hpdcm0_i1"; + }; + + channel@18 { + reg = <18>; + st,min-sample-time-ns = <20000>; + label = "hpdcm0_i2"; + }; +}; + &clk_hse { clock-frequency = <25000000>; }; @@ -46,3 +96,81 @@ assigned-clock-rates = <125000000>; /* Clock PLL3 to 625Mhz in tf-a. */ st,eth-clk-sel; }; + +&gpio0 { + gpio-line-names = "HSIN0_BIAS", "HSIN1_BIAS", "HSIN2_BIAS", "HSIN3_BIAS", + "HSIN4_BIAS", "", "STP_VREF0_LVL", "HSIN_VREF0_LVL", + "STP0_FB_BIAS", "STP1_FB_BIAS", "STP2_FB_BIAS", "STP3_FB_BIAS", + "", "", "", ""; +}; + +&gpio1 { + gpio-line-names = "HSIN5_BIAS", "HSIN6_BIAS", "HSIN7_BIAS", "HSIN8_BIAS", + "HSIN9_BIAS", "", "STP_VREF1_LVL", "HSIN_VREF1_LVL", + "STP4_FB_BIAS", "STP5_FB_BIAS", "STP6_FB_BIAS", "", + "", "", "LSIN8_BIAS", "LSIN9_BIAS"; +}; + +&gpiod { + gpio-line-names = "", "", "", "", + "", "", "", "", + "", "", "", "", + "STP_RESETN", "STP_ENABLEN", "HPOUT0", "HPOUT0_ALERTN"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_hog_d_mecsbc>; +}; + +&gpioe { + gpio-line-names = "HPOUT0_RESETN", "HPOUT1", "HPOUT1_ALERTN", "", + "", "", "HPOUT1_RESETN", + "LPOUT0", "LPOUT0_ALERTN", "LPOUT0_RESETN", + "LPOUT1", "LPOUT1_ALERTN", "LPOUT1_RESETN", + "LPOUT2", "LPOUT2_ALERTN", "LPOUT2_RESETN"; +}; + +&gpiof { + gpio-line-names = "LPOUT3", "LPOUT3_ALERTN", "LPOUT3_RESETN", + "LPOUT4", "LPOUT4_ALERTN", "LPOUT4_RESETN", + "", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpiog { + gpio-line-names = "LPOUT5", "LPOUT5_ALERTN", "", "LPOUT5_RESETN", + "", "", "", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpioh { + gpio-line-names = "", "", "", "", + "", "", "", "", + "GPIO0_RESETN", "", "", "", + "", "", "", ""; +}; + +&gpioi { + gpio-line-names = "", "", "", "", + "", "", "", "", + "HPDCM0_SLEEPN", "HPDCM1_SLEEPN", "GPIO1_RESETN", "", + "", "", "", ""; +}; + +&gpioj { + gpio-line-names = "HSIN10", "HSIN11", "HSIN12", "HSIN13", + "HSIN14", "HSIN15", "", "", + "", "", "", "", + "", "RTD_RESETN", "", ""; +}; + +&gpiok { + gpio-line-names = "", "", "HSIN0", "HSIN1", + "HSIN2", "HSIN3", "HSIN4", "HSIN5"; +}; + +&gpioz { + gpio-line-names = "", "", "", "HSIN6", + "HSIN7", "HSIN8", "HSIN9", ""; +}; + diff --git a/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts b/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts index 16b814c19350c..739cc18c3d3af 100644 --- a/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts +++ b/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts @@ -36,10 +36,154 @@ }; }; +&adc1 { + channel@0 { + reg = <0>; + st,min-sample-time-ns = <20000>; + label = "p24v_hpdcm"; + }; + + channel@1 { + reg = <1>; + st,min-sample-time-ns = <20000>; + label = "p24v_stp"; + }; + + channel@3 { + reg = <3>; + st,min-sample-time-ns = <200000>; + label = "phint1_ain"; + }; + + channel@5 { + reg = <5>; + st,min-sample-time-ns = <20000>; + label = "hpout1_i"; + }; + + channel@9 { + reg = <9>; + st,min-sample-time-ns = <20000>; + label = "hpout0_i"; + }; + + channel@13 { + reg = <13>; + st,min-sample-time-ns = <20000>; + label = "hpdcm0_i2"; + }; + + channel@15 { + reg = <15>; + st,min-sample-time-ns = <20000>; + label = "hpdcm1_i2"; + }; + + channel@18 { + reg = <18>; + st,min-sample-time-ns = <20000>; + label = "hpdcm0_i1"; + }; +}; + &clk_hse { clock-frequency = <24000000>; }; +&gpio0 { + gpio-line-names = "HSIN0_BIAS", "HSIN1_BIAS", "HSIN2_BIAS", "HSIN3_BIAS", + "", "", "HSIN_VREF0_LVL", "HSIN_VREF1_LVL", + "HSIN4_BIAS", "HSIN5_BIAS", "HSIN6_BIAS", "HSIN7_BIAS", + "", "", "", ""; +}; + +&gpio1 { + gpio-line-names = "HSIN8_BIAS", "HSIN9_BIAS", "HSIN10_BIAS", "HSIN11_BIAS", + "", "", "HSIN_VREF2_LVL", "HSIN_VREF3_LVL", + "HSIN12_BIAS", "HSIN13_BIAS", "HSIN14_BIAS", "HSIN15_BIAS", + "", "", "LSIN8_BIAS", "LSIN9_BIAS"; +}; + +&gpioa { + gpio-line-names = "", "", "", "", + "", "", "", "", + "", "", "GPIO1_RESETN", "", + "", "", "", "LPOUT5"; +}; + +&gpiob { + gpio-line-names = "", "", "", "", + "LPOUT4_RESETN", "", "", "", + "", "LPOUT4_ALERTN", "", "", + "", "", "", ""; +}; + +&gpioc { + gpio-line-names = "", "", "", "", + "", "", "", "", + "", "LPOUT4", "", "", + "", "", "", ""; +}; + +&gpiod { + gpio-line-names = "LPOUT2", "", "LPOUT3_RESETN", "", + "LPOUT2_ALERTN", "", "MECIO_ADDR0", "", + "HPOUT1_ALERTN", "HPOUT1_RESETN", "", "", + "", "", "HPOUT0", "HPOUT1"; +}; + +&gpioe { + gpio-line-names = "LPOUT0_RESETN", "", "", "", + "", "LPOUT3", "LPOUT5_ALERTN", "", + "", "", "", "", + "", "", "", "HSIN_RESETN"; +}; + +&gpiof { + gpio-line-names = "LPOUT5_RESETN", "", "", "HPOUT0_ALERTN", + "", "LPOUT1", "", "", + "", "", "", "", + "", "", "", ""; +}; + +&gpiog { + gpio-line-names = "", "", "", "HPOUT0_RESETN", + "", "", "LPOUT3_ALERTN", "", + "", "", "GPIO0_RESETN", "", + "", "", "", "LPOUT2_RESETN"; +}; + +&gpioh { + gpio-line-names = "", "", "", "", + "", "", "", "", + "", "LPOUT0", "", "", + "", "LPOUT0_ALERTN", "STP_ENABLEN", "STP_RESETN"; +}; + +&gpioi { + gpio-line-names = "", "", "", "", + "", "", "", "", + "", "", "SPE_RESETN", "", + "HPDCM0_SLEEPN", "", "", ""; +}; + +&gpioj { + gpio-line-names = "", "", "", "", + "", "", "", "MECIO_ADDR1", + "", "", "", "", + "", "", "", "LPOUT1_RESETN"; +}; + +&gpiok { + gpio-line-names = "", "", "RTD_RESETN", "", + "", "LPOUT1_ALERTN", "", ""; +}; + +&gpioz { + gpio-line-names = "", "", "", "", + "HPDCM1_SLEEPN", "", "", ""; +}; + &m_can1 { pinctrl-names = "default", "sleep"; pinctrl-0 = <&m_can1_pins_b>; diff --git a/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi b/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi index 915ba2526f451..1a4f5a523eb33 100644 --- a/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi +++ b/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi @@ -90,94 +90,45 @@ }; &adc { - /* ANA0, ANA1 are dedicated pins and don't need pinctrl: only in6. */ + /* ANA0, ANA1 are dedicated pins and don't need pinctrl. */ pinctrl-0 = <&adc12_pins_mecsbc>; pinctrl-names = "default"; vdd-supply = <&v3v3>; vdda-supply = <&v3v3>; - vref-supply = <&v3v3>; + vref-supply = <&vrefbuf>; status = "okay"; }; &adc1 { status = "okay"; - channel@0 { - reg = <0>; - /* 16.5 ck_cycles sampling time */ - st,min-sample-time-ns = <5000>; - label = "p24v_stp"; - }; - - channel@1 { - reg = <1>; - st,min-sample-time-ns = <5000>; - label = "p24v_hpdcm"; - }; - channel@2 { reg = <2>; - st,min-sample-time-ns = <5000>; + st,min-sample-time-ns = <20000>; label = "ain0"; }; - channel@3 { - reg = <3>; - st,min-sample-time-ns = <5000>; - label = "hpdcm1_i2"; - }; - - channel@5 { - reg = <5>; - st,min-sample-time-ns = <5000>; - label = "hpout1_i"; - }; - channel@6 { reg = <6>; - st,min-sample-time-ns = <5000>; + st,min-sample-time-ns = <20000>; label = "ain1"; }; - channel@9 { - reg = <9>; - st,min-sample-time-ns = <5000>; - label = "hpout0_i"; - }; - channel@10 { reg = <10>; - st,min-sample-time-ns = <5000>; + st,min-sample-time-ns = <200000>; label = "phint0_ain"; }; - channel@13 { - reg = <13>; - st,min-sample-time-ns = <5000>; - label = "phint1_ain"; - }; - - channel@15 { - reg = <15>; - st,min-sample-time-ns = <5000>; - label = "hpdcm0_i1"; - }; - channel@16 { reg = <16>; - st,min-sample-time-ns = <5000>; + st,min-sample-time-ns = <20000>; label = "lsin"; }; - channel@18 { - reg = <18>; - st,min-sample-time-ns = <5000>; - label = "hpdcm0_i2"; - }; - channel@19 { reg = <19>; - st,min-sample-time-ns = <5000>; + st,min-sample-time-ns = <20000>; label = "hpdcm1_i1"; }; }; @@ -187,14 +138,13 @@ channel@2 { reg = <2>; - /* 16.5 ck_cycles sampling time */ - st,min-sample-time-ns = <5000>; + st,min-sample-time-ns = <20000>; label = "ain2"; }; channel@6 { reg = <6>; - st,min-sample-time-ns = <5000>; + st,min-sample-time-ns = <20000>; label = "ain3"; }; }; @@ -223,69 +173,6 @@ }; }; -&gpiod { - gpio-line-names = "", "", "", "", - "", "", "", "", - "", "", "", "", - "STP_RESETN", "STP_ENABLEN", "HPOUT0", "HPOUT0_ALERTN"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_hog_d_mecsbc>; -}; - -&gpioe { - gpio-line-names = "HPOUT0_RESETN", "HPOUT1", "HPOUT1_ALERTN", "", - "", "", "HPOUT1_RESETN", - "LPOUT0", "LPOUT0_ALERTN", "GPOUT0_RESETN", - "LPOUT1", "LPOUT1_ALERTN", "GPOUT1_RESETN", - "LPOUT2", "LPOUT2_ALERTN", "GPOUT2_RESETN"; -}; - -&gpiof { - gpio-line-names = "LPOUT3", "LPOUT3_ALERTN", "GPOUT3_RESETN", - "LPOUT4", "LPOUT4_ALERTN", "GPOUT4_RESETN", - "", "", - "", "", "", "", - "", "", "", ""; -}; - -&gpiog { - gpio-line-names = "LPOUT5", "LPOUT5_ALERTN", "", "LPOUT5_RESETN", - "", "", "", "", - "", "", "", "", - "", "", "", ""; -}; - -&gpioh { - gpio-line-names = "", "", "", "", - "", "", "", "", - "GPIO0_RESETN", "", "", "", - "", "", "", ""; -}; - -&gpioi { - gpio-line-names = "", "", "", "", - "", "", "", "", - "HPDCM0_SLEEPN", "HPDCM1_SLEEPN", "GPIO1_RESETN", "", - "", "", "", ""; -}; - -&gpioj { - gpio-line-names = "HSIN10", "HSIN11", "HSIN12", "HSIN13", - "HSIN14", "HSIN15", "", "", - "", "", "", "", - "", "RTD_RESETN", "", ""; -}; - -&gpiok { - gpio-line-names = "", "", "HSIN0", "HSIN1", - "HSIN2", "HSIN3", "HSIN4", "HSIN5"; -}; - -&gpioz { - gpio-line-names = "", "", "", "HSIN6", - "HSIN7", "HSIN8", "HSIN9", ""; -}; - &i2c2 { pinctrl-names = "default"; pinctrl-0 = <&i2c2_pins_a>; @@ -297,10 +184,6 @@ reg = <0x20>; gpio-controller; #gpio-cells = <2>; - gpio-line-names = "HSIN0_BIAS", "HSIN1_BIAS", "HSIN2_BIAS", "HSIN3_BIAS", - "", "", "HSIN_VREF0_LVL", "HSIN_VREF1_LVL", - "HSIN4_BIAS", "HSIN5_BIAS", "HSIN6_BIAS", "HSIN9_BIAS", - "", "", "", ""; }; gpio1: gpio@21 { @@ -308,10 +191,6 @@ reg = <0x21>; gpio-controller; #gpio-cells = <2>; - gpio-line-names = "HSIN8_BIAS", "HSIN9_BIAS", "HSIN10_BIAS", "HSIN11_BIAS", - "", "", "HSIN_VREF2_LVL", "HSIN_VREF3_LVL", - "HSIN12_BIAS", "HSIN13_BIAS", "HSIN14_BIAS", "HSIN15_BIAS", - "", "", "LSIN8_BIAS", "LSIN9_BIAS"; }; }; @@ -412,6 +291,13 @@ phy-supply = <&v3v3>; }; +&vrefbuf { + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vdda-supply = <&v3v3>; + status = "okay"; +}; + &pinctrl { adc12_pins_mecsbc: adc12-ain-mecsbc-0 { pins { diff --git a/arch/arm/boot/dts/ti/omap/am335x-sl50.dts b/arch/arm/boot/dts/ti/omap/am335x-sl50.dts index 1dc4e344efd63..c5259eb7d21c7 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-sl50.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-sl50.dts @@ -558,8 +558,8 @@ remote-endpoint = <&codec_endpoint>; dai-format = "dsp_b"; - bitclock-master = <&codec_port>; - frame-master = <&codec_port>; + bitclock-master = <&codec_endpoint>; + frame-master = <&codec_endpoint>; bitclock-inversion; clocks = <&audio_mclk>; }; diff --git a/arch/arm/configs/axm55xx_defconfig b/arch/arm/configs/axm55xx_defconfig index 22b189090e15f..0952e5e94c5e7 100644 --- a/arch/arm/configs/axm55xx_defconfig +++ b/arch/arm/configs/axm55xx_defconfig @@ -195,7 +195,6 @@ CONFIG_PL320_MBOX=y # CONFIG_IOMMU_SUPPORT is not set CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y -CONFIG_EXT4_FS=y CONFIG_AUTOFS_FS=y CONFIG_FUSE_FS=y CONFIG_CUSE=y diff --git a/arch/arm/configs/dove_defconfig b/arch/arm/configs/dove_defconfig index e98c35df675e6..3d978ce34b54f 100644 --- a/arch/arm/configs/dove_defconfig +++ b/arch/arm/configs/dove_defconfig @@ -97,7 +97,6 @@ CONFIG_MV_XOR=y CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y # CONFIG_EXT4_FS_XATTR is not set -CONFIG_EXT4_FS=y CONFIG_ISO9660_FS=y CONFIG_JOLIET=y CONFIG_UDF_FS=m diff --git a/arch/arm/configs/ep93xx_defconfig b/arch/arm/configs/ep93xx_defconfig index 9f3c7324d1cf4..ce41dc8c435cb 100644 --- a/arch/arm/configs/ep93xx_defconfig +++ b/arch/arm/configs/ep93xx_defconfig @@ -105,7 +105,6 @@ CONFIG_EP93XX_DMA=y CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y # CONFIG_EXT4_FS_XATTR is not set -CONFIG_EXT4_FS=y CONFIG_VFAT_FS=y CONFIG_TMPFS=y CONFIG_JFFS2_FS=y diff --git a/arch/arm/configs/mmp2_defconfig b/arch/arm/configs/mmp2_defconfig index a9a212abfd69a..0ea608c75f226 100644 --- a/arch/arm/configs/mmp2_defconfig +++ b/arch/arm/configs/mmp2_defconfig @@ -54,7 +54,6 @@ CONFIG_RTC_DRV_MAX8925=y # CONFIG_RESET_CONTROLLER is not set CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y -CONFIG_EXT4_FS=y # CONFIG_DNOTIFY is not set CONFIG_MSDOS_FS=y CONFIG_FAT_DEFAULT_CODEPAGE=437 diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index bcc9aabc12028..aad12026b20d2 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -1155,7 +1155,7 @@ CONFIG_QCOM_COMMAND_DB=m CONFIG_QCOM_GSBI=y CONFIG_QCOM_OCMEM=m CONFIG_QCOM_RMTFS_MEM=m -CONFIG_QCOM_RPMH=y +CONFIG_QCOM_RPMH=m CONFIG_QCOM_SMEM=y CONFIG_QCOM_SMD_RPM=y CONFIG_QCOM_SMP2P=y @@ -1170,7 +1170,7 @@ CONFIG_KEYSTONE_NAVIGATOR_DMA=y CONFIG_TI_PRUSS=m CONFIG_RASPBERRYPI_POWER=y CONFIG_QCOM_CPR=y -CONFIG_QCOM_RPMHPD=y +CONFIG_QCOM_RPMHPD=m CONFIG_QCOM_RPMPD=y CONFIG_ROCKCHIP_PM_DOMAINS=y CONFIG_TI_SCI_PM_DOMAINS=y diff --git a/arch/arm/configs/mv78xx0_defconfig b/arch/arm/configs/mv78xx0_defconfig index d3a26efe766c4..c6b46141cb697 100644 --- a/arch/arm/configs/mv78xx0_defconfig +++ b/arch/arm/configs/mv78xx0_defconfig @@ -93,7 +93,6 @@ CONFIG_RTC_DRV_M41T80=y CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y # CONFIG_EXT4_FS_XATTR is not set -CONFIG_EXT4_FS=m CONFIG_ISO9660_FS=m CONFIG_JOLIET=y CONFIG_UDF_FS=m diff --git a/arch/arm/configs/sunxi_defconfig b/arch/arm/configs/sunxi_defconfig index a83d29fed1756..f4b8d8f7dbefb 100644 --- a/arch/arm/configs/sunxi_defconfig +++ b/arch/arm/configs/sunxi_defconfig @@ -170,7 +170,6 @@ CONFIG_ROOT_NFS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_CRYPTO_DEV_SUN4I_SS=y -CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG=y CONFIG_CRYPTO_DEV_SUN8I_CE=y CONFIG_CRYPTO_DEV_SUN8I_SS=y CONFIG_DMA_CMA=y diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index d6ae80b5df36f..1593cf3b98008 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -616,8 +616,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) } #define __clear_user(addr, n) (memset((void __force *)addr, 0, n), 0) #endif -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig index 6ea1bd55acf8d..a361840d7a047 100644 --- a/arch/arm/mach-imx/Kconfig +++ b/arch/arm/mach-imx/Kconfig @@ -227,27 +227,6 @@ config SOC_VF610 help This enables support for Freescale Vybrid VF610 processor. -choice - prompt "Clocksource for scheduler clock" - depends on SOC_VF610 - default VF_USE_ARM_GLOBAL_TIMER - - config VF_USE_ARM_GLOBAL_TIMER - bool "Use ARM Global Timer" - depends on ARCH_MULTI_V7 - select ARM_GLOBAL_TIMER - select CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK - help - Use the ARM Global Timer as clocksource - - config VF_USE_PIT_TIMER - bool "Use PIT timer" - select NXP_PIT_TIMER - help - Use SoC Periodic Interrupt Timer (PIT) as clocksource - -endchoice - endif endif diff --git a/arch/arm/mach-imx/cpu-imx31.c b/arch/arm/mach-imx/cpu-imx31.c index 35c544924e509..e81ef9e36a1fa 100644 --- a/arch/arm/mach-imx/cpu-imx31.c +++ b/arch/arm/mach-imx/cpu-imx31.c @@ -36,6 +36,7 @@ static int mx31_read_cpu_rev(void) void __iomem *iim_base; struct device_node *np; u32 i, srev; + int rev = IMX_CHIP_REVISION_UNKNOWN; np = of_find_compatible_node(NULL, NULL, "fsl,imx31-iim"); iim_base = of_iomap(np, 0); @@ -48,13 +49,17 @@ static int mx31_read_cpu_rev(void) for (i = 0; i < ARRAY_SIZE(mx31_cpu_type); i++) if (srev == mx31_cpu_type[i].srev) { + rev = mx31_cpu_type[i].rev; imx_print_silicon_rev(mx31_cpu_type[i].name, mx31_cpu_type[i].rev); - return mx31_cpu_type[i].rev; + goto out; } imx_print_silicon_rev("i.MX31", IMX_CHIP_REVISION_UNKNOWN); - return IMX_CHIP_REVISION_UNKNOWN; + +out: + iounmap(iim_base); + return rev; } int mx31_revision(void) diff --git a/arch/arm/mach-imx/mm-imx3.c b/arch/arm/mach-imx/mm-imx3.c index 0788c5cc7f9e6..9b0b014d7fe27 100644 --- a/arch/arm/mach-imx/mm-imx3.c +++ b/arch/arm/mach-imx/mm-imx3.c @@ -106,6 +106,7 @@ void __init imx31_init_early(void) arm_pm_idle = imx31_idle; np = of_find_compatible_node(NULL, NULL, "fsl,imx31-ccm"); mx3_ccm_base = of_iomap(np, 0); + of_node_put(np); BUG_ON(!mx3_ccm_base); } #endif /* ifdef CONFIG_SOC_IMX31 */ @@ -143,6 +144,7 @@ void __init imx35_init_early(void) arch_ioremap_caller = imx3_ioremap_caller; np = of_find_compatible_node(NULL, NULL, "fsl,imx35-ccm"); mx3_ccm_base = of_iomap(np, 0); + of_node_put(np); BUG_ON(!mx3_ccm_base); } #endif /* ifdef CONFIG_SOC_IMX35 */ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index fe60738e5943b..10c69474f2761 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1154,6 +1154,44 @@ config ARM64_ERRATUM_4193714 If unsure, say Y. +config ARM64_ERRATUM_4118414 + bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + default y + select ARM64_WORKAROUND_REPEAT_TLBI + help + This option adds a workaround for the following errata: + + * ARM C1-Premium erratum 4193780 + * ARM C1-Ultra erratum 4193780 + * ARM Cortex-A76 erratum 4193800 + * ARM Cortex-A76AE erratum 4193801 + * ARM Cortex-A77 erratum 4193798 + * ARM Cortex-A78 erratum 4193791 + * ARM Cortex-A78AE erratum 4193793 + * ARM Cortex-A78C erratum 4193794 + * ARM Cortex-A710 erratum 4193788 + * ARM Cortex-X1 erratum 4193791 + * ARM Cortex-X1C erratum 4193792 + * ARM Cortex-X2 erratum 4193788 + * ARM Cortex-X3 erratum 4193786 + * ARM Cortex-X4 erratum 4118414 + * ARM Cortex-X925 erratum 4193781 + * ARM Neoverse-N1 erratum 4193800 + * ARM Neoverse-N2 erratum 4193789 + * ARM Neoverse-V1 erratum 4193790 + * ARM Neoverse-V2 erratum 4193787 + * ARM Neoverse-V3 erratum 4193784 + * ARM Neoverse-V3AE erratum 4193784 + * Microsoft Azure Cobalt 100 4193789 + * NVIDIA Olympus erratum T410-OLY-1029 + + On affected cores, some memory accesses might not be completed by + broadcast TLB invalidation. + + This issue is also known as CVE-2025-10263. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi b/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi index 5afa8d92acbfb..b71dc1d78987a 100644 --- a/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi @@ -128,7 +128,8 @@ pio: pinctrl@2000000 { compatible = "allwinner,sun55i-a523-pinctrl"; reg = <0x2000000 0x800>; - interrupts = , + interrupts = , + , , , , diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-tqmls1028a-mbls1028a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1028a-tqmls1028a-mbls1028a.dtsi index cf338b2e80064..426a81e1743f1 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-tqmls1028a-mbls1028a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-tqmls1028a-mbls1028a.dtsi @@ -17,8 +17,8 @@ gpio0 = &gpio1; gpio1 = &gpio2; gpio2 = &gpio3; - mmc0 = &esdhc; /* SD-Card */ - mmc1 = &esdhc1; /* eMMC */ + mmc0 = &esdhc1; /* eMMC */ + mmc1 = &esdhc; /* SD-Card */ serial0 = &duart0; serial1 = &duart1; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts b/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts index 9d50d3e2761da..f95e9c19bfc75 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts +++ b/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts @@ -6,7 +6,7 @@ /dts-v1/; -#include "fsl-lx2160a.dtsi" +#include "fsl-lx2160a-rev2.dtsi" #include "fsl-lx2162a-sr-som.dtsi" / { diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts index bc62ae5ca812d..441e009030299 100644 --- a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts @@ -686,8 +686,6 @@ phy-names = "pcie-phy"; pinctrl-0 = <&pinctrl_pcieb>; pinctrl-names = "default"; - reset-gpio = <&lsio_gpio4 0 GPIO_ACTIVE_LOW>; - vpcie-supply = <®_pcieb>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts index e756fe5db56b6..dd59af0ebaae5 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts @@ -254,10 +254,6 @@ status = "okay"; }; -®_nvcc_sd { - sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; -}; - &uart1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; @@ -466,7 +462,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d0 MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d0 MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x19 - MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x40000d0 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0xd0 >; }; @@ -479,7 +475,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d4 MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d4 MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x19 - MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x40000d0 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0xd0 >; }; @@ -492,7 +488,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d6 MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d6 MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x19 - MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x40000d0 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0xd0 >; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi index 96987910609f1..4fb13d8ecfd45 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi @@ -342,7 +342,6 @@ regulator-name = "NVCC_SD (LDO5)"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; - sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; }; }; }; @@ -795,7 +794,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d0 /* SDIO_A_D2 */ MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d0 /* SDIO_A_D3 */ MX8MM_IOMUXC_SD2_WP_USDHC2_WP 0x400000d6 /* SDIO_A_WP */ - MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x40000090 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x90 >; }; @@ -808,7 +807,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d4 /* SDIO_A_D2 */ MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d4 /* SDIO_A_D3 */ MX8MM_IOMUXC_SD2_WP_USDHC2_WP 0x400000d6 /* SDIO_A_WP */ - MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x40000090 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x90 >; }; @@ -821,7 +820,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d6 /* SDIO_A_D2 */ MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d6 /* SDIO_A_D3 */ MX8MM_IOMUXC_SD2_WP_USDHC2_WP 0x400000d6 /* SDIO_A_WP */ - MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x40000090 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x90 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts index 5f37065bf43f3..a8f7c226a61f8 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts +++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts @@ -112,7 +112,7 @@ &ifm_pmic { interrupt-parent = <&gpio2>; - interrupts = <0 GPIO_ACTIVE_LOW>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; }; &iomuxc { diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts index 4dadfb7f78de2..43fd4d0041ef3 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts @@ -99,7 +99,7 @@ &ifm_pmic { interrupt-parent = <&gpio5>; - interrupts = <17 GPIO_ACTIVE_LOW>; + interrupts = <17 IRQ_TYPE_LEVEL_LOW>; }; &iomuxc { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts index dbbc0df0e3d1c..443e4fd5b9bfc 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts @@ -281,7 +281,7 @@ compatible = "nxp,pca9450c"; reg = <0x25>; interrupt-parent = <&gpio1>; - interrupts = <3 GPIO_ACTIVE_LOW>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; pinctrl-0 = <&pinctrl_pmic>; regulators { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-bl-osm-s.dts b/arch/arm64/boot/dts/freescale/imx8mp-kontron-bl-osm-s.dts index 0924ac50fd2db..29ce863403b88 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-bl-osm-s.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-bl-osm-s.dts @@ -49,7 +49,9 @@ reg_vcc_panel: regulator-vcc-panel { compatible = "regulator-fixed"; - gpio = <&gpio4 3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_vcc_panel>; + gpio = <&gpio5 3 GPIO_ACTIVE_HIGH>; enable-active-high; regulator-max-microvolt = <3300000>; regulator-min-microvolt = <3300000>; @@ -63,7 +65,7 @@ eeram@0 { compatible = "microchip,48l640"; reg = <0>; - spi-max-frequency = <20000000>; + spi-max-frequency = <16000000>; }; }; @@ -172,7 +174,7 @@ &gpio5 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio5>; - gpio-line-names = "I2S_BITCLK", "I2S_A_DATA_OUT", "I2S_MCLK", "PWM_2", + gpio-line-names = "I2S_BITCLK", "I2S_A_DATA_OUT", "I2S_MCLK", "VCC_PANEL_EN", "PWM_1", "PWM_0", "SPI_A_SCK", "CAN_ADDR1", "CAN_ADDR0", "SPI_A_CS0", "SPI_B_SCK", "SPI_B_SDO", "SPI_B_SDI", "SPI_B_CS0", "I2C_A_SCL", "I2C_A_SDA", @@ -329,4 +331,10 @@ MX8MP_IOMUXC_ECSPI1_MISO__GPIO5_IO08 0x46 /* CAN_ADR1 */ >; }; + + pinctrl_reg_vcc_panel: regvccpanelgrp { + fsl,pins = < + MX8MP_IOMUXC_SPDIF_TX__GPIO5_IO03 0x46 + >; + }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi index bc1a261bb000e..ea69c639b30b8 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi @@ -311,7 +311,6 @@ regulator-name = "NVCC_SD (LDO5)"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; - sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; }; }; }; @@ -815,7 +814,7 @@ MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1 0x1d0 /* SDIO_A_D1 */ MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2 0x1d0 /* SDIO_A_D2 */ MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3 0x1d0 /* SDIO_A_D3 */ - MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0x400001d0 + MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0x1d0 >; }; @@ -827,7 +826,7 @@ MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1 0x1d4 /* SDIO_A_D1 */ MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2 0x1d4 /* SDIO_A_D2 */ MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3 0x1d4 /* SDIO_A_D3 */ - MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0x400001d0 + MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0x1d0 >; }; @@ -839,7 +838,7 @@ MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1 0x1d6 /* SDIO_A_D1 */ MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2 0x1d6 /* SDIO_A_D2 */ MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3 0x1d6 /* SDIO_A_D3 */ - MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0x400001d0 + MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0x1d0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts index 890d1e525a489..6b0f944095e7c 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts @@ -609,6 +609,9 @@ reset-gpios = <&gpio4 29 GPIO_ACTIVE_LOW>; iov-supply = <®_vcc_1v8>; ldoin-supply = <®_vcc_3v3>; + assigned-clocks = <&clk IMX8MP_CLK_SAI3>; + assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>; + assigned-clock-rates = <12288000>; }; se97_1c: temperature-sensor@1c { diff --git a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts index 623169f7ddb5f..c07138055229c 100644 --- a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts +++ b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts @@ -742,7 +742,6 @@ phy-names = "pcie-phy"; pinctrl-0 = <&pinctrl_pcieb>; pinctrl-names = "default"; - vpcie-supply = <®_pcieb>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts index 290a49bea2f76..5dea66c1e7aa0 100644 --- a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts @@ -166,7 +166,7 @@ ptn5150_1: typec@1d { compatible = "nxp,ptn5150"; reg = <0x1d>; - int-gpios = <&gpiof 3 IRQ_TYPE_EDGE_FALLING>; + int-gpios = <&gpiof 3 GPIO_ACTIVE_LOW>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_typec1>; status = "disabled"; @@ -182,7 +182,7 @@ ptn5150_2: typec@3d { compatible = "nxp,ptn5150"; reg = <0x3d>; - int-gpios = <&gpiof 5 IRQ_TYPE_EDGE_FALLING>; + int-gpios = <&gpiof 5 GPIO_ACTIVE_LOW>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_typec2>; status = "disabled"; diff --git a/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi b/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi index 47895ff8cb244..2415487d3a5de 100644 --- a/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi @@ -631,12 +631,12 @@ */ pinctrl_atmel_adap: atmeladaptergrp { fsl,pins = , /* SODIMM 30 */ - ; /* SODIMM 28 */ + ; /* SODIMM 28 */ }; /* Atmel MXT touchsceen + boards with built-in Capacitive Touch Connector */ pinctrl_atmel_conn: atmelconnectorgrp { - fsl,pins = , /* SODIMM 107 */ + fsl,pins = , /* SODIMM 107 */ ; /* SODIMM 106 */ }; diff --git a/arch/arm64/boot/dts/freescale/imx94.dtsi b/arch/arm64/boot/dts/freescale/imx94.dtsi index c460ece6070f8..379429b3072ac 100644 --- a/arch/arm64/boot/dts/freescale/imx94.dtsi +++ b/arch/arm64/boot/dts/freescale/imx94.dtsi @@ -1481,7 +1481,7 @@ ddr-pmu@4e090dc0 { compatible = "fsl,imx94-ddr-pmu", "fsl,imx93-ddr-pmu"; reg = <0x0 0x4e090dc0 0x0 0x200>; - interrupts = ; + interrupts = ; }; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts index 041fd838fabba..49400bc9ba919 100644 --- a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts @@ -550,7 +550,7 @@ &pcie0_ep { pinctrl-0 = <&pinctrl_pcie0>; pinctrl-names = "default"; - vpcie-supply = <®_pcie0>; + vpcie-supply = <®_m2_pwr>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi b/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi index 8337c8b25f050..ff31f7c48cfb4 100644 --- a/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi @@ -452,7 +452,6 @@ /* Verdin UART_2, through RS485 transceiver */ &lpuart8 { - rs485-rts-active-low; rs485-rx-during-tx; linux,rs485-enabled-at-boot-time; diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi index 71394871d8dd0..4330296ed8e43 100644 --- a/arch/arm64/boot/dts/freescale/imx95.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi @@ -1852,8 +1852,9 @@ <0 0x4c360000 0 0x10000>, <0 0x4c340000 0 0x4000>; reg-names = "dbi", "config", "atu", "app"; - ranges = <0x81000000 0x0 0x00000000 0x0 0x6ff00000 0 0x00100000>, - <0x82000000 0x0 0x10000000 0x9 0x10000000 0 0x10000000>; + ranges = <0x43000000 0x9 0x00000000 0x9 0x00000000 0x0 0xe0000000>, + <0x82000000 0x0 0xe0000000 0x9 0xe0000000 0x0 0x10000000>, + <0x81000000 0x0 0x00000000 0x9 0xf0000000 0x0 0x00100000>; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; @@ -1927,8 +1928,9 @@ <0 0x4c3e0000 0 0x10000>, <0 0x4c3c0000 0 0x4000>; reg-names = "dbi", "config", "atu", "app"; - ranges = <0x81000000 0 0x00000000 0x8 0x8ff00000 0 0x00100000>, - <0x82000000 0 0x10000000 0xa 0x10000000 0 0x10000000>; + ranges = <0x43000000 0xa 0x00000000 0xa 0x00000000 0x0 0xe0000000>, + <0x82000000 0x0 0xe0000000 0xa 0xe0000000 0x0 0x10000000>, + <0x81000000 0x0 0x00000000 0xa 0xf0000000 0x0 0x00100000>; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; diff --git a/arch/arm64/boot/dts/freescale/s32g3.dtsi b/arch/arm64/boot/dts/freescale/s32g3.dtsi index e314f3c7d61d0..7e28dff53a865 100644 --- a/arch/arm64/boot/dts/freescale/s32g3.dtsi +++ b/arch/arm64/boot/dts/freescale/s32g3.dtsi @@ -862,7 +862,7 @@ swt8: watchdog@40500000 { compatible = "nxp,s32g3-swt", "nxp,s32g2-swt"; - reg = <40500000 0x1000>; + reg = <0x40500000 0x1000>; clocks = <&clks 0x3a>, <&clks 0x3b>, <&clks 0x3b>; clock-names = "counter", "module", "register"; status = "disabled"; diff --git a/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts b/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts index b2ce5edd9c6ac..bb0a993996241 100644 --- a/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts +++ b/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts @@ -460,7 +460,7 @@ regulators { ldo2: ldo2 { - regulator-min-microvolt = <1900000>; + regulator-min-microvolt = <2500000>; regulator-max-microvolt = <3100000>; }; diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile index 387faa9c2a09b..a86fb313b1a9d 100644 --- a/arch/arm64/boot/dts/mediatek/Makefile +++ b/arch/arm64/boot/dts/mediatek/Makefile @@ -47,6 +47,8 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-2g5.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-emmc.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-4e.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn13.dtbo +dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn14.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn15.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn18.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-emmc.dtbo @@ -70,18 +72,24 @@ mt7988a-bananapi-bpi-r4-2g5-sd-dtbs := \ dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-2g5-sd.dtb mt7988a-bananapi-bpi-r4-pro-8x-emmc-dtbs := \ mt7988a-bananapi-bpi-r4-pro-8x.dtb \ + mt7988a-bananapi-bpi-r4-pro-cn13.dtbo \ + mt7988a-bananapi-bpi-r4-pro-cn14.dtbo \ mt7988a-bananapi-bpi-r4-pro-emmc.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-emmc.dtb mt7988a-bananapi-bpi-r4-pro-8x-sd-dtbs := \ mt7988a-bananapi-bpi-r4-pro-8x.dtb \ + mt7988a-bananapi-bpi-r4-pro-cn13.dtbo \ + mt7988a-bananapi-bpi-r4-pro-cn14.dtbo \ mt7988a-bananapi-bpi-r4-pro-sd.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-sd.dtb mt7988a-bananapi-bpi-r4-pro-8x-sd-cn15-dtbs := \ mt7988a-bananapi-bpi-r4-pro-8x-sd.dtb \ + mt7988a-bananapi-bpi-r4-pro-cn14.dtbo \ mt7988a-bananapi-bpi-r4-pro-cn15.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-sd-cn15.dtb mt7988a-bananapi-bpi-r4-pro-8x-sd-cn18-dtbs := \ mt7988a-bananapi-bpi-r4-pro-8x-sd.dtb \ + mt7988a-bananapi-bpi-r4-pro-cn13.dtbo \ mt7988a-bananapi-bpi-r4-pro-cn18.dtbo dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-sd-cn18.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8167-pumpkin.dtb diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso new file mode 100644 index 0000000000000..973b76ba0cbfb --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 MediaTek Inc. + * Author: Frank Wunderlich + */ + +/* This enables key-m slot CN13 on pcie2(11280000 1L0) on BPI-R4-Pro */ + +/dts-v1/; +/plugin/; + +#include + +/ { + compatible = "bananapi,bpi-r4-pro", "mediatek,mt7988a"; +}; + +&{/soc/pinctrl@1001f000/pcie-2-hog} { + output-high; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso new file mode 100644 index 0000000000000..90b2a64459c31 --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 MediaTek Inc. + * Author: Frank Wunderlich + */ + +/* This enables key-m slot CN14 on pcie3(11290000 1L1) on BPI-R4-Pro */ + +/dts-v1/; +/plugin/; + +#include + +/ { + compatible = "bananapi,bpi-r4-pro", "mediatek,mt7988a"; +}; + +&{/soc/pinctrl@1001f000/pcie-3-hog} { + output-high; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi index a48132f094114..ff778e8305d5b 100644 --- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi @@ -437,14 +437,12 @@ pcie-2-hog { gpio-hog; gpios = <79 GPIO_ACTIVE_HIGH>; - output-high; }; /* 1L1 0=key-b (CN18), 1=key-m (CN14) */ pcie-3-hog { gpio-hog; gpios = <63 GPIO_ACTIVE_HIGH>; - output-high; }; pwm0_pins: pwm0-pins { diff --git a/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi b/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi index eadf1b2d156f2..95d4db2b79794 100644 --- a/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi @@ -525,6 +525,7 @@ &pcie { pinctrl-names = "default"; pinctrl-0 = <&pcie_pins>; + memory-region = <&wifi_restricted_dma_region>; pcie0: pcie@0,0 { device_type = "pci"; @@ -539,7 +540,6 @@ wifi: wifi@0,0 { reg = <0x10000 0 0 0 0x100000>, <0x10000 0 0x100000 0 0x100000>; - memory-region = <&wifi_restricted_dma_region>; }; }; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts b/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts index 683ac124523b3..1f5222d43e62c 100644 --- a/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts +++ b/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts @@ -18,6 +18,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + memory@80000000 { device_type = "memory"; reg = <0x0 0x80000000 0x0 0x80000000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts index f0b8c2c80aa50..a6d31650245d3 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts @@ -25,6 +25,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + memory@80000000 { device_type = "memory"; reg = <0x0 0x80000000 0x0 0xc0000000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi index 04a95b6658caa..18220cdac9f9b 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi @@ -3605,7 +3605,7 @@ <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_M>, <&bpmp TEGRA234_CLK_MGBE0_RX_PCS>, <&bpmp TEGRA234_CLK_MGBE0_TX_PCS>; - clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m", "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", "rx-pcs", "tx-pcs"; resets = <&bpmp TEGRA234_RESET_MGBE0_MAC>, @@ -3647,7 +3647,7 @@ <&bpmp TEGRA234_CLK_MGBE1_RX_PCS_M>, <&bpmp TEGRA234_CLK_MGBE1_RX_PCS>, <&bpmp TEGRA234_CLK_MGBE1_TX_PCS>; - clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m", "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", "rx-pcs", "tx-pcs"; resets = <&bpmp TEGRA234_RESET_MGBE1_MAC>, @@ -3689,7 +3689,7 @@ <&bpmp TEGRA234_CLK_MGBE2_RX_PCS_M>, <&bpmp TEGRA234_CLK_MGBE2_RX_PCS>, <&bpmp TEGRA234_CLK_MGBE2_TX_PCS>; - clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m", "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", "rx-pcs", "tx-pcs"; resets = <&bpmp TEGRA234_RESET_MGBE2_MAC>, @@ -3731,7 +3731,7 @@ <&bpmp TEGRA234_CLK_MGBE3_RX_PCS_M>, <&bpmp TEGRA234_CLK_MGBE3_RX_PCS>, <&bpmp TEGRA234_CLK_MGBE3_TX_PCS>; - clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m", "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", "rx-pcs", "tx-pcs"; resets = <&bpmp TEGRA234_RESET_MGBE3_MAC>, diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi index 06d8357bdf527..2d8e7e37830ff 100644 --- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi @@ -3277,50 +3277,6 @@ status = "disabled"; }; - gpio_main: gpio@c300000 { - compatible = "nvidia,tegra264-gpio"; - reg = <0x00 0x0c300000 0x0 0x4000>, - <0x00 0x0c310000 0x0 0x4000>; - reg-names = "security", "gpio"; - wakeup-parent = <&pmc>; - interrupts = , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - }; - serial@c4e0000 { compatible = "nvidia,tegra264-utc"; reg = <0x0 0x0c4e0000 0x0 0x8000>, @@ -3586,6 +3542,50 @@ status = "disabled"; }; + gpio_main: gpio@c300000 { + compatible = "nvidia,tegra264-gpio"; + reg = <0x00 0x0c300000 0x0 0x4000>, + <0x00 0x0c310000 0x0 0x4000>; + reg-names = "security", "gpio"; + wakeup-parent = <&pmc>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + i2c14: i2c@c410000 { compatible = "nvidia,tegra264-i2c"; reg = <0x00 0x0c410000 0x0 0x10000>; diff --git a/arch/arm64/boot/dts/qcom/eliza-mtp.dts b/arch/arm64/boot/dts/qcom/eliza-mtp.dts index 90f629800cb02..4708df8eb4a40 100644 --- a/arch/arm64/boot/dts/qcom/eliza-mtp.dts +++ b/arch/arm64/boot/dts/qcom/eliza-mtp.dts @@ -15,7 +15,7 @@ chassis-type = "handset"; aliases { - serial0 = &uart14; + serial0 = &uart13; }; chosen { @@ -382,7 +382,7 @@ <118 1>; /* NFC Secure I/O */ }; -&uart14 { +&uart13 { compatible = "qcom,geni-debug-uart"; status = "okay"; diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi index 7e97361a5dc58..9edfee5e1dde5 100644 --- a/arch/arm64/boot/dts/qcom/eliza.dtsi +++ b/arch/arm64/boot/dts/qcom/eliza.dtsi @@ -639,7 +639,7 @@ #size-cells = <2>; ranges; - uart14: serial@894000 { + uart13: serial@894000 { compatible = "qcom,geni-uart"; reg = <0x0 0x00894000 0x0 0x4000>; @@ -655,7 +655,7 @@ interconnect-names = "qup-core", "qup-config"; - pinctrl-0 = <&qup_uart14_default>; + pinctrl-0 = <&qup_uart13_default>; pinctrl-names = "default"; status = "disabled"; @@ -1338,7 +1338,7 @@ gpio-ranges = <&tlmm 0 0 184>; wakeup-parent = <&pdc>; - qup_uart14_default: qup-uart14-default-state { + qup_uart13_default: qup-uart13-default-state { /* TX, RX */ pins = "gpio18", "gpio19"; function = "qup2_se5"; diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts index 35aaf09e4e2b4..c98dfb3941fa3 100644 --- a/arch/arm64/boot/dts/qcom/glymur-crd.dts +++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts @@ -322,8 +322,6 @@ }; &usb_0 { - dr_mode = "host"; - status = "okay"; }; @@ -353,8 +351,6 @@ }; &usb_1 { - dr_mode = "host"; - status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi index 82436984485d4..63df60b7e010e 100644 --- a/arch/arm64/boot/dts/qcom/glymur.dtsi +++ b/arch/arm64/boot/dts/qcom/glymur.dtsi @@ -968,7 +968,7 @@ i2c19: i2c@88c000 { compatible = "qcom,geni-i2c"; reg = <0x0 0x0088c000 0x0 0x4000>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS @@ -3920,6 +3920,8 @@ snps,dis_u2_susphy_quirk; snps,dis_enblslpm_quirk; + usb-role-switch; + status = "disabled"; ports { @@ -3993,6 +3995,8 @@ snps,dis_u2_susphy_quirk; snps,dis_enblslpm_quirk; + usb-role-switch; + status = "disabled"; ports { @@ -4016,7 +4020,7 @@ }; }; - usb_hs: usb@a2f8800 { + usb_hs: usb@a200000 { compatible = "qcom,glymur-dwc3", "qcom,snps-dwc3"; reg = <0x0 0x0a200000 0x0 0xfc100>; @@ -5865,7 +5869,7 @@ #interconnect-cells = <2>; }; - system-cache-controller@20400000 { + system-cache-controller@21800000 { compatible = "qcom,glymur-llcc"; reg = <0x0 0x21800000 0x0 0x100000>, <0x0 0x21a00000 0x0 0x100000>, @@ -5906,7 +5910,7 @@ #interconnect-cells = <2>; }; - imem: sram@81e08000 { + imem: sram@81e08600 { compatible = "mmio-sram"; reg = <0x0 0x81e08600 0x0 0x300>; diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi index 051dee0764167..4ba751a65142b 100644 --- a/arch/arm64/boot/dts/qcom/hamoa.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -5744,18 +5744,18 @@ mdss_dp0_opp_table: opp-table { compatible = "operating-points-v2"; - opp-162000000 { - opp-hz = /bits/ 64 <162000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - opp-270000000 { opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; + required-opps = <&rpmhpd_opp_low_svs>; }; opp-540000000 { opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; required-opps = <&rpmhpd_opp_svs_l1>; }; @@ -5796,7 +5796,7 @@ <&usb_1_ss1_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>, <&usb_1_ss1_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>; - operating-points-v2 = <&mdss_dp1_opp_table>; + operating-points-v2 = <&mdss_dp0_opp_table>; power-domains = <&rpmhpd RPMHPD_MMCX>; @@ -5829,30 +5829,6 @@ }; }; }; - - mdss_dp1_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-162000000 { - opp-hz = /bits/ 64 <162000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - - opp-270000000 { - opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; - }; - - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - - opp-810000000 { - opp-hz = /bits/ 64 <810000000>; - required-opps = <&rpmhpd_opp_nom>; - }; - }; }; mdss_dp2: displayport-controller@ae9a000 { @@ -5885,7 +5861,7 @@ <&usb_1_ss2_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>, <&usb_1_ss2_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>; - operating-points-v2 = <&mdss_dp2_opp_table>; + operating-points-v2 = <&mdss_dp0_opp_table>; power-domains = <&rpmhpd RPMHPD_MMCX>; @@ -5917,30 +5893,6 @@ }; }; }; - - mdss_dp2_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-162000000 { - opp-hz = /bits/ 64 <162000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - - opp-270000000 { - opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; - }; - - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - - opp-810000000 { - opp-hz = /bits/ 64 <810000000>; - required-opps = <&rpmhpd_opp_nom>; - }; - }; }; mdss_dp3: displayport-controller@aea0000 { @@ -6004,19 +5956,14 @@ mdss_dp3_opp_table: opp-table { compatible = "operating-points-v2"; - opp-162000000 { - opp-hz = /bits/ 64 <162000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - opp-270000000 { opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; + required-opps = <&rpmhpd_opp_low_svs>; }; - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + required-opps = <&rpmhpd_opp_svs>; }; opp-810000000 { diff --git a/arch/arm64/boot/dts/qcom/ipq5424.dtsi b/arch/arm64/boot/dts/qcom/ipq5424.dtsi index f20cda4290949..876bf6a8b8ff0 100644 --- a/arch/arm64/boot/dts/qcom/ipq5424.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq5424.dtsi @@ -639,7 +639,7 @@ status = "disabled"; }; - usb2: usb2@1e00000 { + usb2: usb2@1ef8800 { compatible = "qcom,ipq5424-dwc3", "qcom,dwc3"; reg = <0 0x01ef8800 0 0x400>; #address-cells = <2>; @@ -730,7 +730,7 @@ status = "disabled"; }; - usb3: usb3@8a00000 { + usb3: usb3@8af8800 { compatible = "qcom,ipq5424-dwc3", "qcom,dwc3"; reg = <0 0x08af8800 0 0x400>; diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi index 7cc326aa1a1aa..14e362a4899b6 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi +++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi @@ -2538,7 +2538,11 @@ "qcom,inline-crypto-engine"; reg = <0x0 0x01d88000 0x0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; }; tcsr_mutex: hwlock@1f40000 { diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi index 988ca5f7c8a0e..522b2b61ce7d2 100644 --- a/arch/arm64/boot/dts/qcom/kodiak.dtsi +++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi @@ -2578,8 +2578,12 @@ ice: crypto@1d88000 { compatible = "qcom,sc7280-inline-crypto-engine", "qcom,inline-crypto-engine"; - reg = <0 0x01d88000 0 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + reg = <0 0x01d88000 0 0x18000>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi index fe6e763518230..3cd8d68cc4f91 100644 --- a/arch/arm64/boot/dts/qcom/lemans.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans.dtsi @@ -2694,6 +2694,352 @@ qcom,bcm-voters = <&apps_bcm_voter>; }; + pcie0: pcie@1c00000 { + compatible = "qcom,pcie-sa8775p"; + reg = <0x0 0x01c00000 0x0 0x3000>, + <0x0 0x40000000 0x0 0xf20>, + <0x0 0x40000f20 0x0 0xa8>, + <0x0 0x40001000 0x0 0x4000>, + <0x0 0x40100000 0x0 0x100000>, + <0x0 0x01c03000 0x0 0x1000>; + reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; + device_type = "pci"; + + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, + <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>; + bus-range = <0x00 0xff>; + + dma-coherent; + + linux,pci-domain = <0>; + num-lanes = <2>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_0_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>; + assigned-clock-rates = <19200000>; + + interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, + <0x100 &pcie_smmu 0x0001 0x1>; + + resets = <&gcc GCC_PCIE_0_BCR>, + <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; + reset-names = "pci", + "link_down"; + + power-domains = <&gcc PCIE_0_GDSC>; + + phys = <&pcie0_phy>; + phy-names = "pciephy"; + + eq-presets-8gts = /bits/ 16 <0x5555 0x5555>; + eq-presets-16gts = /bits/ 8 <0x55 0x55>; + + status = "disabled"; + + pcieport0: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie0_ep: pcie-ep@1c00000 { + compatible = "qcom,sa8775p-pcie-ep"; + reg = <0x0 0x01c00000 0x0 0x3000>, + <0x0 0x40000000 0x0 0xf20>, + <0x0 0x40000f20 0x0 0xa8>, + <0x0 0x40001000 0x0 0x4000>, + <0x0 0x40200000 0x0 0x1fe00000>, + <0x0 0x01c03000 0x0 0x1000>, + <0x0 0x40005000 0x0 0x2000>; + reg-names = "parf", "dbi", "elbi", "atu", "addr_space", + "mmio", "dma"; + + clocks = <&gcc GCC_PCIE_0_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + interrupts = , + , + ; + + interrupt-names = "global", "doorbell", "dma"; + + interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + dma-coherent; + iommus = <&pcie_smmu 0x0000 0x7f>; + resets = <&gcc GCC_PCIE_0_BCR>; + reset-names = "core"; + power-domains = <&gcc PCIE_0_GDSC>; + phys = <&pcie0_phy>; + phy-names = "pciephy"; + num-lanes = <2>; + linux,pci-domain = <0>; + + status = "disabled"; + }; + + pcie0_phy: phy@1c04000 { + compatible = "qcom,sa8775p-qmp-gen4x2-pcie-phy"; + reg = <0x0 0x1c04000 0x0 0x2000>; + + clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_EN>, + <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_0_PIPE_CLK>, + <&gcc GCC_PCIE_0_PIPEDIV2_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe", + "pipediv2"; + + assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + resets = <&gcc GCC_PCIE_0_PHY_BCR>; + reset-names = "phy"; + + #clock-cells = <0>; + clock-output-names = "pcie_0_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + + pcie1: pcie@1c10000 { + compatible = "qcom,pcie-sa8775p"; + reg = <0x0 0x01c10000 0x0 0x3000>, + <0x0 0x60000000 0x0 0xf20>, + <0x0 0x60000f20 0x0 0xa8>, + <0x0 0x60001000 0x0 0x4000>, + <0x0 0x60100000 0x0 0x100000>, + <0x0 0x01c13000 0x0 0x1000>; + reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; + device_type = "pci"; + + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>, + <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x1fd00000>; + bus-range = <0x00 0xff>; + + dma-coherent; + + linux,pci-domain = <1>; + num-lanes = <4>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_1_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; + assigned-clock-rates = <19200000>; + + interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, + <0x100 &pcie_smmu 0x0081 0x1>; + + resets = <&gcc GCC_PCIE_1_BCR>, + <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; + reset-names = "pci", + "link_down"; + + power-domains = <&gcc PCIE_1_GDSC>; + + phys = <&pcie1_phy>; + phy-names = "pciephy"; + + eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>; + eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>; + + status = "disabled"; + + pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie1_ep: pcie-ep@1c10000 { + compatible = "qcom,sa8775p-pcie-ep"; + reg = <0x0 0x01c10000 0x0 0x3000>, + <0x0 0x60000000 0x0 0xf20>, + <0x0 0x60000f20 0x0 0xa8>, + <0x0 0x60001000 0x0 0x4000>, + <0x0 0x60200000 0x0 0x1fe00000>, + <0x0 0x01c13000 0x0 0x1000>, + <0x0 0x60005000 0x0 0x2000>; + reg-names = "parf", "dbi", "elbi", "atu", "addr_space", + "mmio", "dma"; + + clocks = <&gcc GCC_PCIE_1_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + interrupts = , + , + ; + + interrupt-names = "global", "doorbell", "dma"; + + interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + dma-coherent; + iommus = <&pcie_smmu 0x80 0x7f>; + resets = <&gcc GCC_PCIE_1_BCR>; + reset-names = "core"; + power-domains = <&gcc PCIE_1_GDSC>; + phys = <&pcie1_phy>; + phy-names = "pciephy"; + num-lanes = <4>; + linux,pci-domain = <1>; + + status = "disabled"; + }; + + pcie1_phy: phy@1c14000 { + compatible = "qcom,sa8775p-qmp-gen4x4-pcie-phy"; + reg = <0x0 0x1c14000 0x0 0x4000>; + + clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_EN>, + <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_1_PIPE_CLK>, + <&gcc GCC_PCIE_1_PIPEDIV2_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe", + "pipediv2"; + + assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + resets = <&gcc GCC_PCIE_1_PHY_BCR>; + reset-names = "phy"; + + #clock-cells = <0>; + clock-output-names = "pcie_1_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + ufs_mem_hc: ufshc@1d84000 { compatible = "qcom,sa8775p-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; reg = <0x0 0x01d84000 0x0 0x3000>; @@ -2758,7 +3104,11 @@ compatible = "qcom,sa8775p-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0x0 0x01d88000 0x0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { @@ -5308,9 +5658,11 @@ <0x0 0x0aec2000 0x0 0x1c8>; clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, - <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>; + <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; #clock-cells = <1>; #phy-cells = <0>; @@ -5327,9 +5679,11 @@ <0x0 0x0aec5000 0x0 0x1c8>; clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK>, - <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>; + <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; #clock-cells = <1>; #phy-cells = <0>; @@ -8601,350 +8955,4 @@ }; }; }; - - pcie0: pcie@1c00000 { - compatible = "qcom,pcie-sa8775p"; - reg = <0x0 0x01c00000 0x0 0x3000>, - <0x0 0x40000000 0x0 0xf20>, - <0x0 0x40000f20 0x0 0xa8>, - <0x0 0x40001000 0x0 0x4000>, - <0x0 0x40100000 0x0 0x100000>, - <0x0 0x01c03000 0x0 0x1000>; - reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; - device_type = "pci"; - - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, - <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>; - bus-range = <0x00 0xff>; - - dma-coherent; - - linux,pci-domain = <0>; - num-lanes = <2>; - - interrupts = , - , - , - , - , - , - , - , - ; - interrupt-names = "msi0", - "msi1", - "msi2", - "msi3", - "msi4", - "msi5", - "msi6", - "msi7", - "global"; - #interrupt-cells = <1>; - interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>; - - clocks = <&gcc GCC_PCIE_0_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>; - assigned-clock-rates = <19200000>; - - interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>; - - resets = <&gcc GCC_PCIE_0_BCR>, - <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; - reset-names = "pci", - "link_down"; - - power-domains = <&gcc PCIE_0_GDSC>; - - phys = <&pcie0_phy>; - phy-names = "pciephy"; - - eq-presets-8gts = /bits/ 16 <0x5555 0x5555>; - eq-presets-16gts = /bits/ 8 <0x55 0x55>; - - status = "disabled"; - - pcieport0: pcie@0 { - device_type = "pci"; - reg = <0x0 0x0 0x0 0x0 0x0>; - bus-range = <0x01 0xff>; - - #address-cells = <3>; - #size-cells = <2>; - ranges; - }; - }; - - pcie0_ep: pcie-ep@1c00000 { - compatible = "qcom,sa8775p-pcie-ep"; - reg = <0x0 0x01c00000 0x0 0x3000>, - <0x0 0x40000000 0x0 0xf20>, - <0x0 0x40000f20 0x0 0xa8>, - <0x0 0x40001000 0x0 0x4000>, - <0x0 0x40200000 0x0 0x1fe00000>, - <0x0 0x01c03000 0x0 0x1000>, - <0x0 0x40005000 0x0 0x2000>; - reg-names = "parf", "dbi", "elbi", "atu", "addr_space", - "mmio", "dma"; - - clocks = <&gcc GCC_PCIE_0_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - interrupts = , - , - ; - - interrupt-names = "global", "doorbell", "dma"; - - interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - dma-coherent; - iommus = <&pcie_smmu 0x0000 0x7f>; - resets = <&gcc GCC_PCIE_0_BCR>; - reset-names = "core"; - power-domains = <&gcc PCIE_0_GDSC>; - phys = <&pcie0_phy>; - phy-names = "pciephy"; - num-lanes = <2>; - linux,pci-domain = <0>; - - status = "disabled"; - }; - - pcie0_phy: phy@1c04000 { - compatible = "qcom,sa8775p-qmp-gen4x2-pcie-phy"; - reg = <0x0 0x1c04000 0x0 0x2000>; - - clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&gcc GCC_PCIE_CLKREF_EN>, - <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>, - <&gcc GCC_PCIE_0_PIPE_CLK>, - <&gcc GCC_PCIE_0_PIPEDIV2_CLK>; - clock-names = "aux", - "cfg_ahb", - "ref", - "rchng", - "pipe", - "pipediv2"; - - assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>; - assigned-clock-rates = <100000000>; - - resets = <&gcc GCC_PCIE_0_PHY_BCR>; - reset-names = "phy"; - - #clock-cells = <0>; - clock-output-names = "pcie_0_pipe_clk"; - - #phy-cells = <0>; - - status = "disabled"; - }; - - pcie1: pcie@1c10000 { - compatible = "qcom,pcie-sa8775p"; - reg = <0x0 0x01c10000 0x0 0x3000>, - <0x0 0x60000000 0x0 0xf20>, - <0x0 0x60000f20 0x0 0xa8>, - <0x0 0x60001000 0x0 0x4000>, - <0x0 0x60100000 0x0 0x100000>, - <0x0 0x01c13000 0x0 0x1000>; - reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; - device_type = "pci"; - - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>, - <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x1fd00000>; - bus-range = <0x00 0xff>; - - dma-coherent; - - linux,pci-domain = <1>; - num-lanes = <4>; - - interrupts = , - , - , - , - , - , - , - , - ; - interrupt-names = "msi0", - "msi1", - "msi2", - "msi3", - "msi4", - "msi5", - "msi6", - "msi7", - "global"; - #interrupt-cells = <1>; - interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>; - - clocks = <&gcc GCC_PCIE_1_AUX_CLK>, - <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; - assigned-clock-rates = <19200000>; - - interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, - <0x100 &pcie_smmu 0x0081 0x1>; - - resets = <&gcc GCC_PCIE_1_BCR>, - <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; - reset-names = "pci", - "link_down"; - - power-domains = <&gcc PCIE_1_GDSC>; - - phys = <&pcie1_phy>; - phy-names = "pciephy"; - - eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>; - eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>; - - status = "disabled"; - - pcie@0 { - device_type = "pci"; - reg = <0x0 0x0 0x0 0x0 0x0>; - bus-range = <0x01 0xff>; - - #address-cells = <3>; - #size-cells = <2>; - ranges; - }; - }; - - pcie1_ep: pcie-ep@1c10000 { - compatible = "qcom,sa8775p-pcie-ep"; - reg = <0x0 0x01c10000 0x0 0x3000>, - <0x0 0x60000000 0x0 0xf20>, - <0x0 0x60000f20 0x0 0xa8>, - <0x0 0x60001000 0x0 0x4000>, - <0x0 0x60200000 0x0 0x1fe00000>, - <0x0 0x01c13000 0x0 0x1000>, - <0x0 0x60005000 0x0 0x2000>; - reg-names = "parf", "dbi", "elbi", "atu", "addr_space", - "mmio", "dma"; - - clocks = <&gcc GCC_PCIE_1_AUX_CLK>, - <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - interrupts = , - , - ; - - interrupt-names = "global", "doorbell", "dma"; - - interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - dma-coherent; - iommus = <&pcie_smmu 0x80 0x7f>; - resets = <&gcc GCC_PCIE_1_BCR>; - reset-names = "core"; - power-domains = <&gcc PCIE_1_GDSC>; - phys = <&pcie1_phy>; - phy-names = "pciephy"; - num-lanes = <4>; - linux,pci-domain = <1>; - - status = "disabled"; - }; - - pcie1_phy: phy@1c14000 { - compatible = "qcom,sa8775p-qmp-gen4x4-pcie-phy"; - reg = <0x0 0x1c14000 0x0 0x4000>; - - clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>, - <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_CLKREF_EN>, - <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>, - <&gcc GCC_PCIE_1_PIPE_CLK>, - <&gcc GCC_PCIE_1_PIPEDIV2_CLK>; - clock-names = "aux", - "cfg_ahb", - "ref", - "rchng", - "pipe", - "pipediv2"; - - assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>; - assigned-clock-rates = <100000000>; - - resets = <&gcc GCC_PCIE_1_PHY_BCR>; - reset-names = "phy"; - - #clock-cells = <0>; - clock-output-names = "pcie_1_pipe_clk"; - - #phy-cells = <0>; - - status = "disabled"; - }; }; diff --git a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts index c1899db46e714..5dc7c950e60d6 100644 --- a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts +++ b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts @@ -786,6 +786,11 @@ status = "okay"; }; +&rmtfs_mem { + /* Increase the size from 2.5 MiB to 6 MiB */ + reg = <0x0 0xe1f00000 0x0 0x600000>; +}; + &sdhc_2 { cd-gpios = <&tlmm 65 GPIO_ACTIVE_HIGH>; diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi index a6e463f3885dc..c4e373872e8c6 100644 --- a/arch/arm64/boot/dts/qcom/milos.dtsi +++ b/arch/arm64/boot/dts/qcom/milos.dtsi @@ -642,7 +642,7 @@ rmtfs_mem: rmtfs@e1f00000 { compatible = "qcom,rmtfs-mem"; - reg = <0x0 0xe1f00000 0x0 0x600000>; + reg = <0x0 0xe1f00000 0x0 0x280000>; no-map; qcom,client-id = <1>; diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi index 7b1d57460f1e6..fa13210fc539a 100644 --- a/arch/arm64/boot/dts/qcom/monaco.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco.dtsi @@ -2737,7 +2737,11 @@ compatible = "qcom,qcs8300-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0x0 0x01d88000 0x0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; }; crypto: crypto@1dfa000 { diff --git a/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi b/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi index 7a1e5f355c175..6b4747025b9f8 100644 --- a/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi +++ b/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi @@ -7,7 +7,7 @@ #include /{ - thermal_zones { + thermal-zones { pmh0104_i0_thermal: pmh0104-i0-thermal { polling-delay-passive = <100>; thermal-sensors = <&pmh0104_i_e0_temp_alarm>; diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi index a4b17564469ee..94a699cc26889 100644 --- a/arch/arm64/boot/dts/qcom/sc7180.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi @@ -1605,7 +1605,11 @@ compatible = "qcom,sc7180-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0 0x01d90000 0 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; ipa: ipa@1e40000 { diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi index f45deb188c6c0..e87e82fa73e9c 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi @@ -2647,7 +2647,7 @@ status = "disabled"; }; - usb_sec_qmpphy: phy@88ee000 { + usb_sec_qmpphy: phy@88ed000 { compatible = "qcom,sc8180x-qmp-usb3-dp-phy"; reg = <0 0x088ed000 0 0x3000>; diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi index bef3213165d66..252c301f0156c 100644 --- a/arch/arm64/boot/dts/qcom/sdm630.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi @@ -494,9 +494,12 @@ no-map; }; - adsp_mem: adsp-region@f6000000 { - reg = <0x0 0xf6000000 0x0 0x800000>; - no-map; + adsp_mem: adsp-region { + compatible = "shared-dma-pool"; + alloc-ranges = <0x0 0x80000000 0x0 0x80000000>; + alignment = <0x0 0x400000>; + size = <0x0 0x800000>; + reusable; }; qseecom_mem: qseecom-region@f6800000 { @@ -2459,27 +2462,27 @@ #address-cells = <1>; #size-cells = <0>; - compute-cb@1 { + compute-cb@3 { compatible = "qcom,fastrpc-compute-cb"; - reg = <1>; + reg = <3>; iommus = <&lpass_smmu 3>; }; - compute-cb@2 { + compute-cb@7 { compatible = "qcom,fastrpc-compute-cb"; - reg = <2>; + reg = <7>; iommus = <&lpass_smmu 7>; }; - compute-cb@3 { + compute-cb@8 { compatible = "qcom,fastrpc-compute-cb"; - reg = <3>; + reg = <8>; iommus = <&lpass_smmu 8>; }; - compute-cb@4 { + compute-cb@9 { compatible = "qcom,fastrpc-compute-cb"; - reg = <4>; + reg = <9>; iommus = <&lpass_smmu 9>; }; }; diff --git a/arch/arm64/boot/dts/qcom/sdm660.dtsi b/arch/arm64/boot/dts/qcom/sdm660.dtsi index 3fd6dd82a9927..0fca9662c64a1 100644 --- a/arch/arm64/boot/dts/qcom/sdm660.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm660.dtsi @@ -350,57 +350,57 @@ #address-cells = <1>; #size-cells = <0>; - compute-cb@5 { + compute-cb@3 { compatible = "qcom,fastrpc-compute-cb"; - reg = <5>; + reg = <3>; iommus = <&cdsp_smmu 3>; }; - compute-cb@6 { + compute-cb@4 { compatible = "qcom,fastrpc-compute-cb"; - reg = <6>; + reg = <4>; iommus = <&cdsp_smmu 4>; }; - compute-cb@7 { + compute-cb@5 { compatible = "qcom,fastrpc-compute-cb"; - reg = <7>; + reg = <5>; iommus = <&cdsp_smmu 5>; }; - compute-cb@8 { + compute-cb@6 { compatible = "qcom,fastrpc-compute-cb"; - reg = <8>; + reg = <6>; iommus = <&cdsp_smmu 6>; }; - compute-cb@9 { + compute-cb@7 { compatible = "qcom,fastrpc-compute-cb"; - reg = <9>; + reg = <7>; iommus = <&cdsp_smmu 7>; }; - compute-cb@10 { + compute-cb@8 { compatible = "qcom,fastrpc-compute-cb"; - reg = <10>; + reg = <8>; iommus = <&cdsp_smmu 8>; }; - compute-cb@11 { + compute-cb@9 { compatible = "qcom,fastrpc-compute-cb"; - reg = <11>; + reg = <9>; iommus = <&cdsp_smmu 9>; }; - compute-cb@12 { + compute-cb@10 { compatible = "qcom,fastrpc-compute-cb"; - reg = <12>; + reg = <10>; iommus = <&cdsp_smmu 10>; }; - compute-cb@13 { + compute-cb@11 { compatible = "qcom,fastrpc-compute-cb"; - reg = <13>; + reg = <11>; iommus = <&cdsp_smmu 11>; }; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso b/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso index dbe1911d8e470..678a17c805f74 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso @@ -16,7 +16,12 @@ status = "okay"; ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + csiphy0_ep: endpoint { data-lanes = <0 1 2 3>; remote-endpoint = <&ov8856_ep>; diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi index 6b7378cf4d493..b0b9baf01ec29 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi @@ -72,7 +72,7 @@ stdout-path = "serial0:115200n8"; - framebuffer: framebuffer@9d400000 { + framebuffer: framebuffer { compatible = "simple-framebuffer"; memory-region = <&cont_splash_mem>; diff --git a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts index 7d81198bc499c..b5fc93f0315fd 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts @@ -448,7 +448,7 @@ pinctrl-0 = <&ts_int_active &ts_reset_active>; pinctrl-1 = <&ts_int_suspend &ts_reset_suspend>; - pinctrl-names = "default", "suspend"; + pinctrl-names = "default", "sleep"; touchscreen-size-x = <1080>; touchscreen-size-y = <2160>; diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi index 1298485c42142..22354968aeaa3 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi @@ -271,7 +271,7 @@ &ipa { qcom,gsi-loader = "self"; memory-region = <&ipa_fw_mem>; - firmware-name = "qcom/sdm845/beryllium/ipa_fws.mbn"; + firmware-name = "qcom/sdm845/Xiaomi/beryllium/ipa_fws.mbn"; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 03bf30b53f289..47b028259d910 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -5373,8 +5373,12 @@ ice: crypto@1d88000 { compatible = "qcom,sm8450-inline-crypto-engine", "qcom,inline-crypto-engine"; - reg = <0 0x01d88000 0 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + reg = <0 0x01d88000 0 0x18000>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi index 912525e9bca6f..fe46a5d41fe07 100644 --- a/arch/arm64/boot/dts/qcom/sm8550.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi @@ -2465,7 +2465,11 @@ "qcom,inline-crypto-engine"; reg = <0 0x01d88000 0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; tcsr_mutex: hwlock@1f40000 { diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi index 1604bc8cff373..e2d98cf6adca8 100644 --- a/arch/arm64/boot/dts/qcom/sm8650.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi @@ -4081,7 +4081,11 @@ "qcom,inline-crypto-engine"; reg = <0 0x01d88000 0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index 18fb52c14acd7..2d2f029c2d6fe 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -2086,7 +2086,11 @@ "qcom,inline-crypto-engine"; reg = <0x0 0x01d88000 0x0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { @@ -3313,7 +3317,7 @@ "dsi_pll"; clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, - <&rpmhcc RPMH_CXO_CLK>; + <&bi_tcxo_div2>; clock-names = "iface", "ref"; diff --git a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts index a721734fbd5d0..803ba00809b25 100644 --- a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts +++ b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts @@ -20,10 +20,9 @@ stdout-path = "serial0:1843200n8"; }; - memory@60600000 { + memory@40000000 { device_type = "memory"; - /* first 518MiB is reserved for other purposes. */ - reg = <0x0 0x60600000 0x0 0x5fa00000>; + reg = <0x0 0x40000000 0x0 0x80000000>; }; memory@1080000000 { @@ -65,6 +64,77 @@ device_type = "memory"; reg = <0x1e 0x00000000 0x1 0x00000000>; }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + /* First 518 MiB is reserved for other purposes. */ + firmware@40000000 { + reg = <0x0 0x40000000 0x0 0x20600000>; + no-map; + }; + + /* Parameters set by IPL. */ + parameters@8c100000 { + reg = <0x0 0x8c100000 0x0 0x00008000>; + no-map; + }; + + /* TFA BL31. */ + tfa-bl31@8c200000 { + reg = <0x0 0x8c200000 0x0 0x00080000>; + no-map; + }; + + /* TEE TZ DRAM. */ + tee@8c400000 { + reg = <0x0 0x8c400000 0x0 0x02000000>; + no-map; + }; + + /* DRAM controller inline ECC areas */ + ecc@10cccc0000 { + reg = <0x10 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@12cccc0000 { + reg = <0x12 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@14cccc0000 { + reg = <0x14 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@16cccc0000 { + reg = <0x16 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@18cccc0000 { + reg = <0x18 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@1a66660000 { + reg = <0x1a 0x66660000 0x0 0x999a0000>; + no-map; + }; + + ecc@1c66660000 { + reg = <0x1c 0x66660000 0x0 0x999a0000>; + no-map; + }; + + ecc@1e66660000 { + reg = <0x1e 0x66660000 0x0 0x999a0000>; + no-map; + }; + }; }; &extal_clk { diff --git a/arch/arm64/boot/dts/renesas/r8a78000.dtsi b/arch/arm64/boot/dts/renesas/r8a78000.dtsi index 3ec1b53d27828..73be51787265c 100644 --- a/arch/arm64/boot/dts/renesas/r8a78000.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a78000.dtsi @@ -689,8 +689,40 @@ #interrupt-cells = <3>; #address-cells = <0>; interrupt-controller; - reg = <0 0x39000000 0 0x10000>, - <0 0x39080000 0 0x800000>; + #redistributor-regions = <32>; + reg = <0x0 0x39000000 0x0 0x10000>, + <0x0 0x39080000 0x0 0x40000>, + <0x0 0x390c0000 0x0 0x40000>, + <0x0 0x39100000 0x0 0x40000>, + <0x0 0x39140000 0x0 0x40000>, + <0x0 0x39180000 0x0 0x40000>, + <0x0 0x391c0000 0x0 0x40000>, + <0x0 0x39200000 0x0 0x40000>, + <0x0 0x39240000 0x0 0x40000>, + <0x0 0x39280000 0x0 0x40000>, + <0x0 0x392c0000 0x0 0x40000>, + <0x0 0x39300000 0x0 0x40000>, + <0x0 0x39340000 0x0 0x40000>, + <0x0 0x39380000 0x0 0x40000>, + <0x0 0x393c0000 0x0 0x40000>, + <0x0 0x39400000 0x0 0x40000>, + <0x0 0x39440000 0x0 0x40000>, + <0x0 0x39480000 0x0 0x40000>, + <0x0 0x394c0000 0x0 0x40000>, + <0x0 0x39500000 0x0 0x40000>, + <0x0 0x39540000 0x0 0x40000>, + <0x0 0x39580000 0x0 0x40000>, + <0x0 0x395c0000 0x0 0x40000>, + <0x0 0x39600000 0x0 0x40000>, + <0x0 0x39640000 0x0 0x40000>, + <0x0 0x39680000 0x0 0x40000>, + <0x0 0x396c0000 0x0 0x40000>, + <0x0 0x39700000 0x0 0x40000>, + <0x0 0x39740000 0x0 0x40000>, + <0x0 0x39780000 0x0 0x40000>, + <0x0 0x397c0000 0x0 0x40000>, + <0x0 0x39800000 0x0 0x40000>, + <0x0 0x39840000 0x0 0x40000>; interrupts = ; }; diff --git a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi index b7e669d8ba4d1..add917af5de78 100644 --- a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi @@ -397,7 +397,7 @@ &mdio { dp83825: ethernet-phy@0 { - compatible = "ethernet-phy-ieee802.3-c22"; + compatible = "ethernet-phy-id2000.a140"; reg = <0x0>; pinctrl-names = "default"; pinctrl-0 = <&phy_rst>; diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi index 4203b335a2633..973b4c5880e24 100644 --- a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi @@ -344,7 +344,7 @@ &mdio { dp83825: ethernet-phy@0 { - compatible = "ethernet-phy-ieee802.3-c22"; + compatible = "ethernet-phy-id2000.a140"; reg = <0x0>; pinctrl-names = "default"; pinctrl-0 = <&phy_rst>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi index 7eca1da78cffa..2f9e39671efc0 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi @@ -18,6 +18,11 @@ stdout-path = "serial2:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + /* * Power Tree * diff --git a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts index 7e499064e0357..985770e3a5e2c 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts @@ -245,7 +245,7 @@ compatible = "rockchip,rk809"; reg = <0x20>; interrupt-parent = <&gpio0>; - interrupts = ; + interrupts = ; clock-output-names = "rk808-clkout1", "rk808-clkout2"; assigned-clocks = <&cru I2S1_MCLKOUT_TX>; assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts b/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts index 718d1a2da8e56..90ce6f0e1dcff 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts +++ b/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts @@ -98,7 +98,7 @@ rgmii_phy0: ethernet-phy@1 { compatible = "ethernet-phy-ieee802.3-c22"; reg = <1>; - pinctrl-0 = <ð_phy0_reset_pin>; + pinctrl-0 = <&gmac0_rstn_gpio0_c5_pin>; pinctrl-names = "default"; }; }; @@ -132,8 +132,8 @@ &pinctrl { gmac0 { - eth_phy0_reset_pin: eth-phy0-reset-pin { - rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>; + gmac0_rstn_gpio0_c5_pin: gmac0-rstn-gpio0-c5-pin { + rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi index 28175d8200d57..e12a2a0cfb891 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi @@ -1281,10 +1281,10 @@ vdec: video-codec@27b00000 { compatible = "rockchip,rk3576-vdec"; - reg = <0x0 0x27b00100 0x0 0x500>, - <0x0 0x27b00000 0x0 0x100>, + reg = <0x0 0x27b00000 0x0 0x100>, + <0x0 0x27b00100 0x0 0x500>, <0x0 0x27b00600 0x0 0x100>; - reg-names = "function", "link", "cache"; + reg-names = "link", "function", "cache"; interrupts = ; clocks = <&cru ACLK_RKVDEC_ROOT>, <&cru HCLK_RKVDEC>, <&cru ACLK_RKVDEC_ROOT_BAK>, <&cru CLK_RKVDEC_CORE>, diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index 4fb8888c281c8..d4dc057e31f0e 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1355,10 +1355,10 @@ vdec0: video-codec@fdc38000 { compatible = "rockchip,rk3588-vdec"; - reg = <0x0 0xfdc38100 0x0 0x500>, - <0x0 0xfdc38000 0x0 0x100>, + reg = <0x0 0xfdc38000 0x0 0x100>, + <0x0 0xfdc38100 0x0 0x500>, <0x0 0xfdc38600 0x0 0x100>; - reg-names = "function", "link", "cache"; + reg-names = "link", "function", "cache"; interrupts = ; clocks = <&cru ACLK_RKVDEC0>, <&cru HCLK_RKVDEC0>, <&cru CLK_RKVDEC0_CA>, <&cru CLK_RKVDEC0_CORE>, <&cru CLK_RKVDEC0_HEVC_CA>; @@ -1387,10 +1387,10 @@ vdec1: video-codec@fdc40000 { compatible = "rockchip,rk3588-vdec"; - reg = <0x0 0xfdc40100 0x0 0x500>, - <0x0 0xfdc40000 0x0 0x100>, + reg = <0x0 0xfdc40000 0x0 0x100>, + <0x0 0xfdc40100 0x0 0x500>, <0x0 0xfdc40600 0x0 0x100>; - reg-names = "function", "link", "cache"; + reg-names = "link", "function", "cache"; interrupts = ; clocks = <&cru ACLK_RKVDEC1>, <&cru HCLK_RKVDEC1>, <&cru CLK_RKVDEC1_CA>, <&cru CLK_RKVDEC1_CORE>, <&cru CLK_RKVDEC1_HEVC_CA>; diff --git a/arch/arm64/boot/dts/st/stm32mp251.dtsi b/arch/arm64/boot/dts/st/stm32mp251.dtsi index 673fbc5632e69..9c63fdb5a885a 100644 --- a/arch/arm64/boot/dts/st/stm32mp251.dtsi +++ b/arch/arm64/boot/dts/st/stm32mp251.dtsi @@ -1202,7 +1202,7 @@ sai1: sai@40290000 { compatible = "st,stm32mp25-sai"; - reg = <0x40290000 0x4>, <0x4029a3f0 0x10>; + reg = <0x40290000 0x4>, <0x402903f0 0x10>; ranges = <0 0x40290000 0x400>; #address-cells = <1>; #size-cells = <1>; @@ -1236,7 +1236,7 @@ sai2: sai@402a0000 { compatible = "st,stm32mp25-sai"; - reg = <0x402a0000 0x4>, <0x402aa3f0 0x10>; + reg = <0x402a0000 0x4>, <0x402a03f0 0x10>; ranges = <0 0x402a0000 0x400>; #address-cells = <1>; #size-cells = <1>; @@ -1270,7 +1270,7 @@ sai3: sai@402b0000 { compatible = "st,stm32mp25-sai"; - reg = <0x402b0000 0x4>, <0x402ba3f0 0x10>; + reg = <0x402b0000 0x4>, <0x402b03f0 0x10>; ranges = <0 0x402b0000 0x400>; #address-cells = <1>; #size-cells = <1>; @@ -1362,7 +1362,7 @@ sai4: sai@40340000 { compatible = "st,stm32mp25-sai"; - reg = <0x40340000 0x4>, <0x4034a3f0 0x10>; + reg = <0x40340000 0x4>, <0x403403f0 0x10>; ranges = <0 0x40340000 0x400>; #address-cells = <1>; #size-cells = <1>; diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts index c1e9067b3bdd5..821a9705bb7d4 100644 --- a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts @@ -154,6 +154,7 @@ gpios = <&main_gpio0 31 GPIO_ACTIVE_HIGH>; states = <1800000 0x0>, <3300000 0x1>; + bootph-all; }; leds { @@ -407,6 +408,7 @@ pinctrl-single,pins = < AM62AX_IOPAD(0x07c, PIN_OUTPUT, 7) /* (N22) GPMC0_CLK.GPIO0_31 */ >; + bootph-all; }; main_ecap0_pins_default: main-ecap0-default-pins { diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 7b518e81dd15b..1b9f0cda1336d 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -97,8 +97,10 @@ #define ARM_CPU_PART_CORTEX_X925 0xD85 #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 +#define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E #define ARM_CPU_PART_C1_PRO 0xD8B +#define ARM_CPU_PART_C1_PREMIUM 0xD90 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -189,8 +191,10 @@ #define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925) #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) +#define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) +#define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8d06b62e7188c..e9b2b0c40ec63 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -157,5 +157,6 @@ extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl); extern unsigned long kvm_nvhe_sym(hyp_nr_cpus); +extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr); #endif /* __ARM64_KVM_HYP_H__ */ diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index cdf3e8422ea12..012d711034d17 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -388,6 +388,14 @@ struct s1_walk_result { bool failed; }; +static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) +{ + wr->fst = fst; + wr->ptw = s1ptw; + wr->s2 = s1ptw; + wr->failed = true; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/include/asm/tlbbatch.h b/arch/arm64/include/asm/tlbbatch.h index 6297631532e59..767f35ea62b39 100644 --- a/arch/arm64/include/asm/tlbbatch.h +++ b/arch/arm64/include/asm/tlbbatch.h @@ -2,17 +2,11 @@ #ifndef _ARCH_ARM64_TLBBATCH_H #define _ARCH_ARM64_TLBBATCH_H -#include - struct arch_tlbflush_unmap_batch { -#ifdef CONFIG_ARM64_ERRATUM_4193714 /* - * Track CPUs that need SME DVMSync on completion of this batch. - * Otherwise, the arm64 HW can do tlb shootdown, so we don't need to - * record cpumask for sending IPI + * For arm64, HW can do TLB shootdown, so we don't need to record a + * cpumask for sending IPIs. */ - cpumask_var_t cpumask; -#endif }; #endif /* _ARCH_ARM64_TLBBATCH_H */ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index c0bf5b3980411..57b4eda6a72bb 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -82,6 +82,8 @@ static inline unsigned long get_trans_granule(void) #ifdef CONFIG_ARM64_ERRATUM_4193714 +extern cpumask_t sme_active_cpus; + void sme_do_dvmsync(const struct cpumask *mask); static inline void sme_dvmsync(struct mm_struct *mm) @@ -92,42 +94,12 @@ static inline void sme_dvmsync(struct mm_struct *mm) sme_do_dvmsync(mm_cpumask(mm)); } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) +static inline void sme_dvmsync_batch(void) { if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) return; - /* - * Order the mm_cpumask() read after the hardware DVMSync. - */ - dsb(ish); - if (cpumask_empty(mm_cpumask(mm))) - return; - - /* - * Allocate the batch cpumask on first use. Fall back to an immediate - * IPI for this mm in case of failure. - */ - if (!cpumask_available(batch->cpumask) && - !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { - sme_do_dvmsync(mm_cpumask(mm)); - return; - } - - cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); -} - -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) -{ - if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - return; - - if (!cpumask_available(batch->cpumask)) - return; - - sme_do_dvmsync(batch->cpumask); - cpumask_clear(batch->cpumask); + sme_do_dvmsync(&sme_active_cpus); } #else @@ -135,11 +107,7 @@ static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) static inline void sme_dvmsync(struct mm_struct *mm) { } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) -{ -} -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void sme_dvmsync_batch(void) { } @@ -285,11 +253,11 @@ static inline void __tlbi_sync_s1ish(struct mm_struct *mm) sme_dvmsync(mm); } -static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void __tlbi_sync_s1ish_batch(void) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); - sme_dvmsync_batch(batch); + sme_dvmsync_batch(); } static inline void __tlbi_sync_s1ish_kernel(void) @@ -434,7 +402,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __tlbi_sync_s1ish_batch(batch); + __tlbi_sync_s1ish_batch(); } /* @@ -722,7 +690,6 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); - sme_dvmsync_add_pending(batch, mm); } static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index b0c83a08dda97..9f5bd9c69c249 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -456,8 +456,7 @@ do { \ unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label); \ } while (0) -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n); static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n) diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index 5891f92c20354..681aa2bbc3995 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -448,12 +448,14 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 apci_id, return *pcpu; } + set_cpu_present(*pcpu, true); return 0; } EXPORT_SYMBOL(acpi_map_cpu); int acpi_unmap_cpu(int cpu) { + set_cpu_present(cpu, false); return 0; } EXPORT_SYMBOL(acpi_unmap_cpu); diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 5377e4c2eba2b..4b0d5d9328972 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -340,7 +340,37 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), }, #endif - {}, +#ifdef CONFIG_ARM64_ERRATUM_4118414 + { + ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) { + MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM), + MIDR_ALL_VERSIONS(MIDR_C1_ULTRA), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100), + {} + })), + }, +#endif + {} }; #endif @@ -693,7 +723,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI { - .desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009", + .desc = "Broken broadcast TLBI completion", .capability = ARM64_WORKAROUND_REPEAT_TLBI, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index 6325db1a2179c..28af4bcd97a50 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -38,10 +38,10 @@ SYM_FUNC_END(fpsimd_load_state) * * x0 - pointer to buffer for state * x1 - pointer to storage for FPSR - * x2 - Save FFR if non-zero + * w2 - Save FFR if non-zero */ SYM_FUNC_START(sve_save_state) - sve_save 0, x1, x2, 3 + sve_save 0, x1, w2, 3 ret SYM_FUNC_END(sve_save_state) @@ -50,10 +50,10 @@ SYM_FUNC_END(sve_save_state) * * x0 - pointer to buffer for state * x1 - pointer to storage for FPSR - * x2 - Restore FFR if non-zero + * w2 - Restore FFR if non-zero */ SYM_FUNC_START(sve_load_state) - sve_load 0, x1, x2, 4 + sve_load 0, x1, w2, 4 ret SYM_FUNC_END(sve_load_state) @@ -103,13 +103,13 @@ SYM_FUNC_END(sme_set_vq) * Save the ZA and ZT state * * x0 - pointer to buffer for state - * x1 - number of ZT registers to save + * w1 - number of ZT registers to save */ SYM_FUNC_START(sme_save_state) _sme_rdsvl 2, 1 // x2 = VL/8 sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - cbz x1, 1f + cbz w1, 1f _str_zt 0 1: ret @@ -119,13 +119,13 @@ SYM_FUNC_END(sme_save_state) * Load the ZA and ZT state * * x0 - pointer to buffer for state - * x1 - number of ZT registers to save + * w1 - number of ZT registers to save */ SYM_FUNC_START(sme_load_state) _sme_rdsvl 2, 1 // x2 = VL/8 sme_load_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - cbz x1, 1f + cbz w1, 1f _ldr_zt 0 1: ret diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 60a45d600b460..ab3b63621fd00 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1366,6 +1366,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) * SME/CME erratum handling. */ static cpumask_t sme_dvmsync_cpus; +cpumask_t sme_active_cpus; /* * These helpers are only called from non-preemptible contexts, so @@ -1379,13 +1380,15 @@ void sme_set_active(void) return; cpumask_set_cpu(cpu, mm_cpumask(current->mm)); + cpumask_set_cpu(cpu, &sme_active_cpus); /* * A subsequent (post ERET) SME access may use a stale address * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for - * the completion of cpumask_set_cpu() above as it appears in program - * order before the SME access. The post-TLBI+DSB read of mm_cpumask() - * will lead to the IPI being issued. + * the completion of the cpumask_set_cpu() operations above as they + * appear in program order before the SME access. The post-TLBI+DSB + * read of mm_cpumask() or sme_active_cpus will lead to the IPI being + * issued. * * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 */ @@ -1403,6 +1406,7 @@ void sme_clear_active(void) * completed on entering EL1. */ cpumask_clear_cpu(cpu, mm_cpumask(current->mm)); + cpumask_clear_cpu(cpu, &sme_active_cpus); } static void sme_dvmsync_ipi(void *unused) diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index ab76b36dce820..73cce8ac83681 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -559,6 +559,15 @@ int hw_breakpoint_arch_parse(struct perf_event *bp, else alignment_mask = 0x7; offset = hw->address & alignment_mask; + + /* + * BAS is an 8-bit field in WCR/BCR; the shift below would + * silently drop the high bits of ctrl.len when offset + len + * exceeds 8, programming hardware to watch fewer bytes than + * the user requested. + */ + if (((u32)hw->ctrl.len << offset) > ARM_BREAKPOINT_LEN_8) + return -EINVAL; } hw->address &= ~alignment_mask; diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 033643cd4e5ed..581f80e9b9b73 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -341,41 +341,8 @@ void flush_thread(void) flush_gcs(); } -#ifdef CONFIG_ARM64_ERRATUM_4193714 - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ - /* - * Clear the inherited cpumask with memset() to cover both cases where - * cpumask_var_t is a pointer or an array. It will be allocated lazily - * in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y. - */ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - memset(&dst->tlb_ubc.arch.cpumask, 0, - sizeof(dst->tlb_ubc.arch.cpumask)); -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - free_cpumask_var(tsk->tlb_ubc.arch.cpumask); -} - -#else - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ -} - -#endif /* CONFIG_ARM64_ERRATUM_4193714 */ - void arch_release_task_struct(struct task_struct *tsk) { - arch_release_tlbbatch_mask(tsk); fpsimd_release_task(tsk); } @@ -391,8 +358,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) *dst = *src; - arch_dup_tlbbatch_mask(dst); - /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 1aa324104afb4..24f8448e1fbba 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -535,23 +535,13 @@ void arch_unregister_cpu(int cpu) { acpi_handle acpi_handle = acpi_get_processor_handle(cpu); struct cpu *c = &per_cpu(cpu_devices, cpu); - acpi_status status; unsigned long long sta; - - if (!acpi_handle) { - pr_err_once("Removing a CPU without associated ACPI handle\n"); - return; - } + acpi_status status; status = acpi_evaluate_integer(acpi_handle, "_STA", NULL, &sta); - if (ACPI_FAILURE(status)) - return; - - /* For now do not allow anything that looks like physical CPU HP */ - if (cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) { + if (!ACPI_FAILURE(status) && + cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) pr_err_once("Changing CPU present bit is not supported\n"); - return; - } unregister_cpu(c); } @@ -566,6 +556,11 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu) } EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc); +static bool acpi_cpu_is_present(int cpu) +{ + return acpi_cpu_get_madt_gicc(cpu)->flags & ACPI_MADT_ENABLED; +} + /* * acpi_map_gic_cpu_interface - parse processor MADT entry * @@ -670,6 +665,10 @@ static void __init acpi_parse_and_init_cpus(void) early_map_cpu_to_node(i, acpi_numa_get_nid(i)); } #else +static bool acpi_cpu_is_present(int cpu) +{ + return false; +} #define acpi_parse_and_init_cpus(...) do { } while (0) #endif @@ -808,7 +807,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (err) continue; - set_cpu_present(cpu, true); + if (acpi_disabled || acpi_cpu_is_present(cpu)) + set_cpu_present(cpu, true); numa_store_cpu_info(cpu); } } diff --git a/arch/arm64/kernel/static_call.c b/arch/arm64/kernel/static_call.c index 8b3a19e108713..c126edced022f 100644 --- a/arch/arm64/kernel/static_call.c +++ b/arch/arm64/kernel/static_call.c @@ -2,6 +2,7 @@ #include #include #include +#include void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) { diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 9453321ef8c67..9ffd5d4079e6c 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2426,6 +2426,8 @@ static int __init init_subsystems(void) switch (err) { case 0: vgic_present = true; + if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + kvm_nvhe_sym(hyp_gicv3_nr_lr) = kvm_vgic_global_state.nr_lr; break; case -ENODEV: case -ENXIO: diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 889c2c15d7bd0..4c58e18ec6936 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -11,14 +11,6 @@ #include #include -static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) -{ - wr->fst = fst; - wr->ptw = s1ptw; - wr->s2 = s1ptw; - wr->failed = true; -} - #define S1_MMU_DISABLED (-127) static int get_ia_size(struct s1_walk_info *wi) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index dba7ced74ca5e..7989c79ce6dbc 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2818,6 +2818,7 @@ static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2, break; case except_type_serror: kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR); + vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2); break; default: WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type); diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index e9b36a3b27bbc..4bd4a2b5cba43 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -448,16 +448,19 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) { + u64 spsr; + *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); /* * Finish potential single step before executing the prologue - * instruction. + * instruction. Modify the hardware SPSR_EL2 directly, as vcpu_cpsr() + * may hold a synthetic (vEL2) value for a guest hypervisor. */ - *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; - write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); + spsr = read_sysreg_el2(SYS_SPSR); + write_sysreg_el2(spsr & ~DBG_SPSR_SS, SYS_SPSR); return true; } @@ -601,8 +604,6 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) return false; break; case ESR_ELx_EC_SYS64: - if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu))) - return false; fallthrough; case ESR_ELx_EC_SVE: if (!sve_guest) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 06db299c37a89..a0da08caa6c27 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -24,6 +24,9 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +/* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ +unsigned int hyp_gicv3_nr_lr; + void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) @@ -128,6 +131,9 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + /* __hyp_running_vcpu must be NULL in a guest context. */ + hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; + hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE); hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & @@ -139,6 +145,12 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3; + /* Bound used_lrs by the number of implemented list registers. */ + hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs = + min_t(unsigned int, + hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs, + hyp_gicv3_nr_lr); + hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; } diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 91a7dfad66866..f218ac0ec1122 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1379,7 +1379,8 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags); if (!ret || ret == -EAGAIN) - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level); + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, + (ret == -EAGAIN) ? TLBI_TTL_UNKNOWN : level); return ret; } diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 4da9281312eb8..6e95514d5a76e 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, if (!kvm_has_mte(kvm)) return; + if (is_zero_pfn(pfn)) { + WARN_ON_ONCE(nr_pages != 1); + return; + } + if (folio_test_hugetlb(folio)) { /* Hugetlb has MTE flags set on head page only */ if (folio_try_hugetlb_mte_tagging(folio)) { diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 6f7bc9a9992e0..41b124f443b36 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -24,6 +24,7 @@ struct vncr_tlb { struct s1_walk_result wr; u64 hpa; + bool hpa_writable; /* -1 when not mapped on a CPU */ int cpu; @@ -904,9 +905,21 @@ static void invalidate_vncr(struct vncr_tlb *vt) clear_fixmap(vncr_fixmap(vt->cpu)); } +/* + * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and + * either can race against a vcpu not being onlined yet (no pseudo-TLB + * allocated). Similarly, the TLB might be invalid. Skip those, as they + * obviously don't participate in the invalidation at this stage. + */ +#define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \ + kvm_for_each_vcpu(idx, vcpup, kvm) \ + if (((tlbp) = vcpup->arch.vncr_tlb) && \ + (tlbp)->valid) + static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) { struct kvm_vcpu *vcpu; + struct vncr_tlb *vt; unsigned long i; lockdep_assert_held_write(&kvm->mmu_lock); @@ -914,24 +927,9 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { u64 ipa_start, ipa_end, ipa_size; - /* - * Careful here: We end-up here from an MMU notifier, - * and this can race against a vcpu not being onlined - * yet, without the pseudo-TLB being allocated. - * - * Skip those, as they obviously don't participate in - * the invalidation at this stage. - */ - if (!vt) - continue; - - if (!vt->valid) - continue; - ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); ipa_start = vt->wr.pa & ~(ipa_size - 1); @@ -961,17 +959,14 @@ static void invalidate_vncr_va(struct kvm *kvm, struct s1e2_tlbi_scope *scope) { struct kvm_vcpu *vcpu; + struct vncr_tlb *vt; unsigned long i; lockdep_assert_held_write(&kvm->mmu_lock); - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { u64 va_start, va_end, va_size; - if (!vt->valid) - continue; - va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); va_start = vt->gva & ~(va_size - 1); @@ -1314,15 +1309,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); - if (!memslot) + if (!memslot) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } *is_gmem = kvm_slot_has_gmem(memslot); if (!*is_gmem) { pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, &writable, &page); - if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) + if (is_error_noslot_pfn(pfn)) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } } else { ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); if (ret) { @@ -1330,124 +1329,123 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) write_fault, false, false); return ret; } + + writable = !(memslot->flags & KVM_MEM_READONLY); + } + + /* + * FIXME: This check is too restrictive as KVM allows cacheable memory + * attributes for PFNMAP VMAs that have cacheable attributes in host + * stage-1. + */ + if (!pfn_is_map_memory(pfn)) { + kvm_release_faultin_page(vcpu->kvm, page, true, false); + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); + return -EINVAL; } scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { - if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) + if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) { + kvm_release_faultin_page(vcpu->kvm, page, true, false); return -EAGAIN; + } vt->gva = va; vt->hpa = pfn << PAGE_SHIFT; + vt->hpa_writable = writable; vt->valid = true; vt->cpu = -1; kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); - kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw); + kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); } - if (vt->wr.pw) + if (vt->wr.pw && vt->hpa_writable) mark_page_dirty(vcpu->kvm, gfn); return 0; } -static void inject_vncr_perm(struct kvm_vcpu *vcpu) +static void handle_vncr_perm(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + u64 fsc; + + /* + * Promote to an external abort if the stage-1 permits writes but the + * HPA is read-only (e.g. RO memslot). + */ + if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable) + fsc = ESR_ELx_FSC_EXTABT; + /* + * Otherwise, inject a permission fault using the guest's translation + * level rather than the host's. + */ + else + fsc = ESR_ELx_FSC_PERM_L(vt->wr.level); - /* Adjust the fault level to reflect that of the guest's */ esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, - ESR_ELx_FSC_PERM_L(vt->wr.level)); + esr |= FIELD_PREP(ESR_ELx_FSC, fsc); kvm_inject_nested_sync(vcpu, esr); } -static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu) -{ - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; - - lockdep_assert_held_read(&vcpu->kvm->mmu_lock); - - if (!vt->valid) - return false; - - if (read_vncr_el2(vcpu) != vt->gva) - return false; - - if (vt->wr.nG) - return get_asid_by_regime(vcpu, TR_EL20) == vt->wr.asid; - - return true; -} - int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + bool is_gmem = false; + bool perm; + int ret; WARN_ON_ONCE(!(esr & ESR_ELx_VNCR)); if (kvm_vcpu_abt_issea(vcpu)) return kvm_handle_guest_sea(vcpu); - if (esr_fsc_is_permission_fault(esr)) { - inject_vncr_perm(vcpu); - } else if (esr_fsc_is_translation_fault(esr)) { - bool valid, is_gmem = false; - int ret; - - scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - valid = kvm_vncr_tlb_lookup(vcpu); - - if (!valid) - ret = kvm_translate_vncr(vcpu, &is_gmem); - else - ret = -EPERM; - - switch (ret) { - case -EAGAIN: - /* Let's try again... */ - break; - case -ENOMEM: - /* - * For guest_memfd, this indicates that it failed to - * create a folio to back the memory. Inform userspace. - */ - if (is_gmem) - return 0; - /* Otherwise, let's try again... */ - break; - case -EFAULT: - case -EIO: - case -EHWPOISON: - if (is_gmem) - return 0; - fallthrough; - case -EINVAL: - case -ENOENT: - case -EACCES: - /* - * Translation failed, inject the corresponding - * exception back to EL2. - */ - BUG_ON(!vt->wr.failed); + if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { + WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); + return 1; + } - esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); + ret = kvm_translate_vncr(vcpu, &is_gmem); + switch (ret) { + case -EAGAIN: + /* Let's try again... */ + return 1; + case -ENOMEM: + /* + * For guest_memfd, this indicates that it failed to + * create a folio to back the memory. Inform userspace. + */ + if (is_gmem) + return 0; + /* Otherwise, let's try again... */ + break; + case -EFAULT: + case -EIO: + case -EHWPOISON: + if (is_gmem) + return 0; + fallthrough; + case -EINVAL: + case -ENOENT: + case -EACCES: + /* + * Translation failed, inject the corresponding + * exception back to EL2. + */ + esr &= ~ESR_ELx_FSC; + esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); - kvm_inject_nested_sync(vcpu, esr); - break; - case -EPERM: - /* Hack to deal with POE until we get kernel support */ - inject_vncr_perm(vcpu); - break; - case 0: - break; - } - } else { - WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); + kvm_inject_nested_sync(vcpu, esr); + break; + case 0: + perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr; + if (!perm) + handle_vncr_perm(vcpu); + break; } return 1; @@ -1485,7 +1483,7 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) vt->cpu = smp_processor_id(); - if (vt->wr.pw && vt->wr.pr) + if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) prot = PAGE_KERNEL_RO; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 053e4f733e4be..428723b1b0f5c 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -352,7 +352,7 @@ static int __pkvm_pgtable_stage2_reclaim(struct kvm_pgtable *pgt, u64 start, u64 page = pfn_to_page(mapping->pfn); WARN_ON_ONCE(mapping->nr_pages != 1); unpin_user_pages_dirty_lock(&page, 1, true); - account_locked_vm(current->mm, 1, false); + account_locked_vm(kvm->mm, 1, false); pkvm_mapping_remove(mapping, &pgt->pkvm_mappings); kfree(mapping); } diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 1e3706ac3b8e9..7abb685c65465 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -507,6 +507,8 @@ static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db) struct kvm_io_device *kvm_io_dev; struct vgic_io_device *iodev; + guard(srcu)(&kvm->srcu); + kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db); if (!kvm_io_dev) return ERR_PTR(-EINVAL); diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 1e9fe8764584d..4cd818ce90933 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -204,6 +204,7 @@ void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu) list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) { if (irq_is_lpi(vcpu->kvm, irq->intid)) { raw_spin_lock(&irq->irq_lock); + irq->pending_latch = false; list_del(&irq->ap_list); irq->vcpu = NULL; raw_spin_unlock(&irq->irq_lock); @@ -795,7 +796,11 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) continue; } - /* This interrupt looks like it has to be migrated. */ + /* + * This interrupt looks like it has to be migrated, + * make sure it is kept alive while locks are dropped. + */ + vgic_get_irq_ref(irq); raw_spin_unlock(&irq->irq_lock); raw_spin_unlock(&vgic_cpu->ap_list_lock); @@ -818,15 +823,16 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) raw_spin_lock(&irq->irq_lock); /* - * If the affinity has been preserved, move the - * interrupt around. Otherwise, it means things have - * changed while the interrupt was unlocked, and we - * need to replay this. + * If the interrupt is still ours and its affinity has + * been preserved, move it around. Otherwise, it means + * things have changed while the interrupt was unlocked + * (it may even have been taken off the list with its + * affinity left untouched), and we need to replay this. * * In all cases, we cannot trust the list not to have * changed, so we restart from the beginning. */ - if (target_vcpu == vgic_target_oracle(irq)) { + if (irq->vcpu == vcpu && target_vcpu == vgic_target_oracle(irq)) { struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu; list_del(&irq->ap_list); @@ -839,6 +845,8 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); + deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq); + if (target_vcpu_needs_kick) { kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu); kvm_vcpu_kick(target_vcpu); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 8242f93f05e4f..fd756390fe925 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1509,7 +1509,13 @@ static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, if (free_mapped) { /* CONT blocks are not supported in the vmemmap */ WARN_ON(pmd_cont(pmd)); - flush_tlb_kernel_range(addr, addr + PMD_SIZE); + /* + * Invalidating a block entry requires just + * a single overlapping TLB invalidation, + * so limit the range of the flush to a single + * page. + */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pmd_page(pmd), PMD_SIZE, altmap); } @@ -1539,7 +1545,8 @@ static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, if (pud_leaf(pud)) { pud_clear(pudp); if (free_mapped) { - flush_tlb_kernel_range(addr, addr + PUD_SIZE); + /* See comment in unmap_hotplug_pmd_range(). */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pud_page(pud), PUD_SIZE, altmap); } diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0816c40fc7af9..9491c987e50c0 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2094,7 +2094,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr image_size = extable_offset + extable_size; ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u64), &header, &image_ptr, - jit_fill_hole); + jit_fill_hole, was_classic); if (!ro_header) goto out_off; @@ -2782,7 +2782,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 6c3ff14e561e6..88b31ae8450f4 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1806,7 +1806,7 @@ Res0 15:8 UnsignedEnum 7:4 BWE 0b0000 NI 0b0001 FEAT_BWE - 0b0002 FEAT_BWE2 + 0b0010 FEAT_BWE2 EndEnum UnsignedEnum 3:0 STEP 0b0000 NI diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index bff77efc0d9a9..1aecf60ec4f5a 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -26,8 +26,7 @@ unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n); -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER __kernel_size_t __clear_user_hexagon(void __user *dest, unsigned long count); #define __clear_user(a, s) __clear_user_hexagon((a), (s)) diff --git a/arch/loongarch/configs/loongson32_defconfig b/arch/loongarch/configs/loongson32_defconfig index d5ef396dffe30..82897236863fb 100644 --- a/arch/loongarch/configs/loongson32_defconfig +++ b/arch/loongarch/configs/loongson32_defconfig @@ -1091,7 +1091,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_DEV_VIRTIO=m -CONFIG_CRYPTO_DEV_LOONGSON_RNG=m CONFIG_DMA_CMA=y CONFIG_CMA_SIZE_MBYTES=0 CONFIG_PRINTK_TIME=y diff --git a/arch/loongarch/configs/loongson64_defconfig b/arch/loongarch/configs/loongson64_defconfig index cba4cdff5acd8..a94e88bd7ec5c 100644 --- a/arch/loongarch/configs/loongson64_defconfig +++ b/arch/loongarch/configs/loongson64_defconfig @@ -1124,7 +1124,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_DEV_VIRTIO=m -CONFIG_CRYPTO_DEV_LOONGSON_RNG=m CONFIG_DMA_CMA=y CONFIG_DMA_NUMA_CMA=y CONFIG_CMA_SIZE_MBYTES=0 diff --git a/arch/loongarch/include/asm/acpi.h b/arch/loongarch/include/asm/acpi.h index eda9d4d0a493c..c05168aedcaa2 100644 --- a/arch/loongarch/include/asm/acpi.h +++ b/arch/loongarch/include/asm/acpi.h @@ -38,6 +38,8 @@ static inline bool acpi_has_cpu_in_madt(void) extern struct list_head acpi_wakeup_device_list; extern struct acpi_madt_core_pic acpi_core_pic[MAX_CORE_PIC]; +extern void acpi_add_early_pio(void); +extern void acpi_remove_early_pio(void); extern int __init parse_acpi_topology(void); #endif /* !CONFIG_ACPI */ diff --git a/arch/loongarch/include/asm/kexec.h b/arch/loongarch/include/asm/kexec.h index 209fa43222e1c..6be136e9f0a06 100644 --- a/arch/loongarch/include/asm/kexec.h +++ b/arch/loongarch/include/asm/kexec.h @@ -41,6 +41,8 @@ struct kimage_arch { unsigned long systable_ptr; }; +struct kimage; + #ifdef CONFIG_KEXEC_FILE extern const struct kexec_file_ops kexec_efi_ops; extern const struct kexec_file_ops kexec_elf_ops; @@ -59,7 +61,6 @@ typedef void (*do_kexec_t)(unsigned long efi_boot, unsigned long start_addr, unsigned long first_ind_entry); -struct kimage; extern const unsigned char relocate_new_kernel[]; extern const size_t relocate_new_kernel_size; extern void kexec_reboot(void); diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 2a0b63ae421f4..223528c04d739 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -429,6 +429,8 @@ static inline pte_t pte_mkwrite_novma(pte_t pte) static inline pte_t pte_wrprotect(pte_t pte) { + if (pte_val(pte) & _PAGE_DIRTY) + pte_val(pte) |= _PAGE_MODIFIED; pte_val(pte) &= ~(_PAGE_WRITE | _PAGE_DIRTY); return pte; } @@ -535,6 +537,8 @@ static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) static inline pmd_t pmd_wrprotect(pmd_t pmd) { + if (pmd_val(pmd) & _PAGE_DIRTY) + pmd_val(pmd) |= _PAGE_MODIFIED; pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_DIRTY); return pmd; } diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h index 438269313e78c..428f373feabf1 100644 --- a/arch/loongarch/include/asm/uaccess.h +++ b/arch/loongarch/include/asm/uaccess.h @@ -292,8 +292,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) return __copy_user((__force void *)to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * __clear_user: - Zero a block of memory in user space, with less checking. diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 058f0dbe8e8f9..8f650c9ffecde 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,33 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) return ioremap_cache(phys, size); } +#define PIO_BASE (unsigned long)PCI_IOBASE +#define PIO_SIZE ALIGN(ISA_IOSIZE, PAGE_SIZE) + +static bool acpi_pio; + +/* Add PIO for early access */ +void acpi_add_early_pio(void) +{ + if (!acpi_disabled) { + acpi_pio = true; + vmap_page_range(PIO_BASE, PIO_BASE + PIO_SIZE, + LOONGSON_LIO_BASE, pgprot_device(PAGE_KERNEL)); + } +} + +/* Remove PIO for PCI register */ +void acpi_remove_early_pio(void) +{ + if (!acpi_pio) + return; + + if (!acpi_disabled) { + acpi_pio = false; + vunmap_range(PIO_BASE, PIO_BASE + PIO_SIZE); + } +} + #ifdef CONFIG_SMP static int set_processor_mask(u32 id, u32 pass) { diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 839b23edee876..bfc6f274e6e28 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -502,6 +502,8 @@ static __init int arch_reserve_pio_range(void) { struct device_node *np; + acpi_add_early_pio(); + for_each_node_by_name(np, "isa") { struct of_range range; struct of_range_parser parser; diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 64a048f1b8803..b077b2aa1ab37 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -705,6 +705,7 @@ static void stop_this_cpu(void *dummy) set_cpu_online(smp_processor_id(), false); calculate_cpu_foreign_map(); local_irq_disable(); + rcutree_report_cpu_dead(); while (true); } diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index 3b95cd0f989b0..264813d45cbe0 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -103,7 +103,6 @@ static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, old = kvm_read_sw_gcsr(csr, csrid); val = (old & ~csr_mask) | (val & csr_mask); kvm_write_sw_gcsr(csr, csrid, val); - old = old & csr_mask; } else pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc); diff --git a/arch/loongarch/kvm/intc/eiointc.c b/arch/loongarch/kvm/intc/eiointc.c index 2ab7fafa86d59..2b14485d14a70 100644 --- a/arch/loongarch/kvm/intc/eiointc.c +++ b/arch/loongarch/kvm/intc/eiointc.c @@ -645,10 +645,14 @@ static int kvm_eiointc_create(struct kvm_device *dev, u32 type) device = &s->device_vext; kvm_iodevice_init(device, &kvm_eiointc_virt_ops); + mutex_lock(&kvm->slots_lock); ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, EIOINTC_VIRT_BASE, EIOINTC_VIRT_SIZE, device); + mutex_unlock(&kvm->slots_lock); if (ret < 0) { + mutex_lock(&kvm->slots_lock); kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &s->device); + mutex_unlock(&kvm->slots_lock); kfree(s); return ret; } @@ -667,8 +671,10 @@ static void kvm_eiointc_destroy(struct kvm_device *dev) kvm = dev->kvm; eiointc = kvm->arch.eiointc; + mutex_lock(&kvm->slots_lock); kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device); kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device_vext); + mutex_unlock(&kvm->slots_lock); kfree(eiointc); kfree(dev); } diff --git a/arch/loongarch/kvm/intc/ipi.c b/arch/loongarch/kvm/intc/ipi.c index 1f6ebbd0af5c8..4fa0897d7bdb0 100644 --- a/arch/loongarch/kvm/intc/ipi.c +++ b/arch/loongarch/kvm/intc/ipi.c @@ -447,7 +447,9 @@ static void kvm_ipi_destroy(struct kvm_device *dev) kvm = dev->kvm; ipi = kvm->arch.ipi; + mutex_lock(&kvm->slots_lock); kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device); + mutex_unlock(&kvm->slots_lock); kfree(ipi); kfree(dev); } diff --git a/arch/loongarch/kvm/intc/pch_pic.c b/arch/loongarch/kvm/intc/pch_pic.c index aa0ed59ae8cf9..175a630aceb41 100644 --- a/arch/loongarch/kvm/intc/pch_pic.c +++ b/arch/loongarch/kvm/intc/pch_pic.c @@ -481,7 +481,9 @@ static void kvm_pch_pic_destroy(struct kvm_device *dev) kvm = dev->kvm; s = kvm->arch.pch_pic; /* unregister pch pic device and free it's memory */ + mutex_lock(&kvm->slots_lock); kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &s->device); + mutex_unlock(&kvm->slots_lock); kfree(s); kfree(dev); } diff --git a/arch/loongarch/kvm/irqfd.c b/arch/loongarch/kvm/irqfd.c index f4f953b224192..40ed1081c4b62 100644 --- a/arch/loongarch/kvm/irqfd.c +++ b/arch/loongarch/kvm/irqfd.c @@ -51,7 +51,8 @@ int kvm_set_routing_entry(struct kvm *kvm, e->irqchip.irqchip = ue->u.irqchip.irqchip; e->irqchip.pin = ue->u.irqchip.pin; - if (e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) + if (e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS || + e->irqchip.irqchip >= KVM_NR_IRQCHIPS) return -EINVAL; return 0; diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index e28084c49e682..7f7f2b71f03ae 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -1108,7 +1108,8 @@ static int kvm_loongarch_cpucfg_get_attr(struct kvm_vcpu *vcpu, return -ENXIO; } - put_user(val, uaddr); + if (put_user(val, uaddr)) + return -EFAULT; return ret; } @@ -1312,7 +1313,7 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) fpu->fcc = vcpu->arch.fpu.fcc; fpu->fcsr = vcpu->arch.fpu.fcsr; for (i = 0; i < NUM_FPU_REGS; i++) - memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64); + memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], sizeof(union fpureg)); return 0; } @@ -1324,7 +1325,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) vcpu->arch.fpu.fcc = fpu->fcc; vcpu->arch.fpu.fcsr = fpu->fcsr; for (i = 0; i < NUM_FPU_REGS; i++) - memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64); + memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], sizeof(union fpureg)); return 0; } @@ -1487,6 +1488,10 @@ void kvm_lose_fpu(struct kvm_vcpu *vcpu) int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) { int intr = (int)irq->irq; + unsigned int vector = abs(intr); + + if (vector >= EXCCODE_INT_NUM) + return -EINVAL; if (intr > 0) kvm_queue_irq(vcpu, intr); diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index f5e910b68229d..614ccc7afccbe 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -234,5 +234,5 @@ int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) clear = __pgprot(_PAGE_PRESENT | _PAGE_VALID); } - return __set_memory(addr, 1, set, clear); + return __set_memory(addr, nr, set, clear); } diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 24913dc7f4e83..e4966c178abc5 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -9,7 +9,13 @@ #define LOONGARCH_MAX_REG_ARGS 8 +#define LOONGARCH_SAVE_RA_NINSNS 1 #define LOONGARCH_LONG_JUMP_NINSNS 5 +#define LOONGARCH_TCC_SLOT_NINSNS 1 + +#define LOONGARCH_PROLOGUE_SKIP_INSNS \ + (LOONGARCH_SAVE_RA_NINSNS + LOONGARCH_LONG_JUMP_NINSNS + LOONGARCH_TCC_SLOT_NINSNS) + #define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) #define LOONGARCH_FENTRY_NINSNS 2 @@ -143,8 +149,13 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; + /* + * Save the original return address to a temporary register to prevent + * it from being overwritten, then reserve space for the long jump and + * fentry trampoline slot for dynamically patching by ftrace at runtime. + * These instructions are bypassed during a tail call invocation. + */ move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); - /* Reserve space for the move_imm + jirl instruction */ for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) emit_insn(ctx, nop); @@ -253,10 +264,11 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); } else { /* - * Call the next bpf prog and skip the first instruction - * of TCC initialization. + * Tail call to the next BPF program, passing offset in number + * of instructions to jirl to bypass the initial setup slots. */ - emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 7); + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, + LOONGARCH_GPR_T3, LOONGARCH_PROLOGUE_SKIP_INSNS); } } @@ -312,12 +324,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) */ emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); emit_insn(ctx, ldd, t3, REG_TCC, 0); - emit_insn(ctx, addid, t3, t3, 1); - emit_insn(ctx, std, t3, REG_TCC, 0); emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); - if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0) + if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0) goto toofar; + emit_insn(ctx, addid, t3, t3, 1); + /* * prog = array->ptrs[index]; * if (!prog) @@ -330,6 +342,8 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) goto toofar; + emit_insn(ctx, std, t3, REG_TCC, 0); + /* goto *(prog->bpf_func + 4); */ off = offsetof(struct bpf_prog, bpf_func); emit_insn(ctx, ldd, t3, t2, off); @@ -1762,7 +1776,7 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_links *tl, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) @@ -2228,7 +2242,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr image_size = prog_size + extable_size; /* Now we know the size of the structure to make */ ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), - &header, &image_ptr, jit_fill_hole); + &header, &image_ptr, jit_fill_hole, + bpf_prog_was_classic(prog)); if (!ro_header) goto out_offset; diff --git a/arch/loongarch/pci/acpi.c b/arch/loongarch/pci/acpi.c index b02698a338eef..ccbcea61fcd9c 100644 --- a/arch/loongarch/pci/acpi.c +++ b/arch/loongarch/pci/acpi.c @@ -65,6 +65,8 @@ static int acpi_prepare_root_resources(struct acpi_pci_root_info *ci) struct resource_entry *entry, *tmp; struct acpi_device *device = ci->bridge; + acpi_remove_early_pio(); + status = acpi_pci_probe_root_resources(ci); if (status > 0) { acpi_evaluate_integer(device->handle, "PCIH", NULL, &pci_h); diff --git a/arch/m68k/coldfire/m5441x.c b/arch/m68k/coldfire/m5441x.c index 7a25cfc7ac075..ac9c17d0ee647 100644 --- a/arch/m68k/coldfire/m5441x.c +++ b/arch/m68k/coldfire/m5441x.c @@ -41,9 +41,9 @@ DEFINE_CLK(0, "mcfpit.0", 32, MCF_BUSCLK); DEFINE_CLK(0, "mcfpit.1", 33, MCF_BUSCLK); DEFINE_CLK(0, "mcfpit.2", 34, MCF_BUSCLK); DEFINE_CLK(0, "mcfpit.3", 35, MCF_BUSCLK); -DEFINE_CLK(0, "mcfeport.0", 37, MCF_CLK); -DEFINE_CLK(0, "mcfadc.0", 38, MCF_CLK); -DEFINE_CLK(0, "mcfdac.0", 39, MCF_CLK); +DEFINE_CLK(0, "mcfeport.0", 36, MCF_CLK); +DEFINE_CLK(0, "mcfadc.0", 37, MCF_CLK); +DEFINE_CLK(0, "mcfdac.0", 38, MCF_CLK); DEFINE_CLK(0, "mcfrtc.0", 42, MCF_CLK); DEFINE_CLK(0, "mcfsim.0", 43, MCF_CLK); DEFINE_CLK(0, "mcfusb-otg.0", 44, MCF_CLK); @@ -103,9 +103,9 @@ static struct clk_lookup m5411x_clk_lookup[] = { CLKDEV_INIT("mcfpit.1", NULL, &__clk_0_33), CLKDEV_INIT("mcfpit.2", NULL, &__clk_0_34), CLKDEV_INIT("mcfpit.3", NULL, &__clk_0_35), - CLKDEV_INIT("mcfeport.0", NULL, &__clk_0_37), - CLKDEV_INIT("mcfadc.0", NULL, &__clk_0_38), - CLKDEV_INIT("mcfdac.0", NULL, &__clk_0_39), + CLKDEV_INIT("mcfeport.0", NULL, &__clk_0_36), + CLKDEV_INIT("mcfadc.0", NULL, &__clk_0_37), + CLKDEV_INIT("mcfdac.0", NULL, &__clk_0_38), CLKDEV_INIT("mcfrtc.0", NULL, &__clk_0_42), CLKDEV_INIT("mcfsim.0", NULL, &__clk_0_43), CLKDEV_INIT("mcfusb-otg.0", NULL, &__clk_0_44), @@ -156,7 +156,7 @@ static struct clk * const enable_clks[] __initconst = { &__clk_0_27, /* uart3 */ &__clk_0_33, /* pit.1 */ - &__clk_0_37, /* eport */ + &__clk_0_36, /* eport */ &__clk_0_48, /* pll */ &__clk_0_51, /* esdhc */ @@ -174,8 +174,8 @@ static struct clk * const disable_clks[] __initconst = { &__clk_0_32, /* pit.0 */ &__clk_0_34, /* pit.2 */ &__clk_0_35, /* pit.3 */ - &__clk_0_38, /* adc */ - &__clk_0_39, /* dac */ + &__clk_0_37, /* adc */ + &__clk_0_38, /* dac.0 */ &__clk_0_44, /* usb otg */ &__clk_0_45, /* usb host */ &__clk_0_47, /* ssi.0 */ diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h index 64914872a5c98..31d133faa45ef 100644 --- a/arch/m68k/include/asm/uaccess.h +++ b/arch/m68k/include/asm/uaccess.h @@ -377,8 +377,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) return __constant_copy_to_user(to, from, n); return __generic_copy_to_user(to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index 3aab2f17e0462..afa0dd8d013fb 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -250,8 +250,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) { return __copy_tofrom_user(to, (__force const void __user *)from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * Copy a null terminated string from userspace. diff --git a/arch/mips/dec/int-handler.S b/arch/mips/dec/int-handler.S index 011d1d678840a..a0b439c90488f 100644 --- a/arch/mips/dec/int-handler.S +++ b/arch/mips/dec/int-handler.S @@ -277,7 +277,7 @@ srlv t3,t1,t2 handle_it: - j dec_irq_dispatch + j do_IRQ nop #if defined(CONFIG_32BIT) && defined(CONFIG_MIPS_FP_SUPPORT) diff --git a/arch/mips/dec/platform.c b/arch/mips/dec/platform.c index 723ce16cbfc0c..a005246a0ac5d 100644 --- a/arch/mips/dec/platform.c +++ b/arch/mips/dec/platform.c @@ -38,6 +38,10 @@ static struct platform_device dec_rtc_device = { .num_resources = ARRAY_SIZE(dec_rtc_resources), }; +static struct platform_device *dec_rtc_devices[] __initdata = { + &dec_rtc_device, +}; + static struct resource dec_dz_resources[] = { { .name = "dz", .flags = IORESOURCE_MEM, }, { .name = "dz", .flags = IORESOURCE_IRQ, }, @@ -137,7 +141,7 @@ static int __init dec_add_devices(void) } num_zs = i; - ret1 = platform_device_register(&dec_rtc_device); + ret1 = platform_add_devices(dec_rtc_devices, 1); ret2 = IS_ENABLED(CONFIG_32BIT) ? platform_add_devices(dec_dz_devices, num_dz) : 0; ret3 = platform_add_devices(dec_zs_devices, num_zs); diff --git a/arch/mips/dec/prom/console.c b/arch/mips/dec/prom/console.c index 31a8441d84314..b4f0dba3fa20e 100644 --- a/arch/mips/dec/prom/console.c +++ b/arch/mips/dec/prom/console.c @@ -2,8 +2,9 @@ /* * DECstation PROM-based early console support. * - * Copyright (C) 2004, 2007 Maciej W. Rozycki + * Copyright (C) 2004, 2007, 2026 Maciej W. Rozycki */ +#include #include #include #include @@ -14,9 +15,11 @@ static void __init prom_console_write(struct console *con, const char *s, unsigned int c) { - char buf[81]; + static char buf[81] __initdata = { 0 }; unsigned int chunk = sizeof(buf) - 1; + BUG_ON((long)buf != (int)(long)buf); + while (c > 0) { if (chunk > c) chunk = c; diff --git a/arch/mips/dec/prom/init.c b/arch/mips/dec/prom/init.c index 8d74d7d6c05b4..e46c7acceffff 100644 --- a/arch/mips/dec/prom/init.c +++ b/arch/mips/dec/prom/init.c @@ -3,7 +3,7 @@ * init.c: PROM library initialisation code. * * Copyright (C) 1998 Harald Koerfgen - * Copyright (C) 2002, 2004 Maciej W. Rozycki + * Copyright (C) 2002, 2004, 2026 Maciej W. Rozycki */ #include #include @@ -20,6 +20,10 @@ #include +#ifdef CONFIG_64BIT +unsigned long o32_stk[O32_STK_SIZE] __initdata = { 0 }; +#endif + int (*__rex_bootinit)(void); int (*__rex_bootread)(void); int (*__rex_getbitmap)(memmap *); diff --git a/arch/mips/dec/setup.c b/arch/mips/dec/setup.c index 87f0a1436bf9c..abe42616498db 100644 --- a/arch/mips/dec/setup.c +++ b/arch/mips/dec/setup.c @@ -776,9 +776,3 @@ void __init arch_init_irq(void) pr_err("Failed to register halt interrupt\n"); } } - -asmlinkage unsigned int dec_irq_dispatch(unsigned int irq) -{ - do_IRQ(irq); - return 0; -} diff --git a/arch/mips/fw/lib/call_o32.S b/arch/mips/fw/lib/call_o32.S index ee856709e0b60..77533cfbdfc10 100644 --- a/arch/mips/fw/lib/call_o32.S +++ b/arch/mips/fw/lib/call_o32.S @@ -74,7 +74,7 @@ NESTED(call_o32, O32_FRAMESZ, ra) PTR_LA t1,6*O32_SZREG(fp) li t2,O32_ARGC-6 1: - lw t3,(t0) + ld t3,(t0) REG_ADDU t0,SZREG sw t3,(t1) REG_SUBU t2,1 diff --git a/arch/mips/include/asm/dec/prom.h b/arch/mips/include/asm/dec/prom.h index 8fcad6984389c..af068e634f1e3 100644 --- a/arch/mips/include/asm/dec/prom.h +++ b/arch/mips/include/asm/dec/prom.h @@ -4,7 +4,7 @@ * * DECstation PROM interface. * - * Copyright (C) 2002 Maciej W. Rozycki + * Copyright (C) 2002, 2026 Maciej W. Rozycki * * Based on arch/mips/dec/prom/prom.h by the Anonymous. */ @@ -97,6 +97,17 @@ extern int (*__pmax_close)(int); #ifdef CONFIG_64BIT +#define O32_STK_SIZE 512 +extern unsigned long o32_stk[]; + +/* Switch the stack if outside the 32-bit address space. */ +static inline unsigned long *o32_get_stk(void) +{ + long fp = (long)__builtin_frame_address(0); + + return fp != (int)fp ? o32_stk + O32_STK_SIZE : NULL; +} + /* * On MIPS64 we have to call PROM functions via a helper * dispatcher to accommodate ABI incompatibilities. @@ -128,7 +139,7 @@ int __DEC_PROM_O32(_prom_printf, (int (*)(char *, ...), void *, char *, ...)); #define prom_getchar() _prom_getchar(__prom_getchar, NULL) #define prom_getenv(x) _prom_getenv(__prom_getenv, NULL, x) -#define prom_printf(x...) _prom_printf(__prom_printf, NULL, x) +#define prom_printf(x...) _prom_printf(__prom_printf, o32_get_stk(), x) #else /* !CONFIG_64BIT */ diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index c0cede273c7c0..f00c36676b737 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -433,8 +433,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) return __cu_len_r; } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size); diff --git a/arch/mips/kernel/mips-mt-fpaff.c b/arch/mips/kernel/mips-mt-fpaff.c index 10172fc4f627b..4fead87d2f433 100644 --- a/arch/mips/kernel/mips-mt-fpaff.c +++ b/arch/mips/kernel/mips-mt-fpaff.c @@ -71,11 +71,16 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, struct task_struct *p; int retval; - if (len < sizeof(new_mask)) - return -EINVAL; - - if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask))) - return -EFAULT; + if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) + return -ENOMEM; + if (len < cpumask_size()) + cpumask_clear(new_mask); + else if (len > cpumask_size()) + len = cpumask_size(); + if (copy_from_user(new_mask, user_mask_ptr, len)) { + retval = -EFAULT; + goto out_free_new_mask; + } cpus_read_lock(); rcu_read_lock(); @@ -84,7 +89,8 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, if (!p) { rcu_read_unlock(); cpus_read_unlock(); - return -ESRCH; + retval = -ESRCH; + goto out_free_new_mask; } /* Prevent p going away */ @@ -95,13 +101,9 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, retval = -ENOMEM; goto out_put_task; } - if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { - retval = -ENOMEM; - goto out_free_cpus_allowed; - } if (!alloc_cpumask_var(&effective_mask, GFP_KERNEL)) { retval = -ENOMEM; - goto out_free_new_mask; + goto out_free_cpus_allowed; } if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) { retval = -EPERM; @@ -142,13 +144,13 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, } out_unlock: free_cpumask_var(effective_mask); -out_free_new_mask: - free_cpumask_var(new_mask); out_free_cpus_allowed: free_cpumask_var(cpus_allowed); out_put_task: put_task_struct(p); cpus_read_unlock(); +out_free_new_mask: + free_cpumask_var(new_mask); return retval; } diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 4868e79f3b30e..0f28b4a62e72e 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -422,6 +423,7 @@ static void stop_this_cpu(void *dummy) set_cpu_online(smp_processor_id(), false); calculate_cpu_foreign_map(); local_irq_disable(); + rcutree_report_cpu_dead(); while (1); } diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 55b25e85122a3..352718e43f697 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -272,9 +272,15 @@ static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages, void *data) { struct maar_walk_info *wi = data; - struct maar_config *cfg = &wi->cfg[wi->num_cfg]; + struct maar_config *cfg; unsigned int maar_align; + /* Ensure we don't overflow the cfg array */ + if (WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg))) + return -1; + + cfg = &wi->cfg[wi->num_cfg]; + /* MAAR registers hold physical addresses right shifted by 4 bits */ maar_align = BIT(MIPS_MAAR_ADDR_SHIFT + 4); @@ -283,9 +289,7 @@ static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages, cfg->upper = ALIGN_DOWN(PFN_PHYS(start_pfn + nr_pages), maar_align) - 1; cfg->attrs = MIPS_MAAR_S; - /* Ensure we don't overflow the cfg array */ - if (!WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg))) - wi->num_cfg++; + wi->num_cfg++; return 0; } @@ -422,10 +426,11 @@ static inline void __init highmem_init(void) unsigned long tmp; /* - * If CPU cannot support HIGHMEM discard the memory above highstart_pfn + * If CPU cannot support HIGHMEM discard any memory above highstart_pfn */ if (cpu_has_dc_aliases) { - memblock_remove(PFN_PHYS(highstart_pfn), -1); + if (highstart_pfn) + memblock_remove(PFN_PHYS(highstart_pfn), -1); return; } diff --git a/arch/mips/n64/init.c b/arch/mips/n64/init.c index dfbd864f46670..66ec28ab41f32 100644 --- a/arch/mips/n64/init.c +++ b/arch/mips/n64/init.c @@ -50,7 +50,7 @@ void __init prom_init(void) #define W 320 #define H 240 -#define REG_BASE ((u32 *) CKSEG1ADDR(0x4400000)) +#define REG_BASE ((u32 __iomem *) CKSEG1ADDR(0x4400000)) static void __init n64rdp_write_reg(const u8 reg, const u32 value) { diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c index a4bdda8541c07..ae7b8cfedd5f7 100644 --- a/arch/mips/ralink/mt7621.c +++ b/arch/mips/ralink/mt7621.c @@ -63,7 +63,7 @@ phys_addr_t mips_cpc_default_phys_base(void) static bool __init mt7621_addr_wraparound_test(phys_addr_t size) { - void *dm = (void *)KSEG1ADDR(&detect_magic); + void __iomem *dm = (void __iomem *)KSEG1ADDR(&detect_magic); if (CPHYSADDR(dm + size) >= MT7621_LOWMEM_MAX_SIZE) return true; diff --git a/arch/mips/sgi-ip22/ip22-gio.c b/arch/mips/sgi-ip22/ip22-gio.c index 9eec8842ffb70..7b7572d112501 100644 --- a/arch/mips/sgi-ip22/ip22-gio.c +++ b/arch/mips/sgi-ip22/ip22-gio.c @@ -30,7 +30,6 @@ static struct { static void gio_bus_release(struct device *dev) { - kfree(dev); } static struct device gio_bus = { @@ -101,6 +100,8 @@ int gio_device_register(struct gio_device *giodev) { giodev->dev.bus = &gio_bus_type; giodev->dev.parent = &gio_bus; + giodev->dev.release = gio_release_dev; + return device_register(&giodev->dev); } EXPORT_SYMBOL_GPL(gio_device_register); @@ -132,13 +133,9 @@ static int gio_device_probe(struct device *dev) if (!drv->probe) return error; - gio_dev_get(gio_dev); - match = gio_match_device(drv->id_table, gio_dev); if (match) error = drv->probe(gio_dev, match); - if (error) - gio_dev_put(gio_dev); return error; } diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index 6ccc9a232c239..5e6e05cc6efc7 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -57,8 +57,7 @@ extern unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); extern unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n); -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER extern long strncpy_from_user(char *__to, const char __user *__from, long __len); diff --git a/arch/openrisc/include/asm/cacheflush.h b/arch/openrisc/include/asm/cacheflush.h index cd8f971c0fecc..7b8c043a831df 100644 --- a/arch/openrisc/include/asm/cacheflush.h +++ b/arch/openrisc/include/asm/cacheflush.h @@ -26,6 +26,7 @@ extern void local_icache_page_inv(struct page *page); extern void local_dcache_range_flush(unsigned long start, unsigned long end); extern void local_dcache_range_inv(unsigned long start, unsigned long end); extern void local_icache_range_inv(unsigned long start, unsigned long end); +extern void local_icache_all_inv(void); /* * Data cache flushing always happen on the local cpu. Instruction cache @@ -35,10 +36,13 @@ extern void local_icache_range_inv(unsigned long start, unsigned long end); #ifndef CONFIG_SMP #define dcache_page_flush(page) local_dcache_page_flush(page) #define icache_page_inv(page) local_icache_page_inv(page) +#define icache_all_inv() local_icache_all_inv() #else /* CONFIG_SMP */ #define dcache_page_flush(page) local_dcache_page_flush(page) #define icache_page_inv(page) smp_icache_page_inv(page) +#define icache_all_inv() smp_icache_all_inv() extern void smp_icache_page_inv(struct page *page); +extern void smp_icache_all_inv(void); #endif /* CONFIG_SMP */ /* diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index d6500a374e183..db934ebc0069f 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -218,8 +218,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long size) { return __copy_tofrom_user((__force void *)to, from, size); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER extern unsigned long __clear_user(void __user *addr, unsigned long size); diff --git a/arch/openrisc/kernel/jump_label.c b/arch/openrisc/kernel/jump_label.c index ab7137c23b46c..9cb63f2d2e2b7 100644 --- a/arch/openrisc/kernel/jump_label.c +++ b/arch/openrisc/kernel/jump_label.c @@ -47,5 +47,5 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry, void arch_jump_label_transform_apply(void) { - kick_all_cpus_sync(); + icache_all_inv(); } diff --git a/arch/openrisc/kernel/patching.c b/arch/openrisc/kernel/patching.c index d186172beb337..5db027b78bc40 100644 --- a/arch/openrisc/kernel/patching.c +++ b/arch/openrisc/kernel/patching.c @@ -49,6 +49,9 @@ static int __patch_insn_write(void *addr, u32 insn) waddr = patch_map(addr, FIX_TEXT_POKE0); ret = copy_to_kernel_nofault(waddr, &insn, OPENRISC_INSN_SIZE); + if (!IS_ENABLED(CONFIG_DCACHE_WRITETHROUGH)) + local_dcache_range_flush((unsigned long)waddr, + (unsigned long)waddr + OPENRISC_INSN_SIZE); local_icache_range_inv((unsigned long)waddr, (unsigned long)waddr + OPENRISC_INSN_SIZE); diff --git a/arch/openrisc/kernel/smp.c b/arch/openrisc/kernel/smp.c index 040ca201b6927..65599252f3d41 100644 --- a/arch/openrisc/kernel/smp.c +++ b/arch/openrisc/kernel/smp.c @@ -346,3 +346,24 @@ void smp_icache_page_inv(struct page *page) on_each_cpu(ipi_icache_page_inv, page, 1); } EXPORT_SYMBOL(smp_icache_page_inv); + +static void ipi_icache_all_inv(void *arg) +{ + local_icache_all_inv(); +} + +void smp_icache_all_inv(void) +{ + if (num_online_cpus() < 2) { + local_icache_all_inv(); + return; + } + + /* + * Ensure stores complete before we request remote icaches + * to invalidate. + */ + mb(); + + on_each_cpu(ipi_icache_all_inv, NULL, 1); +} diff --git a/arch/openrisc/mm/cache.c b/arch/openrisc/mm/cache.c index f33df46dae4e5..2667d90691b59 100644 --- a/arch/openrisc/mm/cache.c +++ b/arch/openrisc/mm/cache.c @@ -63,6 +63,22 @@ void local_icache_page_inv(struct page *page) } EXPORT_SYMBOL(local_icache_page_inv); +void local_icache_all_inv(void) +{ + if (cpu_cache_is_present(SPR_UPR_ICP)) { + unsigned long iccfgr = mfspr(SPR_ICCFGR); + unsigned long sets = 1 << ((iccfgr & SPR_ICCFGR_NCS) >> 3); + unsigned long block_size = 16 << ((iccfgr & SPR_ICCFGR_CBS) >> 7); + unsigned long paddr = 0; + unsigned long end = sets * block_size; + + while (paddr < end) { + mtspr(SPR_ICBIR, paddr); + paddr += block_size; + } + } +} + void local_dcache_range_flush(unsigned long start, unsigned long end) { cache_loop(start, end, SPR_DCBFR, SPR_UPR_DCP); diff --git a/arch/openrisc/mm/init.c b/arch/openrisc/mm/init.c index 89d8c6df88551..db7c844faeeb6 100644 --- a/arch/openrisc/mm/init.c +++ b/arch/openrisc/mm/init.c @@ -193,7 +193,7 @@ void __init mem_init(void) return; } -static int __init map_page(unsigned long va, phys_addr_t pa, pgprot_t prot) +static int map_page(unsigned long va, phys_addr_t pa, pgprot_t prot) { p4d_t *p4d; pud_t *pud; diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index 6c531d2c847eb..0d17f81c8b270 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -197,7 +197,6 @@ unsigned long __must_check raw_copy_to_user(void __user *dst, const void *src, unsigned long len); unsigned long __must_check raw_copy_from_user(void *dst, const void __user *src, unsigned long len); -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER #endif /* __PARISC_UACCESS_H */ diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index e98c628e38993..619270bb7380c 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -511,7 +511,7 @@ static inline void __user *mask_user_address(const void __user *ptr) if (IS_ENABLED(CONFIG_PPC64)) return mask_user_address_simple(ptr); - if (IS_ENABLED(CONFIG_E500)) + if (IS_ENABLED(CONFIG_PPC_E500)) return mask_user_address_isel(ptr); if (TASK_SIZE <= UL(SZ_2G) && border >= UL(SZ_2G)) return mask_user_address_simple(ptr); diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 3af6c06af02f3..e5853daa6a486 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -704,6 +704,15 @@ static void __init cpufeatures_setup_start(u32 isa) if (isa >= ISA_V3_1) { cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; + + /* + * CPU_FTR_P11_PVR is a kernel-internal flag to identify + * Power11 and later processors. While ISA v3.1 is supported + * by Power10+, this flag specifically indicates Power11+ + * for code that needs to distinguish between P10 and P11. + */ + if (PVR_VER(mfspr(SPRN_PVR)) >= PVR_POWER11) + cur_cpu_spec->cpu_features |= CPU_FTR_P11_PVR; } } diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c index 825ab8a88f18e..58c13a59b93b5 100644 --- a/arch/powerpc/kexec/core_64.c +++ b/arch/powerpc/kexec/core_64.c @@ -169,7 +169,7 @@ static void kexec_prepare_cpus_wait(int wait_state) int my_cpu, i, notified=-1; hw_breakpoint_disable(); - my_cpu = get_cpu(); + my_cpu = raw_smp_processor_id(); /* Make sure each CPU has at least made it to the state we need. * * FIXME: There is a (slim) chance of a problem if not all of the CPUs @@ -267,8 +267,6 @@ static void kexec_prepare_cpus(void) /* after we tell the others to go down */ if (ppc_md.kexec_cpu_down) ppc_md.kexec_cpu_down(0, 0); - - put_cpu(); } #else /* ! SMP */ diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 53ab97ad60745..73ff8a64bb7fa 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -295,7 +295,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, - bpf_jit_fill_ill_insns); + bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp)); if (!fhdr) goto out_err; @@ -583,7 +583,7 @@ bool bpf_jit_inlines_helper_call(s32 imm) void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns); + return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/powerpc/perf/core-fsl-emb.c b/arch/powerpc/perf/core-fsl-emb.c index 7120ab20cbfec..02b5dd74c187a 100644 --- a/arch/powerpc/perf/core-fsl-emb.c +++ b/arch/powerpc/perf/core-fsl-emb.c @@ -366,9 +366,10 @@ static void fsl_emb_pmu_del(struct perf_event *event, int flags) cpuhw->n_events--; + put_cpu_var(cpu_hw_events); + out: perf_pmu_enable(event->pmu); - put_cpu_var(cpu_hw_events); } static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags) diff --git a/arch/powerpc/platforms/8xx/cpm1.c b/arch/powerpc/platforms/8xx/cpm1.c index f00734f0590cf..b31376bf67788 100644 --- a/arch/powerpc/platforms/8xx/cpm1.c +++ b/arch/powerpc/platforms/8xx/cpm1.c @@ -472,6 +472,18 @@ static int cpm1_gpio16_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } +static int cpm1_gpio16_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport16 __iomem *iop = cpm1_gc->regs; + u16 pin_mask = 1 << (15 - gpio); + + if (in_be16(&iop->dir) & pin_mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; +} + int cpm1_gpiochip_add16(struct device *dev) { struct device_node *np = dev->of_node; @@ -498,6 +510,7 @@ int cpm1_gpiochip_add16(struct device *dev) gc->ngpio = 16; gc->direction_input = cpm1_gpio16_dir_in; gc->direction_output = cpm1_gpio16_dir_out; + gc->get_direction = cpm1_gpio16_get_direction; gc->get = cpm1_gpio16_get; gc->set = cpm1_gpio16_set; gc->to_irq = cpm1_gpio16_to_irq; @@ -604,6 +617,18 @@ static int cpm1_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } +static int cpm1_gpio32_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport32b __iomem *iop = cpm1_gc->regs; + u32 pin_mask = 1 << (31 - gpio); + + if (in_be32(&iop->dir) & pin_mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; +} + int cpm1_gpiochip_add32(struct device *dev) { struct device_node *np = dev->of_node; @@ -621,6 +646,7 @@ int cpm1_gpiochip_add32(struct device *dev) gc->ngpio = 32; gc->direction_input = cpm1_gpio32_dir_in; gc->direction_output = cpm1_gpio32_dir_out; + gc->get_direction = cpm1_gpio32_get_direction; gc->get = cpm1_gpio32_get; gc->set = cpm1_gpio32_set; gc->parent = dev; diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 10fa9b844fccd..94c1ffa8792e5 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -268,10 +268,12 @@ static int spufs_mem_mmap_access(struct vm_area_struct *vma, if (write && !(vma->vm_flags & VM_WRITE)) return -EACCES; + if (offset >= LS_SIZE) + return -EFAULT; if (spu_acquire(ctx)) return -EINTR; - if ((offset + len) > vma->vm_end) - len = vma->vm_end - offset; + if ((offset + len) > LS_SIZE) + len = LS_SIZE - offset; local_store = ctx->ops->get_ls(ctx); if (write) memcpy_toio(local_store + offset, buf, len); diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index 4dbb47ddbdcc4..06ed5e2aa2658 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -396,7 +396,8 @@ static void pnv_kexec_wait_secondaries_down(void) { int my_cpu, i, notified = -1; - my_cpu = get_cpu(); + /* Called with interrupts disabled, so the CPU is pinned. */ + my_cpu = raw_smp_processor_id(); for_each_online_cpu(i) { uint8_t status; diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index f7052b131a4c5..74910ce3a541c 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -154,6 +154,7 @@ config HV_PERF_CTRS config VPA_PMU tristate "VPA PMU events" depends on KVM_BOOK3S_64_HV && HV_PERF_CTRS + default m help Enable access to the VPA PMU counters via perf. This enables code that support measurement for KVM on PowerVM(KoP) feature. diff --git a/arch/powerpc/platforms/pseries/papr_platform_attributes.c b/arch/powerpc/platforms/pseries/papr_platform_attributes.c index c6159870de0e1..9c3758aa54c6e 100644 --- a/arch/powerpc/platforms/pseries/papr_platform_attributes.c +++ b/arch/powerpc/platforms/pseries/papr_platform_attributes.c @@ -271,11 +271,9 @@ static int __init papr_init(void) esi_buf_size = ESI_HDR_SIZE + (CURR_MAX_ESI_ATTRS * max_esi_attrs); temp_esi_buf = krealloc(esi_buf, esi_buf_size, GFP_KERNEL); - if (temp_esi_buf) - esi_buf = temp_esi_buf; - else - return -ENOMEM; - + if (!temp_esi_buf) + goto out_free_esi_buf; + esi_buf = temp_esi_buf; goto retry; } diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts index 0e1b0b8d394b9..6e9653827cfe2 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts @@ -77,6 +77,15 @@ regulator-max-microvolt = <1200000>; }; + regulator-1v8-syzygy { + compatible = "regulator-fixed"; + gpios = <&gpio2 30 GPIO_ACTIVE_HIGH>; + regulator-name = "syzygy_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + enable-active-high; + }; }; &gpio0 { @@ -118,20 +127,6 @@ "P8_PIN27", "P8_PIN28", "P8_PIN29", "P8_PIN30", "M2_W_DISABLE1", "M2_W_DISABLE2", "VIO_ENABLE", "SD_DET"; status = "okay"; - - vio-enable-hog { - gpio-hog; - gpios = <30 30>; - output-high; - line-name = "VIO_ENABLE"; - }; - - sd-det-hog { - gpio-hog; - gpios = <31 31>; - input; - line-name = "SD_DET"; - }; }; &i2c0 { @@ -316,7 +311,7 @@ mmc@1 { compatible = "mmc-spi-slot"; reg = <1>; - gpios = <&gpio2 31 1>; + cd-gpios = <&gpio2 31 GPIO_ACTIVE_LOW>; voltage-ranges = <3300 3300>; spi-max-frequency = <5000000>; disable-wp; diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi index d535d4c72763c..85d8df6437f23 100644 --- a/arch/riscv/boot/dts/microchip/mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi @@ -499,7 +499,7 @@ reg = <0x0 0x20120000 0x0 0x1000>; interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; interrupts = <0>, <1>, <2>, <3>, <4>, <5>, <6>, <7>, <8>, <9>, <10>, <11>, @@ -516,7 +516,7 @@ reg = <0x0 0x20121000 0x0 0x1000>; interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; interrupts = <32>, <33>, <34>, <35>, <36>, <37>, <38>, <39>, <40>, <41>, <42>, <43>, @@ -535,7 +535,7 @@ reg = <0x0 0x20122000 0x0 0x1000>; interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; interrupts = <64>, <65>, <66>, <67>, <68>, <69>, <70>, <71>, <72>, <73>, <74>, <75>, diff --git a/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts index 2f2ccd77af30a..ed3ff03f3b11b 100644 --- a/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts +++ b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts @@ -63,10 +63,6 @@ }; &gpio0 { - interrupts = <13>, <14>, <15>, <16>, - <17>, <18>, <19>, <20>, - <21>, <22>, <23>, <24>, - <25>, <26>; status ="okay"; gpio-line-names = "", "", "", "", "", "", "", "", @@ -74,12 +70,6 @@ }; &gpio1 { - interrupts = <27>, <28>, <29>, <30>, - <31>, <32>, <33>, <34>, - <35>, <36>, <37>, <38>, - <39>, <40>, <41>, <42>, - <43>, <44>, <45>, <46>, - <47>, <48>, <49>, <50>; status ="okay"; gpio-line-names = "", "", "LED1", "LED2", "LED3", "LED4", "LED5", "LED6", @@ -88,14 +78,6 @@ }; &gpio2 { - interrupts = <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>; pinctrl-names = "default"; pinctrl-0 = <&mdio1_gpio>, <&spi0_gpio>, <&can0_gpio>, <&pcie_gpio>, <&qspi_gpio>, <&uart3_gpio>, <&uart4_gpio>, <&can1_gpio>; @@ -107,6 +89,35 @@ "DIP4", "USR_IO11", "", "", "SWITCH1", "", "", ""; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &mac0 { status = "okay"; phy-mode = "sgmii"; diff --git a/arch/riscv/boot/dts/microchip/pic64gx.dtsi b/arch/riscv/boot/dts/microchip/pic64gx.dtsi index c164d7bc270a2..e6a24cc5715fa 100644 --- a/arch/riscv/boot/dts/microchip/pic64gx.dtsi +++ b/arch/riscv/boot/dts/microchip/pic64gx.dtsi @@ -295,6 +295,14 @@ #size-cells = <1>; #reset-cells = <1>; + irqmux: interrupt-controller@54 { + compatible = "microchip,pic64gx-irqmux", "microchip,mpfs-irqmux"; + reg = <0x54 0x4>; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-map-mask = <0x7f>; + }; + iomux0: pinctrl@200 { compatible = "microchip,pic64gx-pinctrl-iomux0", "microchip,mpfs-pinctrl-iomux0"; @@ -484,9 +492,13 @@ gpio0: gpio@20120000 { compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; reg = <0x0 0x20120000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; + interrupts = <0>, <1>, <2>, <3>, + <4>, <5>, <6>, <7>, + <8>, <9>, <10>, <11>, + <12>, <13>; clocks = <&clkcfg CLK_GPIO0>; gpio-controller; #gpio-cells = <2>; @@ -497,9 +509,15 @@ gpio1: gpio@20121000 { compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; reg = <0x0 0x20121000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; + interrupts = <32>, <33>, <34>, <35>, + <36>, <37>, <38>, <39>, + <40>, <41>, <42>, <43>, + <44>, <45>, <46>, <47>, + <48>, <49>, <50>, <51>, + <52>, <53>, <54>, <55>; clocks = <&clkcfg CLK_GPIO1>; gpio-controller; #gpio-cells = <2>; @@ -510,9 +528,17 @@ gpio2: gpio@20122000 { compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; reg = <0x0 0x20122000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; + interrupts = <64>, <65>, <66>, <67>, + <68>, <69>, <70>, <71>, + <72>, <73>, <74>, <75>, + <76>, <77>, <78>, <79>, + <80>, <81>, <82>, <83>, + <84>, <85>, <86>, <87>, + <88>, <89>, <90>, <91>, + <92>, <93>, <94>, <95>; clocks = <&clkcfg CLK_GPIO2>; gpio-controller; #gpio-cells = <2>; diff --git a/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi b/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi index 509488eee4321..fd8906b313d2e 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi @@ -263,7 +263,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <0>; + reg = <0x0>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -291,7 +291,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <1>; + reg = <0x1>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -319,7 +319,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <2>; + reg = <0x2>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -347,7 +347,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <3>; + reg = <0x3>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -375,7 +375,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <4>; + reg = <0x4>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -403,7 +403,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <5>; + reg = <0x5>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -431,7 +431,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <6>; + reg = <0x6>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -459,7 +459,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <7>; + reg = <0x7>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -487,7 +487,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <8>; + reg = <0x8>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -515,7 +515,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <9>; + reg = <0x9>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -533,7 +533,7 @@ }; }; - cpu10: cpu@10 { + cpu10: cpu@a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -543,7 +543,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <10>; + reg = <0xa>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -561,7 +561,7 @@ }; }; - cpu11: cpu@11 { + cpu11: cpu@b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -571,7 +571,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <11>; + reg = <0xb>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -589,7 +589,7 @@ }; }; - cpu12: cpu@12 { + cpu12: cpu@c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -599,7 +599,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <12>; + reg = <0xc>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -617,7 +617,7 @@ }; }; - cpu13: cpu@13 { + cpu13: cpu@d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -627,7 +627,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <13>; + reg = <0xd>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -645,7 +645,7 @@ }; }; - cpu14: cpu@14 { + cpu14: cpu@e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -655,7 +655,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <14>; + reg = <0xe>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -673,7 +673,7 @@ }; }; - cpu15: cpu@15 { + cpu15: cpu@f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -683,7 +683,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <15>; + reg = <0xf>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -701,7 +701,7 @@ }; }; - cpu16: cpu@16 { + cpu16: cpu@10 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -711,7 +711,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <16>; + reg = <0x10>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -729,7 +729,7 @@ }; }; - cpu17: cpu@17 { + cpu17: cpu@11 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -739,7 +739,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <17>; + reg = <0x11>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -757,7 +757,7 @@ }; }; - cpu18: cpu@18 { + cpu18: cpu@12 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -767,7 +767,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <18>; + reg = <0x12>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -785,7 +785,7 @@ }; }; - cpu19: cpu@19 { + cpu19: cpu@13 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -795,7 +795,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <19>; + reg = <0x13>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -813,7 +813,7 @@ }; }; - cpu20: cpu@20 { + cpu20: cpu@14 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -823,7 +823,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <20>; + reg = <0x14>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -841,7 +841,7 @@ }; }; - cpu21: cpu@21 { + cpu21: cpu@15 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -851,7 +851,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <21>; + reg = <0x15>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -869,7 +869,7 @@ }; }; - cpu22: cpu@22 { + cpu22: cpu@16 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -879,7 +879,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <22>; + reg = <0x16>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -897,7 +897,7 @@ }; }; - cpu23: cpu@23 { + cpu23: cpu@17 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -907,7 +907,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <23>; + reg = <0x17>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -925,7 +925,7 @@ }; }; - cpu24: cpu@24 { + cpu24: cpu@18 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -935,7 +935,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <24>; + reg = <0x18>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -953,7 +953,7 @@ }; }; - cpu25: cpu@25 { + cpu25: cpu@19 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -963,7 +963,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <25>; + reg = <0x19>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -981,7 +981,7 @@ }; }; - cpu26: cpu@26 { + cpu26: cpu@1a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -991,7 +991,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <26>; + reg = <0x1a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1009,7 +1009,7 @@ }; }; - cpu27: cpu@27 { + cpu27: cpu@1b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1019,7 +1019,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <27>; + reg = <0x1b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1037,7 +1037,7 @@ }; }; - cpu28: cpu@28 { + cpu28: cpu@1c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1047,7 +1047,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <28>; + reg = <0x1c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1065,7 +1065,7 @@ }; }; - cpu29: cpu@29 { + cpu29: cpu@1d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1075,7 +1075,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <29>; + reg = <0x1d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1093,7 +1093,7 @@ }; }; - cpu30: cpu@30 { + cpu30: cpu@1e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1103,7 +1103,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <30>; + reg = <0x1e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1121,7 +1121,7 @@ }; }; - cpu31: cpu@31 { + cpu31: cpu@1f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1131,7 +1131,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <31>; + reg = <0x1f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1149,7 +1149,7 @@ }; }; - cpu32: cpu@32 { + cpu32: cpu@20 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1159,7 +1159,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <32>; + reg = <0x20>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1177,7 +1177,7 @@ }; }; - cpu33: cpu@33 { + cpu33: cpu@21 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1187,7 +1187,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <33>; + reg = <0x21>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1205,7 +1205,7 @@ }; }; - cpu34: cpu@34 { + cpu34: cpu@22 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1215,7 +1215,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <34>; + reg = <0x22>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1233,7 +1233,7 @@ }; }; - cpu35: cpu@35 { + cpu35: cpu@23 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1243,7 +1243,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <35>; + reg = <0x23>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1261,7 +1261,7 @@ }; }; - cpu36: cpu@36 { + cpu36: cpu@24 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1271,7 +1271,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <36>; + reg = <0x24>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1289,7 +1289,7 @@ }; }; - cpu37: cpu@37 { + cpu37: cpu@25 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1299,7 +1299,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <37>; + reg = <0x25>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1317,7 +1317,7 @@ }; }; - cpu38: cpu@38 { + cpu38: cpu@26 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1327,7 +1327,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <38>; + reg = <0x26>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1345,7 +1345,7 @@ }; }; - cpu39: cpu@39 { + cpu39: cpu@27 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1355,7 +1355,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <39>; + reg = <0x27>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1373,7 +1373,7 @@ }; }; - cpu40: cpu@40 { + cpu40: cpu@28 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1383,7 +1383,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <40>; + reg = <0x28>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1401,7 +1401,7 @@ }; }; - cpu41: cpu@41 { + cpu41: cpu@29 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1411,7 +1411,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <41>; + reg = <0x29>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1429,7 +1429,7 @@ }; }; - cpu42: cpu@42 { + cpu42: cpu@2a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1439,7 +1439,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <42>; + reg = <0x2a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1457,7 +1457,7 @@ }; }; - cpu43: cpu@43 { + cpu43: cpu@2b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1467,7 +1467,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <43>; + reg = <0x2b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1485,7 +1485,7 @@ }; }; - cpu44: cpu@44 { + cpu44: cpu@2c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1495,7 +1495,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <44>; + reg = <0x2c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1513,7 +1513,7 @@ }; }; - cpu45: cpu@45 { + cpu45: cpu@2d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1523,7 +1523,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <45>; + reg = <0x2d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1541,7 +1541,7 @@ }; }; - cpu46: cpu@46 { + cpu46: cpu@2e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1551,7 +1551,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <46>; + reg = <0x2e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1569,7 +1569,7 @@ }; }; - cpu47: cpu@47 { + cpu47: cpu@2f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1579,7 +1579,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <47>; + reg = <0x2f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1597,7 +1597,7 @@ }; }; - cpu48: cpu@48 { + cpu48: cpu@30 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1607,7 +1607,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <48>; + reg = <0x30>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1625,7 +1625,7 @@ }; }; - cpu49: cpu@49 { + cpu49: cpu@31 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1635,7 +1635,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <49>; + reg = <0x31>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1653,7 +1653,7 @@ }; }; - cpu50: cpu@50 { + cpu50: cpu@32 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1663,7 +1663,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <50>; + reg = <0x32>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1681,7 +1681,7 @@ }; }; - cpu51: cpu@51 { + cpu51: cpu@33 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1691,7 +1691,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <51>; + reg = <0x33>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1709,7 +1709,7 @@ }; }; - cpu52: cpu@52 { + cpu52: cpu@34 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1719,7 +1719,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <52>; + reg = <0x34>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1737,7 +1737,7 @@ }; }; - cpu53: cpu@53 { + cpu53: cpu@35 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1747,7 +1747,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <53>; + reg = <0x35>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1765,7 +1765,7 @@ }; }; - cpu54: cpu@54 { + cpu54: cpu@36 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1775,7 +1775,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <54>; + reg = <0x36>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1793,7 +1793,7 @@ }; }; - cpu55: cpu@55 { + cpu55: cpu@37 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1803,7 +1803,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <55>; + reg = <0x37>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1821,7 +1821,7 @@ }; }; - cpu56: cpu@56 { + cpu56: cpu@38 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1831,7 +1831,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <56>; + reg = <0x38>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1849,7 +1849,7 @@ }; }; - cpu57: cpu@57 { + cpu57: cpu@39 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1859,7 +1859,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <57>; + reg = <0x39>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1877,7 +1877,7 @@ }; }; - cpu58: cpu@58 { + cpu58: cpu@3a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1887,7 +1887,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <58>; + reg = <0x3a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1905,7 +1905,7 @@ }; }; - cpu59: cpu@59 { + cpu59: cpu@3b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1915,7 +1915,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <59>; + reg = <0x3b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1933,7 +1933,7 @@ }; }; - cpu60: cpu@60 { + cpu60: cpu@3c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1943,7 +1943,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <60>; + reg = <0x3c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1961,7 +1961,7 @@ }; }; - cpu61: cpu@61 { + cpu61: cpu@3d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1971,7 +1971,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <61>; + reg = <0x3d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1989,7 +1989,7 @@ }; }; - cpu62: cpu@62 { + cpu62: cpu@3e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1999,7 +1999,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <62>; + reg = <0x3e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2017,7 +2017,7 @@ }; }; - cpu63: cpu@63 { + cpu63: cpu@3f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -2027,7 +2027,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <63>; + reg = <0x3f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; diff --git a/arch/riscv/boot/dts/sophgo/sg2042.dtsi b/arch/riscv/boot/dts/sophgo/sg2042.dtsi index 9fddf3f0b3b99..3af7705497426 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042.dtsi @@ -417,6 +417,7 @@ vendor-id = <0x1f1c>; device-id = <0x2042>; cdns,no-bar-match-nbits = <48>; + dma-coherent; msi-parent = <&msi>; status = "disabled"; }; @@ -439,6 +440,7 @@ vendor-id = <0x1f1c>; device-id = <0x2042>; cdns,no-bar-match-nbits = <48>; + dma-coherent; msi-parent = <&msi>; status = "disabled"; }; @@ -461,6 +463,7 @@ vendor-id = <0x1f1c>; device-id = <0x2042>; cdns,no-bar-match-nbits = <48>; + dma-coherent; msi-parent = <&msi>; status = "disabled"; }; @@ -483,6 +486,7 @@ vendor-id = <0x1f1c>; device-id = <0x2042>; cdns,no-bar-match-nbits = <48>; + dma-coherent; msi-parent = <&msi>; status = "disabled"; }; diff --git a/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi index 3135409c21492..f66a382c95bd9 100644 --- a/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi @@ -14,7 +14,7 @@ cpu0: cpu@0 { compatible = "thead,c920", "riscv"; - reg = <0>; + reg = <0x0>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -50,7 +50,7 @@ cpu1: cpu@1 { compatible = "thead,c920", "riscv"; - reg = <1>; + reg = <0x1>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -86,7 +86,7 @@ cpu2: cpu@2 { compatible = "thead,c920", "riscv"; - reg = <2>; + reg = <0x2>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -122,7 +122,7 @@ cpu3: cpu@3 { compatible = "thead,c920", "riscv"; - reg = <3>; + reg = <0x3>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -158,7 +158,7 @@ cpu4: cpu@4 { compatible = "thead,c920", "riscv"; - reg = <4>; + reg = <0x4>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -194,7 +194,7 @@ cpu5: cpu@5 { compatible = "thead,c920", "riscv"; - reg = <5>; + reg = <0x5>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -230,7 +230,7 @@ cpu6: cpu@6 { compatible = "thead,c920", "riscv"; - reg = <6>; + reg = <0x6>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -266,7 +266,7 @@ cpu7: cpu@7 { compatible = "thead,c920", "riscv"; - reg = <7>; + reg = <0x7>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -302,7 +302,7 @@ cpu8: cpu@8 { compatible = "thead,c920", "riscv"; - reg = <8>; + reg = <0x8>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -338,7 +338,7 @@ cpu9: cpu@9 { compatible = "thead,c920", "riscv"; - reg = <9>; + reg = <0x9>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -372,9 +372,9 @@ }; }; - cpu10: cpu@10 { + cpu10: cpu@a { compatible = "thead,c920", "riscv"; - reg = <10>; + reg = <0xa>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -408,9 +408,9 @@ }; }; - cpu11: cpu@11 { + cpu11: cpu@b { compatible = "thead,c920", "riscv"; - reg = <11>; + reg = <0xb>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -444,9 +444,9 @@ }; }; - cpu12: cpu@12 { + cpu12: cpu@c { compatible = "thead,c920", "riscv"; - reg = <12>; + reg = <0xc>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -480,9 +480,9 @@ }; }; - cpu13: cpu@13 { + cpu13: cpu@d { compatible = "thead,c920", "riscv"; - reg = <13>; + reg = <0xd>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -516,9 +516,9 @@ }; }; - cpu14: cpu@14 { + cpu14: cpu@e { compatible = "thead,c920", "riscv"; - reg = <14>; + reg = <0xe>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -552,9 +552,9 @@ }; }; - cpu15: cpu@15 { + cpu15: cpu@f { compatible = "thead,c920", "riscv"; - reg = <15>; + reg = <0xf>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -588,9 +588,9 @@ }; }; - cpu16: cpu@16 { + cpu16: cpu@10 { compatible = "thead,c920", "riscv"; - reg = <16>; + reg = <0x10>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -624,9 +624,9 @@ }; }; - cpu17: cpu@17 { + cpu17: cpu@11 { compatible = "thead,c920", "riscv"; - reg = <17>; + reg = <0x11>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -660,9 +660,9 @@ }; }; - cpu18: cpu@18 { + cpu18: cpu@12 { compatible = "thead,c920", "riscv"; - reg = <18>; + reg = <0x12>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -696,9 +696,9 @@ }; }; - cpu19: cpu@19 { + cpu19: cpu@13 { compatible = "thead,c920", "riscv"; - reg = <19>; + reg = <0x13>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -732,9 +732,9 @@ }; }; - cpu20: cpu@20 { + cpu20: cpu@14 { compatible = "thead,c920", "riscv"; - reg = <20>; + reg = <0x14>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -768,9 +768,9 @@ }; }; - cpu21: cpu@21 { + cpu21: cpu@15 { compatible = "thead,c920", "riscv"; - reg = <21>; + reg = <0x15>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -804,9 +804,9 @@ }; }; - cpu22: cpu@22 { + cpu22: cpu@16 { compatible = "thead,c920", "riscv"; - reg = <22>; + reg = <0x16>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -840,9 +840,9 @@ }; }; - cpu23: cpu@23 { + cpu23: cpu@17 { compatible = "thead,c920", "riscv"; - reg = <23>; + reg = <0x17>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -876,9 +876,9 @@ }; }; - cpu24: cpu@24 { + cpu24: cpu@18 { compatible = "thead,c920", "riscv"; - reg = <24>; + reg = <0x18>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -912,9 +912,9 @@ }; }; - cpu25: cpu@25 { + cpu25: cpu@19 { compatible = "thead,c920", "riscv"; - reg = <25>; + reg = <0x19>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -948,9 +948,9 @@ }; }; - cpu26: cpu@26 { + cpu26: cpu@1a { compatible = "thead,c920", "riscv"; - reg = <26>; + reg = <0x1a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -984,9 +984,9 @@ }; }; - cpu27: cpu@27 { + cpu27: cpu@1b { compatible = "thead,c920", "riscv"; - reg = <27>; + reg = <0x1b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1020,9 +1020,9 @@ }; }; - cpu28: cpu@28 { + cpu28: cpu@1c { compatible = "thead,c920", "riscv"; - reg = <28>; + reg = <0x1c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1056,9 +1056,9 @@ }; }; - cpu29: cpu@29 { + cpu29: cpu@1d { compatible = "thead,c920", "riscv"; - reg = <29>; + reg = <0x1d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1092,9 +1092,9 @@ }; }; - cpu30: cpu@30 { + cpu30: cpu@1e { compatible = "thead,c920", "riscv"; - reg = <30>; + reg = <0x1e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1128,9 +1128,9 @@ }; }; - cpu31: cpu@31 { + cpu31: cpu@1f { compatible = "thead,c920", "riscv"; - reg = <31>; + reg = <0x1f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1164,9 +1164,9 @@ }; }; - cpu32: cpu@32 { + cpu32: cpu@20 { compatible = "thead,c920", "riscv"; - reg = <32>; + reg = <0x20>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1200,9 +1200,9 @@ }; }; - cpu33: cpu@33 { + cpu33: cpu@21 { compatible = "thead,c920", "riscv"; - reg = <33>; + reg = <0x21>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1236,9 +1236,9 @@ }; }; - cpu34: cpu@34 { + cpu34: cpu@22 { compatible = "thead,c920", "riscv"; - reg = <34>; + reg = <0x22>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1272,9 +1272,9 @@ }; }; - cpu35: cpu@35 { + cpu35: cpu@23 { compatible = "thead,c920", "riscv"; - reg = <35>; + reg = <0x23>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1308,9 +1308,9 @@ }; }; - cpu36: cpu@36 { + cpu36: cpu@24 { compatible = "thead,c920", "riscv"; - reg = <36>; + reg = <0x24>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1344,9 +1344,9 @@ }; }; - cpu37: cpu@37 { + cpu37: cpu@25 { compatible = "thead,c920", "riscv"; - reg = <37>; + reg = <0x25>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1380,9 +1380,9 @@ }; }; - cpu38: cpu@38 { + cpu38: cpu@26 { compatible = "thead,c920", "riscv"; - reg = <38>; + reg = <0x26>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1416,9 +1416,9 @@ }; }; - cpu39: cpu@39 { + cpu39: cpu@27 { compatible = "thead,c920", "riscv"; - reg = <39>; + reg = <0x27>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1452,9 +1452,9 @@ }; }; - cpu40: cpu@40 { + cpu40: cpu@28 { compatible = "thead,c920", "riscv"; - reg = <40>; + reg = <0x28>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1488,9 +1488,9 @@ }; }; - cpu41: cpu@41 { + cpu41: cpu@29 { compatible = "thead,c920", "riscv"; - reg = <41>; + reg = <0x29>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1524,9 +1524,9 @@ }; }; - cpu42: cpu@42 { + cpu42: cpu@2a { compatible = "thead,c920", "riscv"; - reg = <42>; + reg = <0x2a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1560,9 +1560,9 @@ }; }; - cpu43: cpu@43 { + cpu43: cpu@2b { compatible = "thead,c920", "riscv"; - reg = <43>; + reg = <0x2b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1596,9 +1596,9 @@ }; }; - cpu44: cpu@44 { + cpu44: cpu@2c { compatible = "thead,c920", "riscv"; - reg = <44>; + reg = <0x2c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1632,9 +1632,9 @@ }; }; - cpu45: cpu@45 { + cpu45: cpu@2d { compatible = "thead,c920", "riscv"; - reg = <45>; + reg = <0x2d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1668,9 +1668,9 @@ }; }; - cpu46: cpu@46 { + cpu46: cpu@2e { compatible = "thead,c920", "riscv"; - reg = <46>; + reg = <0x2e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1704,9 +1704,9 @@ }; }; - cpu47: cpu@47 { + cpu47: cpu@2f { compatible = "thead,c920", "riscv"; - reg = <47>; + reg = <0x2f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1740,9 +1740,9 @@ }; }; - cpu48: cpu@48 { + cpu48: cpu@30 { compatible = "thead,c920", "riscv"; - reg = <48>; + reg = <0x30>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1776,9 +1776,9 @@ }; }; - cpu49: cpu@49 { + cpu49: cpu@31 { compatible = "thead,c920", "riscv"; - reg = <49>; + reg = <0x31>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1812,9 +1812,9 @@ }; }; - cpu50: cpu@50 { + cpu50: cpu@32 { compatible = "thead,c920", "riscv"; - reg = <50>; + reg = <0x32>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1848,9 +1848,9 @@ }; }; - cpu51: cpu@51 { + cpu51: cpu@33 { compatible = "thead,c920", "riscv"; - reg = <51>; + reg = <0x33>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1884,9 +1884,9 @@ }; }; - cpu52: cpu@52 { + cpu52: cpu@34 { compatible = "thead,c920", "riscv"; - reg = <52>; + reg = <0x34>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1920,9 +1920,9 @@ }; }; - cpu53: cpu@53 { + cpu53: cpu@35 { compatible = "thead,c920", "riscv"; - reg = <53>; + reg = <0x35>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1956,9 +1956,9 @@ }; }; - cpu54: cpu@54 { + cpu54: cpu@36 { compatible = "thead,c920", "riscv"; - reg = <54>; + reg = <0x36>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1992,9 +1992,9 @@ }; }; - cpu55: cpu@55 { + cpu55: cpu@37 { compatible = "thead,c920", "riscv"; - reg = <55>; + reg = <0x37>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2028,9 +2028,9 @@ }; }; - cpu56: cpu@56 { + cpu56: cpu@38 { compatible = "thead,c920", "riscv"; - reg = <56>; + reg = <0x38>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2064,9 +2064,9 @@ }; }; - cpu57: cpu@57 { + cpu57: cpu@39 { compatible = "thead,c920", "riscv"; - reg = <57>; + reg = <0x39>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2100,9 +2100,9 @@ }; }; - cpu58: cpu@58 { + cpu58: cpu@3a { compatible = "thead,c920", "riscv"; - reg = <58>; + reg = <0x3a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2136,9 +2136,9 @@ }; }; - cpu59: cpu@59 { + cpu59: cpu@3b { compatible = "thead,c920", "riscv"; - reg = <59>; + reg = <0x3b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2172,9 +2172,9 @@ }; }; - cpu60: cpu@60 { + cpu60: cpu@3c { compatible = "thead,c920", "riscv"; - reg = <60>; + reg = <0x3c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2208,9 +2208,9 @@ }; }; - cpu61: cpu@61 { + cpu61: cpu@3d { compatible = "thead,c920", "riscv"; - reg = <61>; + reg = <0x3d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2244,9 +2244,9 @@ }; }; - cpu62: cpu@62 { + cpu62: cpu@3e { compatible = "thead,c920", "riscv"; - reg = <62>; + reg = <0x3e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2280,9 +2280,9 @@ }; }; - cpu63: cpu@63 { + cpu63: cpu@3f { compatible = "thead,c920", "riscv"; - reg = <63>; + reg = <0x3f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts index 5790d927b93db..333ac8ebf3f51 100644 --- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts @@ -19,7 +19,7 @@ }; chosen { - stdout-path = "serial0"; + stdout-path = "serial0:115200n8"; }; leds { diff --git a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts index afaad59e6bce2..c47bfb1597306 100644 --- a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts +++ b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts @@ -20,7 +20,7 @@ }; chosen { - stdout-path = "serial0"; + stdout-path = "serial0:115200n8"; }; leds { @@ -367,7 +367,7 @@ opensbi@70000 { reg = <0x70000 0x30000>; }; - uboot@a00000 { + uboot@a0000 { reg = <0xa0000 0x760000>; }; }; diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi index f0bad6855c970..b0b9c19b56a09 100644 --- a/arch/riscv/boot/dts/spacemit/k1.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1.dtsi @@ -333,7 +333,7 @@ osc_32k: clock-32k { compatible = "fixed-clock"; - clock-frequency = <32000>; + clock-frequency = <32768>; clock-output-names = "osc_32k"; #clock-cells = <0>; }; diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi index 815debd16409b..719850aa7aac5 100644 --- a/arch/riscv/boot/dts/spacemit/k3.dtsi +++ b/arch/riscv/boot/dts/spacemit/k3.dtsi @@ -424,7 +424,7 @@ osc_32k: clock-32k { compatible = "fixed-clock"; - clock-frequency = <32000>; + clock-frequency = <32768>; clock-output-names = "osc_32k"; #clock-cells = <0>; }; @@ -803,6 +803,7 @@ clocks = <&syscon_apbc CLK_APBC_AIB>, <&syscon_apbc CLK_APBC_AIB_BUS>; clock-names = "func", "bus"; + spacemit,apbc = <&syscon_apbc>; }; uart10: serial@d401f000 { diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 0092513c3376c..b1a2ac6657926 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -43,20 +43,23 @@ do { \ #ifdef CONFIG_64BIT extern u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1]; extern char _end[]; +static inline void mark_new_valid_map(void) +{ + int i; + + /* + * We don't care if concurrently a cpu resets this value since + * the only place this can happen is in handle_exception() where + * an sfence.vma is emitted. + */ + for (i = 0; i < ARRAY_SIZE(new_vmalloc); ++i) + new_vmalloc[i] = -1ULL; +} #define flush_cache_vmap flush_cache_vmap static inline void flush_cache_vmap(unsigned long start, unsigned long end) { - if (is_vmalloc_or_module_addr((void *)start)) { - int i; - - /* - * We don't care if concurrently a cpu resets this value since - * the only place this can happen is in handle_exception() where - * an sfence.vma is emitted. - */ - for (i = 0; i < ARRAY_SIZE(new_vmalloc); ++i) - new_vmalloc[i] = -1ULL; - } + if (is_vmalloc_or_module_addr((void *)start)) + mark_new_valid_map(); } #define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end) #endif diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h index 176b570ef9827..065811fca594d 100644 --- a/arch/riscv/include/asm/cpu_ops.h +++ b/arch/riscv/include/asm/cpu_ops.h @@ -24,7 +24,7 @@ struct cpu_operations { struct task_struct *tidle); #ifdef CONFIG_HOTPLUG_CPU void (*cpu_stop)(void); - int (*cpu_is_stopped)(unsigned int cpu); + bool (*cpu_is_stopped)(unsigned int cpu); #endif }; diff --git a/arch/riscv/include/asm/kfence.h b/arch/riscv/include/asm/kfence.h index d08bf7fb3aee6..29cb3a6ee113d 100644 --- a/arch/riscv/include/asm/kfence.h +++ b/arch/riscv/include/asm/kfence.h @@ -6,6 +6,7 @@ #include #include #include +#include #include static inline bool arch_kfence_init_pool(void) @@ -17,10 +18,12 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) { pte_t *pte = virt_to_kpte(addr); - if (protect) + if (protect) { set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~_PAGE_PRESENT)); - else + } else { set_pte(pte, __pte(pte_val(ptep_get(pte)) | _PAGE_PRESENT)); + mark_new_valid_map(); + } preempt_disable(); local_flush_tlb_kernel_range(addr, addr + PAGE_SIZE); diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index a1a7c6520a095..5d5756bda82e3 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -93,6 +93,16 @@ */ #define vmemmap ((struct page *)VMEMMAP_START - vmemmap_start_pfn) +/* Needed to limit get_free_mem_region() */ +#if defined(CONFIG_FLATMEM) +#define DIRECT_MAP_PHYSMEM_END (phys_ram_base + KERN_VIRT_SIZE - 1) +#elif defined(CONFIG_SPARSEMEM_VMEMMAP) +#define DIRECT_MAP_PHYSMEM_END \ + ((vmemmap_start_pfn + VMEMMAP_SIZE / sizeof(struct page)) * PAGE_SIZE - 1) +#elif defined(CONFIG_SPARSEMEM) +/* DIRECT_MAP_PHYSMEM_END is not limited by VA space assignment in this case */ +#endif + #define PCI_IO_SIZE SZ_16M #define PCI_IO_END VMEMMAP_START #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index 35bf830a55761..f7998d9ad9b2d 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -12,12 +12,15 @@ * All systems with an MMU have a VDSO, but systems without an MMU don't * support shared libraries and therefore don't have one. */ -#ifdef CONFIG_MMU #define __VDSO_PAGES 4 #ifndef __ASSEMBLER__ + +#ifdef CONFIG_MMU #include +#endif + #ifdef CONFIG_RISCV_USER_CFI #include #endif @@ -38,15 +41,12 @@ #define COMPAT_VDSO_SYMBOL(base, name) \ (void __user *)((unsigned long)(base) + compat__vdso_##name##_offset) -extern char compat_vdso_start[], compat_vdso_end[]; - #endif /* CONFIG_COMPAT */ extern char vdso_start[], vdso_end[]; extern char vdso_cfi_start[], vdso_cfi_end[]; +extern char compat_vdso_start[], compat_vdso_end[]; #endif /* !__ASSEMBLER__ */ -#endif /* CONFIG_MMU */ - #endif /* _ASM_RISCV_VDSO_H */ diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 7642704c7f184..104dc0862c5c2 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -173,15 +173,14 @@ static void __init_or_module _apply_alternatives(struct alt_entry *begin, stage); } -#ifdef CONFIG_MMU -static void __init apply_vdso_alternatives(void) +static void __init apply_vdso_alternatives(void *start) { const Elf_Ehdr *hdr; const Elf_Shdr *shdr; const Elf_Shdr *alt; struct alt_entry *begin, *end; - hdr = (Elf_Ehdr *)vdso_start; + hdr = (Elf_Ehdr *)start; shdr = (void *)hdr + hdr->e_shoff; alt = find_section(hdr, shdr, ".alternative"); if (!alt) @@ -194,9 +193,6 @@ static void __init apply_vdso_alternatives(void) (struct alt_entry *)end, RISCV_ALTERNATIVES_BOOT); } -#else -static void __init apply_vdso_alternatives(void) { } -#endif void __init apply_boot_alternatives(void) { @@ -207,7 +203,11 @@ void __init apply_boot_alternatives(void) (struct alt_entry *)__alt_end, RISCV_ALTERNATIVES_BOOT); - apply_vdso_alternatives(); + if (IS_ENABLED(CONFIG_MMU)) + apply_vdso_alternatives(vdso_start); + + if (IS_ENABLED(CONFIG_RISCV_USER_CFI)) + apply_vdso_alternatives(vdso_cfi_start); } /* diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index af827448a609e..a75f0cfea1e9f 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -501,8 +501,8 @@ void asm_offsets(void) OFFSET(SBI_HART_BOOT_STACK_PTR_OFFSET, sbi_hart_boot_data, stack_ptr); DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe), STACK_ALIGN)); - OFFSET(STACKFRAME_FP, stackframe, fp); - OFFSET(STACKFRAME_RA, stackframe, ra); + DEFINE(STACKFRAME_FP, offsetof(struct stackframe, fp) - sizeof(struct stackframe)); + DEFINE(STACKFRAME_RA, offsetof(struct stackframe, ra) - sizeof(struct stackframe)); #ifdef CONFIG_FUNCTION_TRACER DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func)); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c index 26b085dbdd073..6c9a1ef2d45af 100644 --- a/arch/riscv/kernel/cacheinfo.c +++ b/arch/riscv/kernel/cacheinfo.c @@ -133,7 +133,7 @@ int populate_cache_leaves(unsigned int cpu) ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); levels = level; } - of_node_put(np); + of_node_put(prev); return 0; } diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index a0ee426f6d938..0bc56d8381b6b 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -57,8 +57,8 @@ void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) /* Verify from the firmware if the cpu is really stopped*/ if (cpu_ops->cpu_is_stopped) ret = cpu_ops->cpu_is_stopped(cpu); - if (ret) - pr_warn("CPU%u may not have stopped: %d\n", cpu, ret); + if (!ret) + pr_warn("CPU%u may not have stopped\n", cpu); } /* diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index 00aff669f5f2f..146ceab1011fe 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -88,16 +88,19 @@ static void sbi_cpu_stop(void) pr_crit("Unable to stop the cpu %d (%d)\n", smp_processor_id(), ret); } -static int sbi_cpu_is_stopped(unsigned int cpuid) +static bool sbi_cpu_is_stopped(unsigned int cpuid) { int rc; unsigned long hartid = cpuid_to_hartid_map(cpuid); rc = sbi_hsm_hart_get_status(hartid); - if (rc == SBI_HSM_STATE_STOPPED) - return 0; - return rc; + if (rc != SBI_HSM_STATE_STOPPED) { + pr_warn("HART%lu isn't stopped; status %d\n", hartid, rc); + return false; + } + + return true; } #endif diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index d011fb51c59a0..9a06ed983bb06 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -75,8 +75,11 @@ /* Atomically reset the current cpu bit in new_vmalloc */ amoxor.d a0, a1, (a0) - /* Only emit a sfence.vma if the uarch caches invalid entries */ - ALTERNATIVE("sfence.vma", "nop", 0, RISCV_ISA_EXT_SVVPTC, 1) + /* + * A sfence.vma is required here. Even if we had Svvptc, there's no + * guarantee that after returning we wouldn't just fault again. + */ + sfence.vma REG_L a0, TASK_TI_A0(tp) REG_L a1, TASK_TI_A1(tp) @@ -136,8 +139,10 @@ SYM_CODE_START(handle_exception) #ifdef CONFIG_64BIT /* - * The RISC-V kernel does not eagerly emit a sfence.vma after each - * new vmalloc mapping, which may result in exceptions: + * The RISC-V kernel does not flush TLBs on all CPUS after each new + * vmalloc mapping or kfence_unprotect(), which may result in + * exceptions: + * * - if the uarch caches invalid entries, the new mapping would not be * observed by the page table walker and an invalidation is needed. * - if the uarch does not cache invalid entries, a reordered access @@ -381,8 +386,8 @@ SYM_CODE_END(ret_from_fork_user_asm) SYM_FUNC_START(call_on_irq_stack) /* Create a frame record to save ra and s0 (fp) */ addi sp, sp, -STACKFRAME_SIZE_ON_STACK - REG_S ra, STACKFRAME_RA(sp) - REG_S s0, STACKFRAME_FP(sp) + REG_S ra, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_RA)(sp) + REG_S s0, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_FP)(sp) addi s0, sp, STACKFRAME_SIZE_ON_STACK /* Switch to the per-CPU shadow call stack */ @@ -400,8 +405,8 @@ SYM_FUNC_START(call_on_irq_stack) /* Switch back to the thread stack and restore ra and s0 */ addi sp, s0, -STACKFRAME_SIZE_ON_STACK - REG_L ra, STACKFRAME_RA(sp) - REG_L s0, STACKFRAME_FP(sp) + REG_L ra, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_RA)(sp) + REG_L s0, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_FP)(sp) addi sp, sp, STACKFRAME_SIZE_ON_STACK ret diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c index 2306ce3e5f229..738df176ff6f1 100644 --- a/arch/riscv/kernel/machine_kexec.c +++ b/arch/riscv/kernel/machine_kexec.c @@ -41,6 +41,9 @@ machine_kexec_prepare(struct kimage *image) if (image->segment[i].memsz <= sizeof(fdt)) continue; + if (!image->segment[i].buf) + continue; + if (image->file_mode) memcpy(&fdt, image->segment[i].buf, sizeof(fdt)); else if (copy_from_user(&fdt, image->segment[i].buf, sizeof(fdt))) diff --git a/arch/riscv/kernel/probes/rethook_trampoline.S b/arch/riscv/kernel/probes/rethook_trampoline.S index f2cd83d9b0f00..c3aa8d8cf5aff 100644 --- a/arch/riscv/kernel/probes/rethook_trampoline.S +++ b/arch/riscv/kernel/probes/rethook_trampoline.S @@ -41,6 +41,9 @@ REG_S x29, PT_T4(sp) REG_S x30, PT_T5(sp) REG_S x31, PT_T6(sp) + /* save original sp */ + addi a0, sp, PT_SIZE_ON_STACK + REG_S a0, PT_SP(sp) .endm .macro restore_all_base_regs diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index b41b6255751cb..31fd3abb57d32 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -129,7 +129,7 @@ void notrace walk_stackframe(struct task_struct *task, while (!kstack_end(ksp)) { if (__kernel_text_address(pc) && unlikely(!fn(arg, pc))) break; - pc = READ_ONCE_NOCHECK(*ksp++) - 0x4; + pc = READ_ONCE_NOCHECK(*ksp++); } } diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index a842dc034571d..43ee881f6c6fd 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -69,9 +69,9 @@ CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY endif # Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) +CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) +CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) # Force dependency $(obj)/$(vdso_o): $(obj)/$(vdso_so) diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S index e82987dc37394..d6f96b1abe409 100644 --- a/arch/riscv/kernel/vdso/rt_sigreturn.S +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -7,11 +7,19 @@ #include #include +/* + * WARNING: Do NOT add a CFI landing pad at the start of this function. + * Unwinders such as libgcc identify the sigreturn trampoline by matching the + * instruction sequence. Adding a landing pad here would break unwinding from + * signal handlers. + * + * This trampoline is used only for signal return and not via an indirect + * call/jump from userspace, so adding CFI landing pad is unnecessary. + */ .text SYM_FUNC_START(__vdso_rt_sigreturn) .cfi_startproc .cfi_signal_frame - vdso_lpad li a7, __NR_rt_sigreturn ecall .cfi_endproc diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 2f1109dbf105b..404a98a0fc90e 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1321,7 +1321,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + return bpf_prog_pack_alloc(size, bpf_fill_ill_insns, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 4365d07aaf547..ce3bd3762e08c 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -109,7 +109,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr bpf_jit_binary_pack_alloc(prog_size + extable_size, &jit_data->ro_image, sizeof(u32), &jit_data->header, &jit_data->image, - bpf_fill_ill_insns); + bpf_fill_ill_insns, + bpf_prog_was_classic(prog)); if (!jit_data->ro_header) goto out_offset; diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 9921a3772bf73..c7a65e129dbc1 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -163,7 +163,6 @@ config S390 select ARCH_WANTS_THP_SWAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 - select DCACHE_WORD_ACCESS if !KMSAN select DYNAMIC_FTRACE if FUNCTION_TRACER select FUNCTION_ALIGNMENT_8B if CC_IS_GCC select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d23ea0c94e4ea..99748c20e7671 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -12,7 +12,6 @@ #define EX_TYPE_UA_FAULT 3 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 -#define EX_TYPE_ZEROPAD 7 #define EX_TYPE_FPC 8 #define EX_TYPE_UA_MVCOS_TO 9 #define EX_TYPE_UA_MVCOS_FROM 10 @@ -80,9 +79,6 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) -#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ - __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) - #define EX_TABLE_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 2c6cee8241e04..4740c75649ebb 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -980,6 +980,8 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) static inline void set_pte(pte_t *ptep, pte_t pte) { + if (pte_present(pte)) + pte = clear_pte_bit(pte, __pgprot(_PAGE_UNUSED)); WRITE_ONCE(*ptep, pte); } @@ -1332,8 +1334,6 @@ pgprot_t pgprot_writecombine(pgprot_t prot); static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry, unsigned int nr) { - if (pte_present(entry)) - entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED)); page_table_check_ptes_set(mm, addr, ptep, entry, nr); for (;;) { set_pte(ptep, entry); diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index dff035372601e..a9f32c53f699b 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -30,8 +30,7 @@ void debug_user_asce(int exit); #define uaccess_kmsan_or_inline __always_inline #endif -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER static uaccess_kmsan_or_inline __must_check unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long size) diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index eaa19dee76994..e9287036392d0 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -4,7 +4,6 @@ #include #include -#include #include struct word_at_a_time { @@ -41,25 +40,4 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } -/* - * Load an unaligned word from kernel space. - * - * In the (very unlikely) case of the word being a page-crosser - * and the next page not being mapped, take the exception and - * return zeroes in the non-existing part. - */ -static inline unsigned long load_unaligned_zeropad(const void *addr) -{ - unsigned long data; - - asm_inline volatile( - "0: lg %[data],0(%[addr])\n" - "1: nopr %%r7\n" - EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) - EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) - : [data] "=d" (data) - : [addr] "a" (addr), "m" (*(unsigned long *)addr)); - return data; -} - #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/s390/kernel/diag/diag310.c b/arch/s390/kernel/diag/diag310.c index f411562aa7f6b..c68ec9d28513d 100644 --- a/arch/s390/kernel/diag/diag310.c +++ b/arch/s390/kernel/diag/diag310.c @@ -190,17 +190,18 @@ static int memtop_get_stride_len(unsigned long *res) static int memtop_get_page_count(unsigned long *res, unsigned long level) { static unsigned long memtop_pages[DIAG310_LEVELMAX]; - unsigned long pages; + unsigned long pages, idx; int rc; if (level > DIAG310_LEVELMAX || level < DIAG310_LEVELMIN) return -EINVAL; - pages = READ_ONCE(memtop_pages[level - 1]); + idx = array_index_nospec(level - 1, ARRAY_SIZE(memtop_pages)); + pages = READ_ONCE(memtop_pages[idx]); if (!pages) { rc = diag310_get_memtop_size(&pages, level); if (rc) return rc; - WRITE_ONCE(memtop_pages[level - 1], pages); + WRITE_ONCE(memtop_pages[idx], pages); } *res = pages; return 0; diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 7aa655664eccb..2076ac22e2c49 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -768,6 +769,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_user)); ev = cpumf_generic_events_user[ev]; } } else if (!attr->exclude_kernel && attr->exclude_user) { @@ -778,6 +780,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_basic)); ev = cpumf_generic_events_basic[ev]; } } diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 0df95dcb21012..e4bd273e31f4e 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -50,7 +50,7 @@ void ret_from_fork(void) asm("ret_from_fork"); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs) { - void (*func)(void *arg); + int (*func)(void *arg); schedule_tail(prev); diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 4a41c0247ffa2..a4fe664f65ee1 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -45,7 +45,7 @@ int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc) mc->pts[mc->n_pts] = o; } for ( ; mc->n_rmaps < KVM_S390_MMU_CACHE_N_RMAPS; mc->n_rmaps++) { - o = kzalloc_obj(*mc->rmaps[0], GFP_KERNEL_ACCOUNT); + o = kzalloc_obj(struct vsie_rmap, GFP_KERNEL_ACCOUNT); if (!o) return -ENOMEM; mc->rmaps[mc->n_rmaps] = o; diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 20e28b183c1ac..36102b2727fbf 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1419,8 +1419,8 @@ static int walk_guest_tables(struct gmap *sg, unsigned long saddr, struct pgtwal return kvm_s390_get_guest_page(kvm, entries + LEVEL_MEM, table.pte.pfra, wr); } -static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union pte *ptep, - struct guest_fault *f, bool p) +static int _do_shadow_pte(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gpa_t raddr, + union pte *ptep_h, union pte *ptep, struct guest_fault *f, bool p) { union pgste pgste; union pte newpte; @@ -1430,7 +1430,7 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union lockdep_assert_held(&sg->parent->children_lock); scoped_guard(spinlock, &sg->host_to_rmap_lock) - rc = gmap_insert_rmap(sg, f->gfn, gpa_to_gfn(raddr), TABLE_TYPE_PAGE_TABLE); + rc = gmap_insert_rmap(mc, sg, f->gfn, gpa_to_gfn(raddr), TABLE_TYPE_PAGE_TABLE); if (rc) return rc; @@ -1462,8 +1462,8 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union return 0; } -static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, union crste *table, - struct guest_fault *f, bool p) +static int _do_shadow_crste(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gpa_t raddr, + union crste *host, union crste *table, struct guest_fault *f, bool p) { union crste newcrste, oldcrste; unsigned long mask; @@ -1476,7 +1476,7 @@ static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, uni mask = is_pmd(*table) ? _SEGMENT_FR_MASK : _REGION3_FR_MASK; r_gfn = gpa_to_gfn(raddr) & mask; scoped_guard(spinlock, &sg->host_to_rmap_lock) - rc = gmap_insert_rmap(sg, f->gfn & mask, r_gfn, host->h.tt); + rc = gmap_insert_rmap(mc, sg, f->gfn & mask, r_gfn, host->h.tt); if (rc) return rc; @@ -1578,39 +1578,52 @@ static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, if (KVM_BUG_ON(l > TABLE_TYPE_REGION3, sg->kvm)) return -EFAULT; if (l == TABLE_TYPE_PAGE_TABLE) - return _do_shadow_pte(sg, saddr, ptep_h, ptep, entries + LEVEL_MEM, w->p); - return _do_shadow_crste(sg, saddr, host, table, entries + LEVEL_MEM, w->p); + return _do_shadow_pte(mc, sg, saddr, ptep_h, ptep, entries + LEVEL_MEM, w->p); + return _do_shadow_crste(mc, sg, saddr, host, table, entries + LEVEL_MEM, w->p); } -static inline int _gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, - unsigned long seq, struct pgtwalk *walk) +static inline int ___gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, + unsigned long seq, struct pgtwalk *walk) { struct gmap *parent; int rc; - if (kvm_s390_array_needs_retry_unsafe(vcpu->kvm, seq, walk->raw_entries)) + if (kvm_s390_array_needs_retry_safe(vcpu->kvm, seq, walk->raw_entries)) return -EAGAIN; -again: - rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc); - if (rc) - return rc; - scoped_guard(read_lock, &vcpu->kvm->mmu_lock) { - if (kvm_s390_array_needs_retry_safe(vcpu->kvm, seq, walk->raw_entries)) - return -EAGAIN; - parent = READ_ONCE(sg->parent); - if (!parent) + parent = READ_ONCE(sg->parent); + if (!parent) + return -EAGAIN; + scoped_guard(spinlock, &parent->children_lock) { + if (READ_ONCE(sg->parent) != parent) return -EAGAIN; - scoped_guard(spinlock, &parent->children_lock) { - if (READ_ONCE(sg->parent) != parent) - return -EAGAIN; - sg->invalidated = false; - rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk); - } - if (rc == -ENOMEM) - goto again; - if (!rc) - kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false); + sg->invalidated = false; + rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk); } + if (!rc) + kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false); + return rc; +} + +static inline int _gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, + unsigned long seq, struct pgtwalk *walk) +{ + int rc; + + if (kvm_s390_array_needs_retry_unsafe(vcpu->kvm, seq, walk->raw_entries)) + return -EAGAIN; + + do { + rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc); + if (rc) + return rc; + rc = radix_tree_preload(GFP_KERNEL); + if (rc) + return rc; + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + rc = ___gaccess_shadow_fault(vcpu, sg, saddr, seq, walk); + radix_tree_preload_end(); + } while (rc == -ENOMEM); + return rc; } diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 52d55ddea8d4f..1d289f8fa3b20 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -1000,7 +1000,8 @@ int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interr return 0; } -int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level) +int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, + gfn_t r_gfn, int level) { struct vsie_rmap *rmap __free(kvfree) = NULL; struct vsie_rmap *temp; @@ -1010,7 +1011,7 @@ int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level) KVM_BUG_ON(!is_shadow(sg), sg->kvm); lockdep_assert_held(&sg->host_to_rmap_lock); - rmap = kzalloc_obj(*rmap, GFP_ATOMIC); + rmap = kvm_s390_mmu_cache_alloc_rmap(mc); if (!rmap) return -ENOMEM; @@ -1057,7 +1058,7 @@ int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gf if (level <= TABLE_TYPE_REGION1) { bitmask = -1UL << (8 + 11 * level); scoped_guard(spinlock, &sg->host_to_rmap_lock) - rc = gmap_insert_rmap(sg, p_gfn, r_gfn & bitmask, level); + rc = gmap_insert_rmap(mc, sg, p_gfn, r_gfn & bitmask, level); } if (rc) return rc; diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap.h index 5374f21aaf8df..1c040472f56d7 100644 --- a/arch/s390/kvm/gmap.h +++ b/arch/s390/kvm/gmap.h @@ -100,7 +100,8 @@ int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long cou void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count); int gmap_enable_skeys(struct gmap *gmap); int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible); -int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level); +int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, + gfn_t r_gfn, int level); int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, kvm_pfn_t pfn, int level, bool wr); void gmap_set_cmma_all_dirty(struct gmap *gmap); @@ -279,7 +280,16 @@ static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, unio gmap_handle_vsie_unshadow_event(gmap, gfn); else _gmap_handle_vsie_unshadow_event(gmap, gfn); - dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce); + if (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce)) + return false; + /* + * Return false even if the swap was successful, as it only + * indicates that the best effort clearing of the vsie_notif + * bit was successful. The caller will have to try again + * regardless, since the desired value has not been set. + * This pointless check is needed to silence a potential + * __must_check warning. + */ return false; } if (!oldcrste.s.fc1.d && newcrste.s.fc1.d && !newcrste.s.fc1.s) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ffb20a64d328f..6afd45b5f7dc0 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -2252,7 +2252,7 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, return 0; } - values = vmalloc(args->count); + values = vzalloc(args->count); if (!values) return -ENOMEM; diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 5b075c38998e3..720bb58cabe2e 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -300,9 +300,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, gaite->gisc = fib->fmt0.isc; gaite->count++; - gaite->aisbo = fib->fmt0.aisbo; - gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb & - ~PAGE_MASK)); + if (fib->fmt0.sum == 1) { + gaite->aisbo = fib->fmt0.aisbo; + gaite->aisb = virt_to_phys(page_address(aisb_page) + + (fib->fmt0.aisb & ~PAGE_MASK)); + } else { + gaite->aisbo = 0; + gaite->aisb = 0; + } aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); @@ -328,6 +333,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, unpin1: unpin_user_page(aibv_page); out: + kvm_s390_gisc_unregister(kvm, fib->fmt0.isc); return rc; } diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 7498e858c4019..063b4346742d9 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -50,22 +50,6 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, return true; } -static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) -{ - unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); - unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data); - unsigned long data, addr, offset; - - addr = regs->gprs[reg_addr]; - offset = addr & (sizeof(unsigned long) - 1); - addr &= ~(sizeof(unsigned long) - 1); - data = *(unsigned long *)addr; - data <<= BITS_PER_BYTE * offset; - regs->gprs[reg_data] = data; - regs->psw.addr = extable_fixup(ex); - return true; -} - static bool ex_handler_fpc(const struct exception_table_entry *ex, struct pt_regs *regs) { fpu_sfpc(0); @@ -134,8 +118,6 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_ua_load_reg(ex, false, regs); case EX_TYPE_UA_LOAD_REGPAIR: return ex_handler_ua_load_reg(ex, true, regs); - case EX_TYPE_ZEROPAD: - return ex_handler_zeropad(ex, regs); case EX_TYPE_FPC: return ex_handler_fpc(ex, regs); case EX_TYPE_UA_MVCOS_TO: diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c index 1cfe4724fbe2a..4bf7c9012feb6 100644 --- a/arch/s390/mm/gmap_helpers.c +++ b/arch/s390/mm/gmap_helpers.c @@ -51,15 +51,15 @@ pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long vmaddr, spinlock_t pgd = pgdp_get(pgdp); if (pgd_none(pgd) || !pgd_present(pgd)) return NULL; - p4dp = p4d_offset(pgdp, vmaddr); + p4dp = p4d_offset_lockless(pgdp, pgd, vmaddr); p4d = p4dp_get(p4dp); if (p4d_none(p4d) || !p4d_present(p4d)) return NULL; - pudp = pud_offset(p4dp, vmaddr); + pudp = pud_offset_lockless(p4dp, p4d, vmaddr); pud = pudp_get(pudp); if (pud_none(pud) || pud_leaf(pud) || !pud_present(pud)) return NULL; - pmdp = pmd_offset(pudp, vmaddr); + pmdp = pmd_offset_lockless(pudp, pud, vmaddr); pmd = pmdp_get_lockless(pmdp); if (pmd_none(pmd) || pmd_leaf(pmd) || !pmd_present(pmd)) return NULL; @@ -181,7 +181,8 @@ void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr) if (IS_ERR_OR_NULL(ptep)) return; - __atomic64_or(_PAGE_UNUSED, (long *)ptep); + if (pte_present(*ptep)) + __atomic64_or(_PAGE_UNUSED, (long *)ptep); pte_unmap_unlock(ptep, ptl); } EXPORT_SYMBOL_GPL(gmap_helper_try_set_pte_unused); diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 2a222a7e14f48..ef7bfc87758c8 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -64,7 +64,7 @@ static inline unsigned long mmap_base(unsigned long rnd, return PAGE_ALIGN(STACK_TOP - gap - rnd); } -static int get_align_mask(struct file *filp, unsigned long flags) +static unsigned long get_align_mask(struct file *filp, unsigned long flags) { if (filp && is_file_hugepages(filp)) return huge_page_mask_align(filp); diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index a79609eb14be4..02e7a066538ec 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -95,8 +95,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) { return __copy_user((__force void *)to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * Clear the area and return remaining number of bytes diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h index c1bccbedf567e..9f0b54f709088 100644 --- a/arch/sparc/include/asm/page_32.h +++ b/arch/sparc/include/asm/page_32.h @@ -20,10 +20,12 @@ #define clear_user_page(addr, vaddr, page) \ do { clear_page(addr); \ sparc_flush_page_to_ram(page); \ + (void)(vaddr); \ } while (0) #define copy_user_page(to, from, vaddr, page) \ do { copy_page(to, from); \ sparc_flush_page_to_ram(page); \ + (void)(vaddr); \ } while (0) /* The following structure is used to hold the physical diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 43284b6ec46a6..5542d5b32994f 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -190,8 +190,7 @@ static inline unsigned long raw_copy_from_user(void *to, const void __user *from return __copy_user((__force void __user *) to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER static inline unsigned long __clear_user(void __user *addr, unsigned long size) { diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index b825a5dd0210e..e2989cfba626d 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -231,8 +231,7 @@ unsigned long __must_check raw_copy_from_user(void *to, unsigned long __must_check raw_copy_to_user(void __user *to, const void *from, unsigned long size); -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c index f4fb82b019bb9..9b53ac1fe533d 100644 --- a/arch/sparc/kernel/led.c +++ b/arch/sparc/kernel/led.c @@ -78,7 +78,7 @@ static ssize_t led_proc_write(struct file *file, const char __user *buffer, return PTR_ERR(buf); /* work around \n when echo'ing into proc */ - if (buf[count - 1] == '\n') + if (count > 0 && buf[count - 1] == '\n') buf[count - 1] = '\0'; /* before we change anything we want to stop any running timers, diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index 0df9ea4abda83..4417c8b1d37a6 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -27,8 +27,7 @@ static inline int __access_ok(const void __user *ptr, unsigned long size); #define __access_ok __access_ok #define __clear_user __clear_user -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER #include diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index d7ba9219cb47d..df003a42d25a6 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -334,10 +334,6 @@ menuconfig PROCESSOR_SELECT This lets you choose what x86 vendor support code your kernel will include. -config BROADCAST_TLB_FLUSH - def_bool y - depends on CPU_SUP_AMD && 64BIT - config CPU_SUP_INTEL default y bool "Support Intel processors" if PROCESSOR_SELECT @@ -458,3 +454,7 @@ config CPU_SUP_VORTEX_32 makes the kernel a tiny bit smaller. If unsure, say N. + +config BROADCAST_TLB_FLUSH + def_bool y + depends on CPU_SUP_AMD && 64BIT diff --git a/arch/x86/boot/compressed/acpi.c b/arch/x86/boot/compressed/acpi.c index f196b1d1ddf86..aed27604c11f0 100644 --- a/arch/x86/boot/compressed/acpi.c +++ b/arch/x86/boot/compressed/acpi.c @@ -184,10 +184,15 @@ static unsigned long get_cmdline_acpi_rsdp(void) char val[MAX_ADDR_LEN] = { }; int ret; - ret = cmdline_find_option("acpi_rsdp", val, MAX_ADDR_LEN); + ret = cmdline_find_option("acpi_rsdp", val, sizeof(val)); if (ret < 0) return 0; + if (ret >= sizeof(val)) { + warn("acpi_rsdp= value too long; ignoring"); + return 0; + } + if (boot_kstrtoul(val, 16, &addr)) return 0; #endif diff --git a/arch/x86/boot/early_serial_console.c b/arch/x86/boot/early_serial_console.c index 023bf1c3de8b7..5b83beab89e15 100644 --- a/arch/x86/boot/early_serial_console.c +++ b/arch/x86/boot/early_serial_console.c @@ -117,7 +117,7 @@ static unsigned int probe_baud(int port) static void parse_console_uart8250(void) { char optstr[64], *options; - int baud = DEFAULT_BAUD; + int baud; int port = 0; /* @@ -136,10 +136,13 @@ static void parse_console_uart8250(void) else return; - if (options && (options[0] == ',')) - baud = simple_strtoull(options + 1, &options, 0); - else + if (options && (options[0] == ',')) { + baud = simple_strtoull(options + 1, NULL, 0); + if (!baud) + baud = DEFAULT_BAUD; + } else { baud = probe_baud(port); + } if (port) early_serial_init(port, baud); diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index 06f35a6b58a5b..dc564688f3d73 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -259,13 +259,13 @@ void amd_brs_disable_all(void) amd_brs_disable(); } -static bool amd_brs_match_plm(struct perf_event *event, u64 to) +static bool amd_brs_match_plm(struct perf_event *event, u64 from, u64 to) { int type = event->attr.branch_sample_type; int plm_k = PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_HV; int plm_u = PERF_SAMPLE_BRANCH_USER; - if (!(type & plm_k) && kernel_ip(to)) + if (!(type & plm_k) && (kernel_ip(to) || kernel_ip(from))) return 0; if (!(type & plm_u) && !kernel_ip(to)) @@ -338,11 +338,11 @@ void amd_brs_drain(void) */ to = (u64)(((s64)to << shift) >> shift); - if (!amd_brs_match_plm(event, to)) - continue; - rdmsrq(brs_from(brs_idx), from); + if (!amd_brs_match_plm(event, from, to)) + continue; + perf_clear_branch_entry_bitfields(br+nr); br[nr].from = from; diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 0c92ed5f464b1..4b34975dcfc58 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -752,13 +752,11 @@ static void amd_pmu_enable_event(struct perf_event *event) x86_pmu_enable_event(event); } -static void amd_pmu_enable_all(int added) +static void __amd_pmu_enable_all(void) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int idx; - amd_brs_enable_all(); - for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { /* only activate events which are marked as active */ if (!test_bit(idx, cpuc->active_mask)) @@ -773,6 +771,12 @@ static void amd_pmu_enable_all(int added) } } +static void amd_pmu_enable_all(int added) +{ + amd_brs_enable_all(); + __amd_pmu_enable_all(); +} + static void amd_pmu_v2_enable_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -1412,12 +1416,12 @@ static int __init amd_core_pmu_init(void) u64 even_ctr_mask = 0ULL; int i; - if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) - return 0; - /* Avoid calculating the value each time in the NMI handler */ perf_nmi_window = msecs_to_jiffies(100); + if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) + return 0; + /* * If core performance counter extensions exists, we must use * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also @@ -1559,7 +1563,7 @@ static inline void amd_pmu_reload_virt(void) * set global enable bits once again */ amd_pmu_v2_disable_all(); - amd_pmu_enable_all(0); + __amd_pmu_enable_all(); amd_pmu_v2_enable_all(0); return; } diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index d24da377df776..b9722e60d8bc7 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -125,7 +125,8 @@ static void amd_pmu_lbr_filter(void) } /* If type does not correspond, then discard */ - if (type == X86_BR_NONE || (br_sel & type) != type) { + if (type == X86_BR_NONE || (br_sel & type) != type || + (!(br_sel & X86_BR_KERNEL) && kernel_ip(cpuc->lbr_entries[i].from))) { cpuc->lbr_entries[i].from = 0; /* mark invalid */ compress = true; } diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index dd956cfcadefa..a0364ca2f9173 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -700,7 +700,7 @@ void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) info.split.aux_data = 0; info.split.num_pmcs = NUM_COUNTERS_NB; info.split.gid = 0; - info.split.cid = topology_logical_package_id(cpu); + info.split.cid = topology_amd_node_id(cpu); if (pmu_version >= 2) { ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); @@ -999,8 +999,8 @@ void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx); info.split.aux_data = ecx; /* stash active mask */ info.split.num_pmcs = ebx.split.num_umc_pmc; - info.split.gid = topology_logical_package_id(cpu); - info.split.cid = topology_logical_package_id(cpu); + info.split.gid = topology_amd_node_id(cpu); + info.split.cid = topology_amd_node_id(cpu); *per_cpu_ptr(uncore->info, cpu) = info; } diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 583cbd06b9b88..60e1200c4691d 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -481,8 +481,8 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) struct intel_uncore_discovery_unit *unit; unit = intel_uncore_find_discovery_unit(box->pmu->type->boxes, - -1, box->pmu->pmu_idx); - if (WARN_ON_ONCE(!unit)) + box->dieid, box->pmu->pmu_idx); + if (!unit) return 0; return unit->addr; diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index 3dbc6bacbd9de..edddd4f9ab5fc 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -563,12 +563,6 @@ void tgl_uncore_cpu_init(void) skl_uncore_msr_ops.init_box = rkl_uncore_msr_init_box; } -static void adl_uncore_msr_init_box(struct intel_uncore_box *box) -{ - if (box->pmu->pmu_idx == 0) - wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); -} - static void adl_uncore_msr_enable_box(struct intel_uncore_box *box) { wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); @@ -587,7 +581,6 @@ static void adl_uncore_msr_exit_box(struct intel_uncore_box *box) } static struct intel_uncore_ops adl_uncore_msr_ops = { - .init_box = adl_uncore_msr_init_box, .enable_box = adl_uncore_msr_enable_box, .disable_box = adl_uncore_msr_disable_box, .exit_box = adl_uncore_msr_exit_box, diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 215d33e260ede..c9ce206fcbb6e 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -4261,7 +4261,7 @@ static int upi_fill_topology(struct pci_dev *dev, struct intel_uncore_topology * static int skx_upi_topology_cb(struct intel_uncore_type *type, int segment, int die, u64 cpu_bus_msr) { - int idx, ret; + int idx, ret = 0; struct intel_uncore_topology *upi; unsigned int devfn; struct pci_dev *dev = NULL; @@ -4274,12 +4274,12 @@ static int skx_upi_topology_cb(struct intel_uncore_type *type, int segment, dev = pci_get_domain_bus_and_slot(segment, bus, devfn); if (dev) { ret = upi_fill_topology(dev, upi, idx); + pci_dev_put(dev); if (ret) break; } } - pci_dev_put(dev); return ret; } @@ -5499,6 +5499,7 @@ static int discover_upi_topology(struct intel_uncore_type *type, int ubox_did, i devfn); if (dev) { ret = upi_fill_topology(dev, upi, idx); + pci_dev_put(dev); if (ret) goto err; } @@ -5506,7 +5507,6 @@ static int discover_upi_topology(struct intel_uncore_type *type, int ubox_did, i } err: pci_dev_put(ubox); - pci_dev_put(dev); return ret; } diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index 80c1696d8d597..bf3c802654d18 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -153,6 +153,7 @@ struct arch_va_list { struct sysv_va_list args; }; extern void *__warn_args(struct arch_va_list *args, struct pt_regs *regs); +static __always_inline __printf(1, 2) void __WARN_validate_printf(const char *fmt, ...) { } #endif /* __ASSEMBLER__ */ #define __WARN_bug_entry(flags, format) ({ \ @@ -172,6 +173,7 @@ extern void *__warn_args(struct arch_va_list *args, struct pt_regs *regs); #define __WARN_print_arg(flags, format, arg...) \ do { \ int __flags = (flags) | BUGFLAG_WARNING | BUGFLAG_ARGS ; \ + __WARN_validate_printf(format, ## arg); \ static_call_mod(WARN_trap)(__WARN_bug_entry(__flags, format), ## arg); \ asm (""); /* inhibit tail-call optimization */ \ } while (0) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 4f4b5e8a15743..b68892e6d58c4 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -388,6 +388,10 @@ extern void srso_alias_return_thunk(void); extern void entry_untrain_ret(void); extern void write_ibpb(void); +#ifdef CONFIG_BPF_JIT +extern void bpf_arch_ibpb(void); +#endif + #ifdef CONFIG_X86_64 extern void clear_bhb_loop(void); #endif diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 83f51cab0b1e3..d9af230c05125 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -1651,8 +1652,21 @@ static inline const char *spectre_v2_module_string(void) { return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; } + +/* + * The "retpoline sequence" is the "call;mov;ret" sequence that + * replaces normal indirect branch instructions. Differentiate + * *the* retpoline sequence from the LFENCE-prefixed indirect + * branches that simply use the retpoline infrastructure. + */ +static inline bool retpoline_seq_enabled(void) +{ + return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE); +} + #else static inline const char *spectre_v2_module_string(void) { return ""; } +static inline bool retpoline_seq_enabled(void) { return false; } #endif #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" @@ -2095,8 +2109,7 @@ static void __init bhi_apply_mitigation(void) return; /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */ - if (boot_cpu_has(X86_FEATURE_RETPOLINE) && - !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { + if (retpoline_seq_enabled()) { spec_ctrl_disable_kernel_rrsba(); if (rrsba_disabled) return; @@ -2238,6 +2251,27 @@ static void __init spectre_v2_update_mitigation(void) pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]); } +#ifdef CONFIG_BPF_JIT +static void __bpf_arch_ibpb(void *unused) +{ + write_ibpb(); +} + +void bpf_arch_ibpb(void) +{ + on_each_cpu(__bpf_arch_ibpb, NULL, 1); +} + +static bool __init cpu_wants_ibpb_bpf(void) +{ + /* A genuine retpoline already neutralizes ring0 indirect predictions */ + if (retpoline_seq_enabled()) + return false; + + return boot_cpu_has(X86_FEATURE_IBPB); +} +#endif + static void __init spectre_v2_apply_mitigation(void) { if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) @@ -2314,6 +2348,14 @@ static void __init spectre_v2_apply_mitigation(void) setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); pr_info("Enabling Restricted Speculation for firmware calls\n"); } + +#ifdef CONFIG_BPF_JIT + if (cpu_wants_ibpb_bpf()) { + static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb); + static_branch_enable(&bpf_pred_flush_enabled); + pr_info("Enabling IBPB for BPF\n"); + } +#endif } static void update_stibp_msr(void * __unused) @@ -3490,9 +3532,7 @@ static const char *spectre_bhi_state(void) return "; BHI: BHI_DIS_S"; else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP)) return "; BHI: SW loop, KVM: SW loop"; - else if (boot_cpu_has(X86_FEATURE_RETPOLINE) && - !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) && - rrsba_disabled) + else if (retpoline_seq_enabled() && rrsba_disabled) return "; BHI: Retpoline"; else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT)) return "; BHI: Vulnerable, KVM: SW loop"; diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 5c7a3a71191a1..dca2d5845e427 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h @@ -75,7 +75,6 @@ static inline struct amd_northbridge *amd_init_l3_cache(int index) } #endif -unsigned int aperfmperf_get_khz(int cpu); void cpu_select_mitigations(void); extern void x86_spec_ctrl_setup_ap(void); diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 59215fef3924c..50ee1f640c68d 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -258,6 +258,11 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) return -EINVAL; + if (cpumask_empty(&hdr->cpu_mask)) { + pr_warn_once("Domain %d has no CPUs\n", hdr->id); + return -EINVAL; + } + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); hw_dom = resctrl_to_arch_mon_dom(d); cpu = cpumask_any(&hdr->cpu_mask); diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index ebb1baf1eb1d0..3af979fb41d38 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -761,9 +761,9 @@ void arch_uprobe_clear_state(struct mm_struct *mm) destroy_uprobe_trampoline(tramp); } -static bool __in_uprobe_trampoline(unsigned long ip) +static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip) { - struct vm_area_struct *vma = vma_lookup(current->mm, ip); + struct vm_area_struct *vma = vma_lookup(mm, ip); return vma && vma_is_special_mapping(vma, &tramp_mapping); } @@ -776,14 +776,14 @@ static bool in_uprobe_trampoline(unsigned long ip) rcu_read_lock(); if (mmap_lock_speculate_try_begin(mm, &seq)) { - found = __in_uprobe_trampoline(ip); + found = __in_uprobe_trampoline(mm, ip); retry = mmap_lock_speculate_retry(mm, seq); } rcu_read_unlock(); if (retry) { mmap_read_lock(mm); - found = __in_uprobe_trampoline(ip); + found = __in_uprobe_trampoline(mm, ip); mmap_read_unlock(mm); } return found; @@ -1044,7 +1044,7 @@ static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, return 0; } -static bool __is_optimized(uprobe_opcode_t *insn, unsigned long vaddr) +static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned long vaddr) { struct __packed __arch_relative_insn { u8 op; @@ -1053,7 +1053,7 @@ static bool __is_optimized(uprobe_opcode_t *insn, unsigned long vaddr) if (!is_call_insn(insn)) return false; - return __in_uprobe_trampoline(vaddr + 5 + call->raddr); + return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr); } static int is_optimized(struct mm_struct *mm, unsigned long vaddr) @@ -1064,7 +1064,7 @@ static int is_optimized(struct mm_struct *mm, unsigned long vaddr) err = copy_from_vaddr(mm, vaddr, &insn, 5); if (err) return err; - return __is_optimized((uprobe_opcode_t *)&insn, vaddr); + return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr); } static bool should_optimize(struct arch_uprobe *auprobe) @@ -1246,9 +1246,15 @@ static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs long correction = utask->vaddr - utask->xol_vaddr; regs->ip += correction; } else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) { + unsigned long retaddr = utask->vaddr + auprobe->defparam.ilen; + int err; + regs->sp += sizeof_long(regs); /* Pop incorrect return address */ - if (emulate_push_stack(regs, utask->vaddr + auprobe->defparam.ilen)) + if (emulate_push_stack(regs, retaddr)) return -ERESTART; + err = shstk_update_last_frame(retaddr); + if (err) + return err; } /* popf; tell the caller to not touch TF */ if (auprobe->defparam.fixups & UPROBE_FIX_SETF) @@ -1338,6 +1344,10 @@ static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs) */ if (emulate_push_stack(regs, new_ip)) return false; + if (shstk_push(new_ip) == -EFAULT) { + regs->sp += sizeof_long(regs); + return false; + } } else if (!check_jmp_cond(auprobe, regs)) { offs = 0; } diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 4438ecac9a89b..ad59c22979bde 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1839,6 +1839,11 @@ static bool hv_is_vp_in_sparse_set(u32 vp_id, u64 valid_bank_mask, u64 sparse_ba int valid_bit_nr = vp_id / HV_VCPUS_PER_SPARSE_BANK; unsigned long sbank; + BUILD_BUG_ON(BITS_PER_TYPE(valid_bank_mask) != HV_MAX_SPARSE_VCPU_BANKS); + + if (valid_bit_nr >= HV_MAX_SPARSE_VCPU_BANKS) + return false; + if (!test_bit(valid_bit_nr, (unsigned long *)&valid_bank_mask)) return false; diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 9519fec09ee6f..c5c2f778c6692 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -488,8 +488,10 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) { ret = kvm_pi_update_irte(irqfd, &irqfd->irq_entry); - if (ret) + if (ret) { kvm->arch.nr_possible_bypass_irqs--; + irqfd->producer = NULL; + } } spin_unlock_irq(&kvm->irqfds.lock); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 4078e624ca667..d806abaeda3de 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -980,6 +980,8 @@ static void apic_update_ppr(struct kvm_lapic *apic) if (__apic_update_ppr(apic, &ppr) && apic_has_interrupt_for_ppr(apic, ppr) != -1) kvm_make_request(KVM_REQ_EVENT, apic->vcpu); + else + kvm_lapic_update_cr8_intercept(apic->vcpu); } void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) @@ -1730,7 +1732,7 @@ static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) #define APIC_REGS_MASK(first, count) \ (APIC_REG_MASK(first) * ((1ull << (count)) - 1)) -u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic) +static u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic) { /* Leave bits '0' for reserved and write-only registers. */ u64 valid_reg_mask = @@ -1766,7 +1768,24 @@ u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic) return valid_reg_mask; } -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lapic_readable_reg_mask); + +u64 kvm_x2apic_disable_read_intercept_reg_mask(struct kvm_vcpu *vcpu) +{ + if (WARN_ON_ONCE(!lapic_in_kernel(vcpu))) + return 0; + + /* + * TMMCT, a.k.a. the current APIC timer count, reads aren't accelerated + * by hardware (Intel or AMD) as the timer is emulated in software (by + * KVM), i.e. reads from the virtual APIC page would return garbage. + * Intercept RDMSR, as handling the fault-like APIC-access VM-Exit is + * more expensive than handling a RDMSR VM-Exit (the APIC-access exit + * requires slow emulation of the code stream). + */ + return kvm_lapic_readable_reg_mask(vcpu->arch.apic) & + ~APIC_REG_MASK(APIC_TMCCT); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_x2apic_disable_read_intercept_reg_mask); static int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, void *data) @@ -2744,6 +2763,32 @@ u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) return (tpr & 0xf0) >> 4; } +void kvm_lapic_update_cr8_intercept(struct kvm_vcpu *vcpu) +{ + int max_irr, tpr; + + if (!kvm_x86_ops.update_cr8_intercept) + return; + + if (!lapic_in_kernel(vcpu)) + return; + + if (vcpu->arch.apic->apicv_active) + return; + + if (!vcpu->arch.apic->vapic_addr) + max_irr = kvm_lapic_find_highest_irr(vcpu); + else + max_irr = -1; + + if (max_irr != -1) + max_irr >>= 4; + + tpr = kvm_lapic_get_cr8(vcpu); + + kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr); +} + static void __kvm_apic_set_base(struct kvm_vcpu *vcpu, u64 value) { u64 old_value = vcpu->arch.apic_base; @@ -3326,6 +3371,12 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, struct kvm_lapic *apic) { int vector; + + if (unlikely(!pv_eoi_enabled(vcpu))) { + __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); + return; + } + /* * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host * and KVM_PV_EOI_ENABLED in guest memory as follows: @@ -3337,8 +3388,6 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: * -> host enabled PV EOI, guest executed EOI. */ - BUG_ON(!pv_eoi_enabled(vcpu)); - if (pv_eoi_test_and_clr_pending(vcpu)) return; vector = apic_set_eoi(apic); diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 274885af4ebc4..71970213dc1fc 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -100,6 +100,7 @@ int kvm_apic_accept_events(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); +void kvm_lapic_update_cr8_intercept(struct kvm_vcpu *vcpu); void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu); void kvm_apic_set_version(struct kvm_vcpu *vcpu); void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu); @@ -156,7 +157,7 @@ int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data); int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len); void kvm_lapic_exit(void); -u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic); +u64 kvm_x2apic_disable_read_intercept_reg_mask(struct kvm_vcpu *vcpu); static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) { diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f0144ae8d891d..0c2e35e3ffbdb 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2453,13 +2453,15 @@ static struct kvm_mmu_page *kvm_mmu_get_child_sp(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, bool direct, unsigned int access) { - union kvm_mmu_page_role role; + union kvm_mmu_page_role role = kvm_mmu_child_role(sptep, direct, access); - if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep) && - spte_to_child_sp(*sptep) && spte_to_child_sp(*sptep)->gfn == gfn) + if (is_shadow_present_pte(*sptep) && + !is_large_pte(*sptep) && + spte_to_child_sp(*sptep) && + spte_to_child_sp(*sptep)->gfn == gfn && + spte_to_child_sp(*sptep)->role.word == role.word) return ERR_PTR(-EEXIST); - role = kvm_mmu_child_role(sptep, direct, access); return kvm_mmu_get_shadow_page(vcpu, gfn, role); } @@ -7286,13 +7288,19 @@ static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, sp = sptep_to_sp(sptep); /* - * We cannot do huge page mapping for indirect shadow pages, - * which are found on the last rmap (level = 1) when not using - * tdp; such shadow pages are synced with the page table in - * the guest, and the guest page table is using 4K page size - * mapping if the indirect sp has level = 1. + * Direct shadow page can be replaced by a hugepage if the host + * mapping level allows it and the memslot maps all of the host + * hugepage. Note! If the memslot maps only part of the + * hugepage, sp->gfn may be below slot->base_gfn, and querying + * the max mapping level would cause an out-of-bounds lpage_info + * access. So the gfn bounds check *must* be done first. + * + * Indirect shadow pages are created when the guest page tables + * are using 4K pages. Since the host mapping is always + * constrained by the page size in the guest, indirect shadow + * pages are never collapsible. */ - if (sp->role.direct && + if (sp->role.direct && is_gfn_in_memslot(slot, sp->gfn) && sp->role.level < kvm_mmu_max_mapping_level(kvm, NULL, slot, sp->gfn)) { kvm_zap_one_rmap_spte(kvm, rmap_head, sptep); diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index cdd5a6dc646f1..0726f88e679aa 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -122,38 +122,8 @@ static u32 x2avic_max_physical_id; static void avic_set_x2apic_msr_interception(struct vcpu_svm *svm, bool intercept) { - static const u32 x2avic_passthrough_msrs[] = { - X2APIC_MSR(APIC_ID), - X2APIC_MSR(APIC_LVR), - X2APIC_MSR(APIC_TASKPRI), - X2APIC_MSR(APIC_ARBPRI), - X2APIC_MSR(APIC_PROCPRI), - X2APIC_MSR(APIC_EOI), - X2APIC_MSR(APIC_RRR), - X2APIC_MSR(APIC_LDR), - X2APIC_MSR(APIC_DFR), - X2APIC_MSR(APIC_SPIV), - X2APIC_MSR(APIC_ISR), - X2APIC_MSR(APIC_TMR), - X2APIC_MSR(APIC_IRR), - X2APIC_MSR(APIC_ESR), - X2APIC_MSR(APIC_ICR), - X2APIC_MSR(APIC_ICR2), - - /* - * Note! Always intercept LVTT, as TSC-deadline timer mode - * isn't virtualized by hardware, and the CPU will generate a - * #GP instead of a #VMEXIT. - */ - X2APIC_MSR(APIC_LVTTHMR), - X2APIC_MSR(APIC_LVTPC), - X2APIC_MSR(APIC_LVT0), - X2APIC_MSR(APIC_LVT1), - X2APIC_MSR(APIC_LVTERR), - X2APIC_MSR(APIC_TMICT), - X2APIC_MSR(APIC_TMCCT), - X2APIC_MSR(APIC_TDCR), - }; + struct kvm_vcpu *vcpu = &svm->vcpu; + u64 rd_regs; int i; if (intercept == svm->x2avic_msrs_intercepted) @@ -162,9 +132,16 @@ static void avic_set_x2apic_msr_interception(struct vcpu_svm *svm, if (!x2avic_enabled) return; - for (i = 0; i < ARRAY_SIZE(x2avic_passthrough_msrs); i++) - svm_set_intercept_for_msr(&svm->vcpu, x2avic_passthrough_msrs[i], - MSR_TYPE_RW, intercept); + rd_regs = kvm_x2apic_disable_read_intercept_reg_mask(vcpu); + + for_each_set_bit(i, (unsigned long *)&rd_regs, BITS_PER_TYPE(rd_regs)) + svm_set_intercept_for_msr(vcpu, APIC_BASE_MSR + i, + MSR_TYPE_R, intercept); + + svm_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_W, intercept); + svm_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W, intercept); + svm_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W, intercept); + svm_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_ICR), MSR_TYPE_W, intercept); svm->x2avic_msrs_intercepted = intercept; } diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 6c6a6d663e296..996f398a577ff 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -1396,6 +1396,7 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) s_off = vaddr & ~PAGE_MASK; d_off = dst_vaddr & ~PAGE_MASK; len = min_t(size_t, (PAGE_SIZE - s_off), size); + len = min_t(size_t, len, PAGE_SIZE - d_off); if (dec) ret = __sev_dbg_decrypt_user(kvm, @@ -2141,8 +2142,9 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (ret) return ret; + /* Do not allow SNP VM migration until additional state transfer is implemented */ if (kvm->arch.vm_type != source_kvm->arch.vm_type || - sev_guest(kvm) || !sev_guest(source_kvm)) { + sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) { ret = -EINVAL; goto out_unlock; } @@ -2469,6 +2471,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) sev_populate_args.type = params.type; count = kvm_gmem_populate(kvm, params.gfn_start, src, npages, + params.type == KVM_SEV_SNP_PAGE_TYPE_CPUID, sev_gmem_post_populate, &sev_populate_args); if (count < 0) { argp->error = sev_populate_args.fw_error; @@ -2863,8 +2866,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) * disallow out-of-band SEV/SEV-ES init if the target is already an * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being * created after SEV/SEV-ES initialization, e.g. to init intercepts. + * Also do not allow SNP VM mirroring until additional state transfer is implemented. */ - if (sev_guest(kvm) || !sev_guest(source_kvm) || + if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) || is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { ret = -EINVAL; goto e_unlock; @@ -4520,9 +4524,12 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu) sev_es_sync_from_ghcb(svm); /* SEV-SNP guest requires that the GHCB GPA must be registered */ - if (is_sev_snp_guest(vcpu) && !ghcb_gpa_is_registered(svm, ghcb_gpa)) { - vcpu_unimpl(&svm->vcpu, "vmgexit: GHCB GPA [%#llx] is not registered.\n", ghcb_gpa); - return -EINVAL; + if (is_sev_snp_guest(vcpu) && + !ghcb_gpa_is_registered(svm, control->ghcb_gpa)) { + vcpu_unimpl(vcpu, "vmgexit: GHCB GPA [%#llx] is not registered.\n", + control->ghcb_gpa); + svm_vmgexit_bad_input(svm, GHCB_ERR_NOT_REGISTERED); + return 1; } ret = sev_es_validate_vmgexit(svm); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e02a38da5296e..2544c551e702f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3663,6 +3663,9 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) vcpu->arch.cr3 = svm->vmcb->save.cr3; } + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) + return 0; + if (is_guest_mode(vcpu)) { int vmexit; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 3fe88f29be7a9..52ab52b0e1ccf 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -553,6 +553,9 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { + gpa_t vtpr_gpa = vmcs12->virtual_apic_page_addr + APIC_TASKPRI; + u32 vtpr; + if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) return 0; @@ -562,6 +565,32 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4)) return -EINVAL; + /* + * Do the illegal vTPR vs. TPR Threshold consistency check if and only + * if KVM is configured to WARN on missed consistency checks, otherwise + * it's a waste of time. KVM needs to rely on hardware to fully detect + * an illegal combination due to the vTPR being writable by L1 at all + * times (it's an in-memory value, not a VMCS field). I.e. even if the + * check passes now, it might fail at the actual VM-Enter. + * + * If reading guest memory fails, skip the check as KVM's de facto ABI + * for VMX instruction accesses to non-existent memory is to provide + * PCI Bus Error semantics (reads return 0xFFs), in which case the vTPR + * is guaranteed to greater than or equal to the threshold. + * + * Note! Deliberately use the VM-scoped API when reading guest memory, + * to ensure the read doesn't hit SMRAM when restoring L2 state on RSM, + * and only perform the check when in KVM_RUN, to avoid a false failure + * if userspace hasn't yet configured memslots during state restore. + */ + if (warn_on_missed_cc && vcpu->wants_to_run && + nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && + !nested_cpu_has_vid(vmcs12) && + !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && + !kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) && + CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > ((vtpr >> 4) & GENMASK(3, 0)))) + return -EINVAL; + return 0; } @@ -3085,38 +3114,6 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, return 0; } -static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu, - struct vmcs12 *vmcs12) -{ - void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva; - u32 vtpr = vapic ? (*(u32 *)(vapic + APIC_TASKPRI)) >> 4 : 0; - - /* - * Don't bother with the consistency checks if KVM isn't configured to - * WARN on missed consistency checks, as KVM needs to rely on hardware - * to fully detect an illegal vTPR vs. TRP Threshold combination due to - * the vTPR being writable by L1 at all times (it's an in-memory value, - * not a VMCS field). I.e. even if the check passes now, it might fail - * at the actual VM-Enter. - * - * Keying off the module param also allows treating an invalid vAPIC - * mapping as a consistency check failure without increasing the risk - * of breaking a "real" VM. - */ - if (!warn_on_missed_cc) - return 0; - - if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) && - nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && - !nested_cpu_has_vid(vmcs12) && - !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && - (CC(!vapic) || - CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0))))) - return -EINVAL; - - return 0; -} - static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { @@ -3666,11 +3663,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, return NVMX_VMENTRY_KVM_INTERNAL_ERROR; } - if (nested_vmx_check_controls_late(vcpu, vmcs12)) { - vmx_switch_vmcs(vcpu, &vmx->vmcs01); - return NVMX_VMENTRY_VMFAIL; - } - if (nested_vmx_check_guest_state(vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic = EXIT_REASON_INVALID_STATE; @@ -3755,6 +3747,8 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, if (!from_vmentry) return NVMX_VMENTRY_VMEXIT; + nested_put_vmcs12_pages(vcpu); + load_vmcs12_host_state(vcpu, vmcs12); vmcs12->vm_exit_reason = exit_reason.full; if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 27eb76e6b6a03..22138c487216e 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -308,13 +308,15 @@ static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, */ local_irq_disable(); if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) { + int err = 0; + if (read) rdmsrq(index, msr_info->data); else - wrmsrq(index, msr_info->data); + err = wrmsrq_safe(index, msr_info->data); __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); local_irq_enable(); - return true; + return !err; } clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); local_irq_enable(); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 04ce321ebdf39..556bf9fe1f5d0 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2387,20 +2387,20 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params, ret = -ENOMEM; - tdr_page = alloc_page(GFP_KERNEL); + tdr_page = alloc_page(GFP_KERNEL_ACCOUNT); if (!tdr_page) goto free_hkid; kvm_tdx->td.tdcs_nr_pages = tdx_sysinfo->td_ctrl.tdcs_base_size / PAGE_SIZE; /* TDVPS = TDVPR(4K page) + TDCX(multiple 4K pages), -1 for TDVPR. */ kvm_tdx->td.tdcx_nr_pages = tdx_sysinfo->td_ctrl.tdvps_base_size / PAGE_SIZE - 1; - tdcs_pages = kzalloc_objs(*kvm_tdx->td.tdcs_pages, - kvm_tdx->td.tdcs_nr_pages); + tdcs_pages = kzalloc_objs(*kvm_tdx->td.tdcs_pages, kvm_tdx->td.tdcs_nr_pages, + GFP_KERNEL_ACCOUNT); if (!tdcs_pages) goto free_tdr; for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) { - tdcs_pages[i] = alloc_page(GFP_KERNEL); + tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT); if (!tdcs_pages[i]) goto free_tdcs; } @@ -2725,7 +2725,11 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) goto out; } - if (init_vm->cpuid.padding) { + /* + * Reject the request if userspace changes cpuid.nent between the + * initial read and the subsequent copy. + */ + if (init_vm->cpuid.padding || init_vm->cpuid.nent != nr_user_entries) { ret = -EINVAL; goto out; } @@ -2875,7 +2879,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) int ret, i; u64 err; - page = alloc_page(GFP_KERNEL); + page = alloc_page(GFP_KERNEL_ACCOUNT); if (!page) return -ENOMEM; tdx->vp.tdvpr_page = page; @@ -2888,14 +2892,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) tdx->vp.tdvpr_pa = page_to_phys(tdx->vp.tdvpr_page); tdx->vp.tdcx_pages = kcalloc(kvm_tdx->td.tdcx_nr_pages, sizeof(*tdx->vp.tdcx_pages), - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!tdx->vp.tdcx_pages) { ret = -ENOMEM; goto free_tdvpr; } for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) { - page = alloc_page(GFP_KERNEL); + page = alloc_page(GFP_KERNEL_ACCOUNT); if (!page) { ret = -ENOMEM; goto free_tdcx; @@ -3185,7 +3189,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c }; gmem_ret = kvm_gmem_populate(kvm, gpa_to_gfn(region.gpa), u64_to_user_ptr(region.source_addr), - 1, tdx_gmem_post_populate, &arg); + 1, false, tdx_gmem_post_populate, &arg); if (gmem_ret < 0) { ret = gmem_ret; break; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b9103de01428e..f389ac9fcb33b 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1908,6 +1908,24 @@ void vmx_inject_exception(struct kvm_vcpu *vcpu) u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; struct vcpu_vmx *vmx = to_vmx(vcpu); + /* + * When injecting a #DB, single-stepping is enabled in RFLAGS, and STI + * or MOV-SS blocking is active, set vmcs.PENDING_DBG_EXCEPTIONS.BS to + * prevent a false positive from VM-Entry consistency check. VM-Entry + * asserts that a single-step #DB _must_ be pending in this scenario, + * as the previous instruction cannot have toggled RFLAGS.TF 0=>1 + * (because STI and POP/MOV don't modify RFLAGS), therefore the one + * instruction delay when activating single-step breakpoints must have + * already expired. However, the CPU isn't smart enough to peek at + * vmcs.VM_ENTRY_INTR_INFO_FIELD and so doesn't realize that yes, there + * is indeed a #DB pending/imminent. + */ + if (ex->vector == DB_VECTOR && + (vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && + vmx_get_interrupt_shadow(vcpu)) + vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, + vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); + kvm_deliver_exception_payload(vcpu, ex); if (ex->has_error_code) { @@ -4141,7 +4159,7 @@ static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) * mode, only the current timer count needs on-demand emulation by KVM. */ if (mode & MSR_BITMAP_MODE_X2APIC_APICV) - msr_bitmap[read_idx] = ~kvm_lapic_readable_reg_mask(vcpu->arch.apic); + msr_bitmap[read_idx] = ~kvm_x2apic_disable_read_intercept_reg_mask(vcpu); else msr_bitmap[read_idx] = ~0ull; msr_bitmap[write_idx] = ~0ull; @@ -4154,7 +4172,6 @@ static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) !(mode & MSR_BITMAP_MODE_X2APIC)); if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { - vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW); vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); if (enable_ipiv) @@ -5477,26 +5494,9 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) * avoid single-step #DB and MTF updates, as ICEBP is * higher priority. Note, skipping ICEBP still clears * STI and MOVSS blocking. - * - * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS - * if single-step is enabled in RFLAGS and STI or MOVSS - * blocking is active, as the CPU doesn't set the bit - * on VM-Exit due to #DB interception. VM-Entry has a - * consistency check that a single-step #DB is pending - * in this scenario as the previous instruction cannot - * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV - * don't modify RFLAGS), therefore the one instruction - * delay when activating single-step breakpoints must - * have already expired. Note, the CPU sets/clears BS - * as appropriate for all other VM-Exits types. */ if (is_icebp(intr_info)) WARN_ON(!skip_emulated_instruction(vcpu)); - else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && - (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & - (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS))) - vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, - vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); return 1; @@ -6697,6 +6697,9 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) if (enable_pml && !is_guest_mode(vcpu)) vmx_flush_pml_buffer(vcpu); + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) + return 0; + /* * KVM should never reach this point with a pending nested VM-Enter. * More specifically, short-circuiting VM-Entry to emulate L2 due to @@ -6855,11 +6858,10 @@ int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { - struct vmcs12 *vmcs12 = get_vmcs12(vcpu); int tpr_threshold; if (is_guest_mode(vcpu) && - nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) + nested_cpu_has(get_vmcs12(vcpu), CPU_BASED_TPR_SHADOW)) return; guard(vmx_vmcs01)(vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0550359ed798f..88f0e3fd8919c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -128,7 +128,6 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) -static void update_cr8_intercept(struct kvm_vcpu *vcpu); static void process_nmi(struct kvm_vcpu *vcpu); static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); @@ -5340,7 +5339,6 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, r = kvm_apic_set_state(vcpu, s); if (r) return r; - update_cr8_intercept(vcpu); return 0; } @@ -10595,33 +10593,6 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) kvm_run->flags |= KVM_RUN_X86_GUEST_MODE; } -static void update_cr8_intercept(struct kvm_vcpu *vcpu) -{ - int max_irr, tpr; - - if (!kvm_x86_ops.update_cr8_intercept) - return; - - if (!lapic_in_kernel(vcpu)) - return; - - if (vcpu->arch.apic->apicv_active) - return; - - if (!vcpu->arch.apic->vapic_addr) - max_irr = kvm_lapic_find_highest_irr(vcpu); - else - max_irr = -1; - - if (max_irr != -1) - max_irr >>= 4; - - tpr = kvm_lapic_get_cr8(vcpu); - - kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr); -} - - int kvm_check_nested_events(struct kvm_vcpu *vcpu) { if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { @@ -11362,7 +11333,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(enable_irq_window)(vcpu); if (kvm_lapic_enabled(vcpu)) { - update_cr8_intercept(vcpu); + kvm_lapic_update_cr8_intercept(vcpu); kvm_lapic_sync_to_vapic(vcpu); } } @@ -11607,9 +11578,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (vcpu->arch.apic_attention) kvm_lapic_sync_from_vapic(vcpu); - if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) - return 0; - r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath); return r; @@ -12481,8 +12449,6 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); - kvm_set_cr8(vcpu, sregs->cr8); - *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; kvm_x86_call(set_efer)(vcpu, sregs->efer); @@ -12511,7 +12477,7 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - update_cr8_intercept(vcpu); + kvm_set_cr8(vcpu, sregs->cr8); /* Older userspace won't unhalt the vcpu on reset. */ if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index df2261fa4f985..7e20b22d658b9 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1014,7 +1014,7 @@ static void __meminit free_pagetable(struct page *page, int order) #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE enum bootmem_type type = bootmem_type(page); - if (type == SECTION_INFO || type == MIX_SECTION_INFO) { + if (type == MIX_SECTION_INFO) { while (nr_pages--) put_page_bootmem(page++); } else { @@ -1028,13 +1028,24 @@ static void __meminit free_pagetable(struct page *page, int order) } } -static void __meminit free_hugepage_table(struct page *page, +static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, struct vmem_altmap *altmap) { - if (altmap) - vmem_altmap_free(altmap, PMD_SIZE / PAGE_SIZE); - else - free_pagetable(page, get_order(PMD_SIZE)); + unsigned long nr_pages = 1u << order; + + if (altmap) { + vmem_altmap_free(altmap, nr_pages); + } else if (PageReserved(page)) { + if (IS_ENABLED(CONFIG_HAVE_BOOTMEM_INFO_NODE) && + bootmem_type(page) == SECTION_INFO) { + while (nr_pages--) + put_page_bootmem(page++); + } else { + free_reserved_pages(page, nr_pages); + } + } else { + __free_pages(page, order); + } } static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) @@ -1118,7 +1129,8 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end, return; if (!direct) - free_pagetable(pte_page(*pte), 0); + /* We never populate base pages from the altmap. */ + free_vmemmap_pages(pte_page(*pte), 0, NULL); spin_lock(&init_mm.page_table_lock); pte_clear(&init_mm, addr, pte); @@ -1153,19 +1165,19 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end, if (IS_ALIGNED(addr, PMD_SIZE) && IS_ALIGNED(next, PMD_SIZE)) { if (!direct) - free_hugepage_table(pmd_page(*pmd), - altmap); + free_vmemmap_pages(pmd_page(*pmd), + PMD_ORDER, altmap); spin_lock(&init_mm.page_table_lock); pmd_clear(pmd); spin_unlock(&init_mm.page_table_lock); pages++; } else if (vmemmap_pmd_is_unused(addr, next)) { - free_hugepage_table(pmd_page(*pmd), - altmap); - spin_lock(&init_mm.page_table_lock); - pmd_clear(pmd); - spin_unlock(&init_mm.page_table_lock); + free_vmemmap_pages(pmd_page(*pmd), PMD_ORDER, + altmap); + spin_lock(&init_mm.page_table_lock); + pmd_clear(pmd); + spin_unlock(&init_mm.page_table_lock); } continue; } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index ea9e707e8abff..ef8b112fd7b69 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3520,7 +3520,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) @@ -3831,7 +3831,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* allocate module memory for x86 insns and extable */ header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size, &image, align, &rw_header, &rw_image, - jit_fill_hole); + jit_fill_hole, + bpf_prog_was_classic(prog)); if (!header) goto out_addrs; prog->aux->extable = (void *) image + roundup(proglen, align); diff --git a/arch/x86/platform/olpc/olpc-xo15-sci.c b/arch/x86/platform/olpc/olpc-xo15-sci.c index 82c51b6ec5288..276caf756a9c5 100644 --- a/arch/x86/platform/olpc/olpc-xo15-sci.c +++ b/arch/x86/platform/olpc/olpc-xo15-sci.c @@ -186,6 +186,7 @@ static int xo15_sci_add(struct acpi_device *device) static void xo15_sci_remove(struct acpi_device *device) { + device_init_wakeup(&device->dev, false); acpi_disable_gpe(NULL, xo15_sci_gpe); acpi_remove_gpe_handler(NULL, xo15_sci_gpe, xo15_sci_gpe_handler); cancel_work_sync(&sci_work); diff --git a/arch/x86/video/video-common.c b/arch/x86/video/video-common.c index 152789f00fcda..8ed82fff7638e 100644 --- a/arch/x86/video/video-common.c +++ b/arch/x86/video/video-common.c @@ -43,21 +43,26 @@ bool video_is_primary_device(struct device *dev) if (!pci_is_display(pdev)) return false; - if (pdev == vga_default_device()) - return true; - #ifdef CONFIG_SCREEN_INFO numres = screen_info_resources(si, res, ARRAY_SIZE(res)); - for (i = 0; i < numres; ++i) { - if (!(res[i].flags & IORESOURCE_MEM)) - continue; + if (numres > 0) { + for (i = 0; i < numres; ++i) { + if (!(res[i].flags & IORESOURCE_MEM)) + continue; + + if (pci_find_resource(pdev, &res[i])) + return true; + } - if (pci_find_resource(pdev, &res[i])) - return true; + return false; } #endif - return false; + /* + * No framebuffer was set up by the firmware/bootloader, so fall back + * to the default VGA device. + */ + return pdev == vga_default_device(); } EXPORT_SYMBOL(video_is_primary_device); diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 41f76f15caa13..b210f09eba464 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -527,6 +527,8 @@ void snp_prepare(void) cpus_read_lock(); + wbinvd_on_all_cpus(); + /* * MtrrFixDramModEn is not shared between threads on a core, * therefore it must be set on all CPUs prior to enabling SNP. diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 56aec6d504fee..6538a29a2bbd4 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -237,8 +237,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) prefetch(from); return __xtensa_copy_user((__force void *)to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * We need to return the number of bytes not cleared. Our memset() diff --git a/block/bdev.c b/block/bdev.c index bb0ffa3bb4dfb..85ce57bd2ae4f 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -446,15 +446,10 @@ EXPORT_SYMBOL_GPL(blockdev_superblock); void __init bdev_cache_init(void) { - int err; - bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| SLAB_ACCOUNT|SLAB_PANIC), init_once); - err = register_filesystem(&bd_type); - if (err) - panic("Cannot register bdev pseudo-fs"); blockdev_mnt = kern_mount(&bd_type); if (IS_ERR(blockdev_mnt)) panic("Cannot create bdev pseudo-fs"); @@ -1250,7 +1245,13 @@ void bdev_mark_dead(struct block_device *bdev, bool surprise) bdev->bd_holder_ops->mark_dead(bdev, surprise); else { mutex_unlock(&bdev->bd_holder_lock); - sync_blockdev(bdev); + /* + * On surprise removal the device is already gone; syncing is + * futile and can hang forever waiting on I/O that will never + * complete. Match fs_bdev_mark_dead(), which also skips it. + */ + if (!surprise) + sync_blockdev(bdev); } invalidate_bdev(bdev); diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index bc63bd220865d..51a9c35313256 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -164,20 +164,10 @@ static void blkg_free(struct blkcg_gq *blkg) static void __blkg_release(struct rcu_head *rcu) { struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); - struct blkcg *blkcg = blkg->blkcg; - int cpu; #ifdef CONFIG_BLK_CGROUP_PUNT_BIO WARN_ON(!bio_list_empty(&blkg->async_bios)); #endif - /* - * Flush all the non-empty percpu lockless lists before releasing - * us, given these stat belongs to us. - * - * blkg_stat_lock is for serializing blkg stat update - */ - for_each_possible_cpu(cpu) - __blkcg_rstat_flush(blkcg, cpu); /* release the blkcg and parent blkg refs this blkg has been holding */ css_put(&blkg->blkcg->css); @@ -195,6 +185,17 @@ static void __blkg_release(struct rcu_head *rcu) static void blkg_release(struct percpu_ref *ref) { struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); + struct blkcg *blkcg = blkg->blkcg; + int cpu; + + /* + * Flush all the non-empty percpu lockless lists before releasing + * us, given these stat belongs to us. + * + * blkg_stat_lock is for serializing blkg stat update + */ + for_each_possible_cpu(cpu) + __blkcg_rstat_flush(blkcg, cpu); call_rcu(&blkg->rcu_head, __blkg_release); } diff --git a/block/blk.h b/block/blk.h index b998a7761faf3..17e03656ba9ac 100644 --- a/block/blk.h +++ b/block/blk.h @@ -689,6 +689,7 @@ static inline int req_ref_read(struct request *req) static inline u64 blk_time_get_ns(void) { struct blk_plug *plug = current->plug; + u64 now; if (!plug || !in_task()) return ktime_get_ns(); @@ -697,12 +698,18 @@ static inline u64 blk_time_get_ns(void) * 0 could very well be a valid time, but rather than flag "this is * a valid timestamp" separately, just accept that we'll do an extra * ktime_get_ns() if we just happen to get 0 as the current time. + * + * cur_ktime can be zeroed by pre-emption the moment PF_BLOCK_TS is set. */ - if (!plug->cur_ktime) { - plug->cur_ktime = ktime_get_ns(); + now = READ_ONCE(plug->cur_ktime); + if (!now) { + now = ktime_get_ns(); + WRITE_ONCE(plug->cur_ktime, now); + /* Ensure PF_BLOCK_TS is set after cur_ktime. */ + barrier(); current->flags |= PF_BLOCK_TS; } - return plug->cur_ktime; + return now; } static inline ktime_t blk_time_get(void) diff --git a/block/genhd.c b/block/genhd.c index 7d6854fd28e95..0da6cdf3d5fb0 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -407,10 +407,6 @@ static void add_disk_final(struct gendisk *disk) struct device *ddev = disk_to_dev(disk); if (!(disk->flags & GENHD_FL_HIDDEN)) { - /* Make sure the first partition scan will be proceed */ - if (get_capacity(disk) && disk_has_partscan(disk)) - set_bit(GD_NEED_PART_SCAN, &disk->state); - bdev_add(disk->part0, ddev->devt); if (get_capacity(disk)) disk_scan_partitions(disk, BLK_OPEN_READ); diff --git a/block/ioctl.c b/block/ioctl.c index ab2c9ed799468..3d4ea1537457d 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -951,7 +951,7 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) u32 cmd_op = cmd->cmd_op; /* Read what we need from the SQE on the first issue */ - if (!(issue_flags & IORING_URING_CMD_REISSUE)) { + if (!(cmd->flags & IORING_URING_CMD_REISSUE)) { const struct io_uring_sqe *sqe = cmd->sqe; if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len || diff --git a/block/partitions/aix.c b/block/partitions/aix.c index 29b8f4cebb63d..f3c4174e003e9 100644 --- a/block/partitions/aix.c +++ b/block/partitions/aix.c @@ -226,6 +226,15 @@ int aix_partition(struct parsed_partitions *state) int next_lp_ix = 1; int lp_ix; + /* + * pvd was read into a fixed-size struct pvd whose ppe[] array + * holds ARRAY_SIZE(pvd->ppe) entries. pp_count is an + * unvalidated on-disk __be16, so clamp the scan to the array + * size to avoid walking past the allocation. + */ + if (numpps > ARRAY_SIZE(pvd->ppe)) + numpps = ARRAY_SIZE(pvd->ppe); + for (i = 0; i < numpps; i += 1) { struct ppe *p = pvd->ppe + i; unsigned int lv_ix; diff --git a/block/partitions/of.c b/block/partitions/of.c index c22b606610981..53664ea06b654 100644 --- a/block/partitions/of.c +++ b/block/partitions/of.c @@ -74,8 +74,10 @@ int of_partition(struct parsed_partitions *state) struct device_node *partitions_np = of_node_get(ddev->of_node); if (!partitions_np || - !of_device_is_compatible(partitions_np, "fixed-partitions")) + !of_device_is_compatible(partitions_np, "fixed-partitions")) { + of_node_put(partitions_np); return 0; + } slot = 1; /* Validate parition offset and size */ @@ -104,5 +106,6 @@ int of_partition(struct parsed_partitions *state) seq_buf_puts(&state->pp_buf, "\n"); + of_node_put(partitions_np); return 1; } diff --git a/crypto/Kconfig b/crypto/Kconfig index 103d1f58cb7c2..b07e7de61198a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -358,8 +358,8 @@ config CRYPTO_AES tristate "AES (Advanced Encryption Standard)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES - select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM - select CRYPTO_HASH if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM + select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC != n || CRYPTO_XCBC != n || CRYPTO_CCM != n + select CRYPTO_HASH if CRYPTO_CMAC != n || CRYPTO_XCBC != n || CRYPTO_CCM != n help AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 5a00c18eb145c..48c53f488e0fd 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -584,6 +584,8 @@ static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) return -EINVAL; con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg); + if (con->aead_assoclen >= 0x80000000u) + return -EINVAL; break; default: @@ -973,7 +975,7 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, ssize_t plen; /* use the existing memory in an allocated page */ - if (ctx->merge && !(msg->msg_flags & MSG_SPLICE_PAGES)) { + if (ctx->merge) { sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); sg = sgl->sg + sgl->cur - 1; @@ -1017,60 +1019,37 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, if (sgl->cur) sg_unmark_end(sg + sgl->cur - 1); - if (msg->msg_flags & MSG_SPLICE_PAGES) { - struct sg_table sgtable = { - .sgl = sg, - .nents = sgl->cur, - .orig_nents = sgl->cur, - }; - - plen = extract_iter_to_sg(&msg->msg_iter, len, &sgtable, - MAX_SGL_ENTS - sgl->cur, 0); - if (plen < 0) { - err = plen; + do { + struct page *pg; + unsigned int i = sgl->cur; + + plen = min_t(size_t, len, PAGE_SIZE); + + pg = alloc_page(GFP_KERNEL); + if (!pg) { + err = -ENOMEM; + goto unlock; + } + + sg_assign_page(sg + i, pg); + + err = memcpy_from_msg(page_address(sg_page(sg + i)), + msg, plen); + if (err) { + __free_page(sg_page(sg + i)); + sg_assign_page(sg + i, NULL); goto unlock; } - for (; sgl->cur < sgtable.nents; sgl->cur++) - get_page(sg_page(&sg[sgl->cur])); + sg[i].length = plen; len -= plen; ctx->used += plen; copied += plen; size -= plen; - } else { - do { - struct page *pg; - unsigned int i = sgl->cur; - - plen = min_t(size_t, len, PAGE_SIZE); - - pg = alloc_page(GFP_KERNEL); - if (!pg) { - err = -ENOMEM; - goto unlock; - } - - sg_assign_page(sg + i, pg); - - err = memcpy_from_msg( - page_address(sg_page(sg + i)), - msg, plen); - if (err) { - __free_page(sg_page(sg + i)); - sg_assign_page(sg + i, NULL); - goto unlock; - } - - sg[i].length = plen; - len -= plen; - ctx->used += plen; - copied += plen; - size -= plen; - sgl->cur++; - } while (len && sgl->cur < MAX_SGL_ENTS); - - ctx->merge = plen & (PAGE_SIZE - 1); - } + sgl->cur++; + } while (len && sgl->cur < MAX_SGL_ENTS); + + ctx->merge = plen & (PAGE_SIZE - 1); if (!size) sg_mark_end(sg + sgl->cur - 1); diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index cb651ab58d629..c6c2ce21895dd 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -9,10 +9,10 @@ * The following concept of the memory management is used: * * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is - * filled by user space with the data submitted via sendmsg (maybe with - * MSG_SPLICE_PAGES). Filling up the TX SGL does not cause a crypto operation - * -- the data will only be tracked by the kernel. Upon receipt of one recvmsg - * call, the caller must provide a buffer which is tracked with the RX SGL. + * filled by user space with the data submitted via sendmsg. Filling up the TX + * SGL does not cause a crypto operation -- the data will only be tracked by the + * kernel. Upon receipt of one recvmsg call, the caller must provide a buffer + * which is tracked with the RX SGL. * * During the processing of the recvmsg operation, the cipher request is * allocated and prepared. As part of the recvmsg operation, the processed diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index ba0a17fd95aca..35ebc3e0201b0 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -79,20 +79,6 @@ static int algif_skcipher_export(struct sock *sk, struct skcipher_request *req) return err; } -static void algif_skcipher_done(void *data, int err) -{ - struct af_alg_async_req *areq = data; - struct sock *sk = areq->sk; - - if (err) - goto out; - - err = algif_skcipher_export(sk, &areq->cra_u.skcipher_req); - -out: - af_alg_async_cb(data, err); -} - static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored, int flags) { @@ -171,43 +157,30 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, cflags |= CRYPTO_SKCIPHER_REQ_CONT; } - if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { - /* AIO operation */ - sock_hold(sk); - areq->iocb = msg->msg_iocb; - - /* Remember output size that will be generated. */ - areq->outlen = len; - - skcipher_request_set_callback(&areq->cra_u.skcipher_req, - cflags | - CRYPTO_TFM_REQ_MAY_SLEEP, - algif_skcipher_done, areq); - err = ctx->enc ? - crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : - crypto_skcipher_decrypt(&areq->cra_u.skcipher_req); - - /* AIO operation in progress */ - if (err == -EINPROGRESS) - return -EIOCBQUEUED; - - sock_put(sk); - } else { - /* Synchronous operation */ - skcipher_request_set_callback(&areq->cra_u.skcipher_req, - cflags | - CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &ctx->wait); - err = crypto_wait_req(ctx->enc ? - crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : - crypto_skcipher_decrypt(&areq->cra_u.skcipher_req), - &ctx->wait); - - if (!err) - err = algif_skcipher_export( - sk, &areq->cra_u.skcipher_req); - } + /* + * Force synchronous processing. The async (AIO) path passed the + * socket-wide ctx->iv into the request, which the worker + * dereferenced after the socket lock had been dropped, letting a + * concurrent sendmsg(ALG_SET_IV) inject an attacker IV. Mainline + * removed the AIO socket path in commit fcc77d33a34c ("net: Remove + * support for AIO on sockets"); the minimal stable fix is to always + * complete synchronously, so ctx->iv is only ever dereferenced under + * the socket lock. This also keeps the IV chaining intact: for + * ciphers with statesize == 0 (e.g. ctr, cbc) the chained IV is + * carried by the req->iv writeback into ctx->iv, which is only + * consistent on the synchronous path. + */ + skcipher_request_set_callback(&areq->cra_u.skcipher_req, + cflags | + CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &ctx->wait); + err = crypto_wait_req(ctx->enc ? + crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : + crypto_skcipher_decrypt(&areq->cra_u.skcipher_req), + &ctx->wait); + if (!err) + err = algif_skcipher_export(sk, &areq->cra_u.skcipher_req); free: af_alg_free_resources(areq); diff --git a/crypto/asymmetric_keys/verify_pefile.c b/crypto/asymmetric_keys/verify_pefile.c index 1f3b227ba7f22..cec99db14129a 100644 --- a/crypto/asymmetric_keys/verify_pefile.c +++ b/crypto/asymmetric_keys/verify_pefile.c @@ -305,6 +305,8 @@ static int pefile_digest_pe_contents(const void *pebuf, unsigned int pelen, if (pelen > hashed_bytes) { tmp = hashed_bytes + ctx->certs_size; + if (tmp <= hashed_bytes || pelen < tmp) + return -ELIBBAD; ret = crypto_shash_update(desc, pebuf + hashed_bytes, pelen - tmp); diff --git a/crypto/asymmetric_keys/x509_loader.c b/crypto/asymmetric_keys/x509_loader.c index a417413269989..0d516c77cc26a 100644 --- a/crypto/asymmetric_keys/x509_loader.c +++ b/crypto/asymmetric_keys/x509_loader.c @@ -20,7 +20,7 @@ int x509_load_certificate_list(const u8 cert_list[], */ if (end - p < 4) goto dodgy_cert; - if (p[0] != 0x30 && + if (p[0] != 0x30 || p[1] != 0x82) goto dodgy_cert; plen = (p[2] << 8) | p[3]; diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index b4b5a7198d841..27df9e1eb0580 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -375,6 +375,7 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, struct aead_instance *inst; struct chachapoly_instance_ctx *ctx; struct skcipher_alg_common *chacha; + const char *poly_name; int err; if (ivsize > CHACHAPOLY_IV_SIZE) @@ -396,9 +397,15 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, goto err_free_inst; chacha = crypto_spawn_skcipher_alg_common(&ctx->chacha); + poly_name = crypto_attr_alg_name(tb[2]); + if (IS_ERR(poly_name)) { + err = PTR_ERR(poly_name); + goto err_free_inst; + } + err = -EINVAL; - if (strcmp(crypto_attr_alg_name(tb[2]), "poly1305") && - strcmp(crypto_attr_alg_name(tb[2]), "poly1305-generic")) + if (strcmp(poly_name, "poly1305") && + strcmp(poly_name, "poly1305-generic")) goto err_free_inst; /* Need 16-byte IV size, including Initial Block Counter value */ if (chacha->ivsize != CHACHA_IV_SIZE) diff --git a/crypto/df_sp80090a.c b/crypto/df_sp80090a.c index b8134be6f7ad9..f4bb7be016e85 100644 --- a/crypto/df_sp80090a.c +++ b/crypto/df_sp80090a.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -141,10 +142,10 @@ int crypto_drbg_ctr_df(struct aes_enckey *aeskey, /* 10.4.2 step 2 -- calculate the entire length of all input data */ list_for_each_entry(seed, seedlist, list) inputlen += seed->len; - drbg_cpu_to_be32(inputlen, &L_N[0]); + put_unaligned_be32(inputlen, &L_N[0]); /* 10.4.2 step 3 */ - drbg_cpu_to_be32(bytes_to_return, &L_N[4]); + put_unaligned_be32(bytes_to_return, &L_N[4]); /* 10.4.2 step 5: length is L_N, input_string, one byte, padding */ padlen = (inputlen + sizeof(L_N) + 1) % (blocklen_bytes); @@ -175,7 +176,7 @@ int crypto_drbg_ctr_df(struct aes_enckey *aeskey, * holds zeros after allocation -- even the increment of i * is irrelevant as the increment remains within length of i */ - drbg_cpu_to_be32(i, iv); + put_unaligned_be32(i, iv); /* 10.4.2 step 9.2 -- BCC and concatenation with temp */ drbg_ctr_bcc(aeskey, temp + templen, K, &bcc_list, blocklen_bytes, keylen); diff --git a/crypto/drbg.c b/crypto/drbg.c index 9204e6edb4268..e3065fb9541b2 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -103,6 +103,7 @@ #include #include #include +#include /*************************************************************** * Backend cipher definitions available to DRBG @@ -377,7 +378,7 @@ static int drbg_ctr_generate(struct drbg_state *drbg, if (addtl && !list_empty(addtl)) { ret = drbg_ctr_update(drbg, addtl, 2); if (ret) - return 0; + return ret; } /* 10.2.1.5.2 step 4.1 */ @@ -601,7 +602,7 @@ static int drbg_hash_df(struct drbg_state *drbg, /* 10.4.1 step 3 */ input[0] = 1; - drbg_cpu_to_be32((outlen * 8), &input[1]); + put_unaligned_be32(outlen * 8, &input[1]); /* 10.4.1 step 4.1 -- concatenation of data for input into hash */ drbg_string_fill(&data, input, 5); @@ -1736,7 +1737,6 @@ static int drbg_kcapi_seed(struct crypto_rng *tfm, */ static inline int __init drbg_healthcheck_sanity(void) { - int len = 0; #define OUTBUFLEN 16 unsigned char buf[OUTBUFLEN]; struct drbg_state *drbg = NULL; @@ -1781,11 +1781,11 @@ static inline int __init drbg_healthcheck_sanity(void) max_request_bytes = drbg_max_request_bytes(drbg); drbg_string_fill(&addtl, buf, max_addtllen + 1); /* overflow addtllen with additional info string */ - len = drbg_generate(drbg, buf, OUTBUFLEN, &addtl); - BUG_ON(0 < len); + ret = drbg_generate(drbg, buf, OUTBUFLEN, &addtl); + BUG_ON(ret == 0); /* overflow max_bits */ - len = drbg_generate(drbg, buf, (max_request_bytes + 1), NULL); - BUG_ON(0 < len); + ret = drbg_generate(drbg, buf, max_request_bytes + 1, NULL); + BUG_ON(ret == 0); /* overflow max addtllen with personalization string */ ret = drbg_seed(drbg, &addtl, false); @@ -1832,7 +1832,7 @@ static inline void __init drbg_fill_array(struct rng_alg *alg, * it is selected. */ if (fips_enabled) - alg->base.cra_priority += 200; + alg->base.cra_priority += 2000; alg->base.cra_ctxsize = sizeof(struct drbg_state); alg->base.cra_module = THIS_MODULE; diff --git a/crypto/ecc.c b/crypto/ecc.c index 43b0def3a225c..6eb4d97a5f0d3 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -393,14 +393,26 @@ static uint128_t mul_64_64(u64 left, u64 right) return result; } -static uint128_t add_128_128(uint128_t a, uint128_t b) +/* Calculate addition with overflow checking. Returns true on wrap-around, + * false otherwise. + */ +static bool check_add_128_128_overflow(uint128_t *result, uint128_t a, + uint128_t b) { - uint128_t result; + bool carry; - result.m_low = a.m_low + b.m_low; - result.m_high = a.m_high + b.m_high + (result.m_low < a.m_low); + result->m_low = a.m_low + b.m_low; + carry = (result->m_low < a.m_low); - return result; + result->m_high = a.m_high + b.m_high + carry; + + /* Using constant-time bitwise arithmetic to prevent timing + * side-channels. + */ + carry = (result->m_high < a.m_high) | + ((result->m_high == a.m_high) & carry); + + return carry; } static void vli_mult(u64 *result, const u64 *left, const u64 *right, @@ -425,9 +437,7 @@ static void vli_mult(u64 *result, const u64 *left, const u64 *right, uint128_t product; product = mul_64_64(left[i], right[k - i]); - - r01 = add_128_128(r01, product); - r2 += (r01.m_high < product.m_high); + r2 += check_add_128_128_overflow(&r01, r01, product); } result[k] = r01.m_low; @@ -450,7 +460,7 @@ static void vli_umult(u64 *result, const u64 *left, u32 right, uint128_t product; product = mul_64_64(left[k], right); - r01 = add_128_128(r01, product); + check_add_128_128_overflow(&r01, r01, product); /* no carry */ result[k] = r01.m_low; r01.m_low = r01.m_high; @@ -487,8 +497,7 @@ static void vli_square(u64 *result, const u64 *left, unsigned int ndigits) product.m_low <<= 1; } - r01 = add_128_128(r01, product); - r2 += (r01.m_high < product.m_high); + r2 += check_add_128_128_overflow(&r01, r01, product); } result[k] = r01.m_low; diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c index 2c0602f0cd406..0cd7eb3676041 100644 --- a/crypto/ecrdsa.c +++ b/crypto/ecrdsa.c @@ -145,7 +145,7 @@ int ecrdsa_param_curve(void *context, size_t hdrlen, unsigned char tag, struct ecrdsa_ctx *ctx = context; ctx->curve_oid = look_up_OID(value, vlen); - if (!ctx->curve_oid) + if (ctx->curve_oid == OID__NR) return -EINVAL; ctx->curve = get_curve_by_oid(ctx->curve_oid); return 0; diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index c7ea40f900a77..03395b89cc61e 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -207,7 +207,7 @@ struct crypto_aead *krb5_prepare_encryption(const struct krb5_enctype *krb5, struct crypto_aead *ci = NULL; int ret = -ENOMEM; - ci = crypto_alloc_aead(krb5->encrypt_name, 0, 0); + ci = crypto_alloc_aead(krb5->encrypt_name, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(ci)) { ret = PTR_ERR(ci); if (ret == -ENOENT) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index ed0feaba23832..9f372442981e6 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -122,6 +122,8 @@ static int pcrypt_aead_encrypt(struct aead_request *req) return -EINPROGRESS; if (err == -EBUSY) { /* try non-parallel mode */ + aead_request_set_callback(creq, flags, req->base.complete, + req->base.data); return crypto_aead_encrypt(creq); } @@ -173,6 +175,8 @@ static int pcrypt_aead_decrypt(struct aead_request *req) return -EINPROGRESS; if (err == -EBUSY) { /* try non-parallel mode */ + aead_request_set_callback(creq, flags, req->base.complete, + req->base.data); return crypto_aead_decrypt(creq); } diff --git a/crypto/rng.c b/crypto/rng.c index 1d4b9177bad4d..a07569ed1e5e0 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -232,5 +232,16 @@ void crypto_unregister_rngs(struct rng_alg *algs, int count) } EXPORT_SYMBOL_GPL(crypto_unregister_rngs); +static void __exit rng_exit(void) +{ + int err; + + err = crypto_del_default_rng(); + if (err) + pr_err("Failed delete default RNG: %d\n", err); +} + +module_exit(rng_exit); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Random Number Generator"); diff --git a/drivers/accel/amdxdna/Makefile b/drivers/accel/amdxdna/Makefile index cf9bf19dedb9e..5c7911554c46e 100644 --- a/drivers/accel/amdxdna/Makefile +++ b/drivers/accel/amdxdna/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only amdxdna-y := \ + aie.o \ aie2_ctx.o \ aie2_error.o \ aie2_message.o \ diff --git a/drivers/accel/amdxdna/aie.c b/drivers/accel/amdxdna/aie.c new file mode 100644 index 0000000000000..4b3d4493128e9 --- /dev/null +++ b/drivers/accel/amdxdna/aie.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include + +#include "aie.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" + +void aie_dump_mgmt_chann_debug(struct aie_device *aie) +{ + struct amdxdna_dev *xdna = aie->xdna; + + XDNA_DBG(xdna, "i2x tail 0x%x", aie->mgmt_i2x.mb_tail_ptr_reg); + XDNA_DBG(xdna, "i2x head 0x%x", aie->mgmt_i2x.mb_head_ptr_reg); + XDNA_DBG(xdna, "i2x ringbuf 0x%x", aie->mgmt_i2x.rb_start_addr); + XDNA_DBG(xdna, "i2x rsize 0x%x", aie->mgmt_i2x.rb_size); + XDNA_DBG(xdna, "x2i tail 0x%x", aie->mgmt_x2i.mb_tail_ptr_reg); + XDNA_DBG(xdna, "x2i head 0x%x", aie->mgmt_x2i.mb_head_ptr_reg); + XDNA_DBG(xdna, "x2i ringbuf 0x%x", aie->mgmt_x2i.rb_start_addr); + XDNA_DBG(xdna, "x2i rsize 0x%x", aie->mgmt_x2i.rb_size); + XDNA_DBG(xdna, "x2i chann index 0x%x", aie->mgmt_chan_idx); + XDNA_DBG(xdna, "mailbox protocol major 0x%x", aie->mgmt_prot_major); + XDNA_DBG(xdna, "mailbox protocol minor 0x%x", aie->mgmt_prot_minor); +} + +void aie_destroy_chann(struct aie_device *aie, struct mailbox_channel **chann) +{ + struct amdxdna_dev *xdna = aie->xdna; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (!*chann) + return; + + xdna_mailbox_stop_channel(*chann); + xdna_mailbox_free_channel(*chann); + *chann = NULL; +} + +int aie_send_mgmt_msg_wait(struct aie_device *aie, struct xdna_mailbox_msg *msg) +{ + struct amdxdna_dev *xdna = aie->xdna; + struct xdna_notify *hdl = msg->handle; + int ret; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (!aie->mgmt_chann) + return -ENODEV; + + ret = xdna_send_msg_wait(xdna, aie->mgmt_chann, msg); + if (ret == -ETIME) + aie_destroy_chann(aie, &aie->mgmt_chann); + + if (!ret && *hdl->status) { + XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", + msg->opcode, *hdl->data); + ret = -EINVAL; + } + + return ret; +} + +int aie_check_protocol(struct aie_device *aie, u32 fw_major, u32 fw_minor) +{ + const struct amdxdna_fw_feature_tbl *feature; + bool found = false; + + for (feature = aie->xdna->dev_info->fw_feature_tbl; + feature->major; feature++) { + if (feature->major != fw_major) + continue; + if (fw_minor < feature->min_minor) + continue; + if (feature->max_minor > 0 && fw_minor > feature->max_minor) + continue; + + aie->feature_mask |= feature->features; + + /* firmware version matches one of the driver support entry */ + found = true; + } + + return found ? 0 : -EOPNOTSUPP; +} diff --git a/drivers/accel/amdxdna/aie.h b/drivers/accel/amdxdna/aie.h new file mode 100644 index 0000000000000..1bea14b79c7c9 --- /dev/null +++ b/drivers/accel/amdxdna/aie.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ +#ifndef _AIE_H_ +#define _AIE_H_ + +#include "amdxdna_pci_drv.h" +#include "amdxdna_mailbox.h" + +struct aie_device { + struct amdxdna_dev *xdna; + struct mailbox_channel *mgmt_chann; + struct xdna_mailbox_chann_res mgmt_x2i; + struct xdna_mailbox_chann_res mgmt_i2x; + u32 mgmt_chan_idx; + u32 mgmt_prot_major; + u32 mgmt_prot_minor; + unsigned long feature_mask; +}; + +#define DECLARE_AIE_MSG(name, op) \ + DECLARE_XDNA_MSG_COMMON(name, op, -1) +#define AIE_FEATURE_ON(aie, feature) test_bit(feature, &(aie)->feature_mask) + +void aie_dump_mgmt_chann_debug(struct aie_device *aie); +void aie_destroy_chann(struct aie_device *aie, struct mailbox_channel **chann); +int aie_send_mgmt_msg_wait(struct aie_device *aie, struct xdna_mailbox_msg *msg); +int aie_check_protocol(struct aie_device *aie, u32 fw_major, u32 fw_minor); + +#endif /* _AIE_H_ */ diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index eed3d0ec54133..a417b7d9ffac3 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -270,17 +270,13 @@ aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size) struct amdxdna_sched_job *job = handle; int ret = 0; - if (unlikely(!data)) - goto out; - - if (unlikely(size != sizeof(u32))) { + if (unlikely(!data || size != sizeof(u32))) { + job->drv_cmd->result = U32_MAX; ret = -EINVAL; - goto out; + } else { + job->drv_cmd->result = readl(data); } - job->drv_cmd->result = readl(data); - -out: aie2_sched_notify(job); return ret; } @@ -360,6 +356,7 @@ aie2_sched_job_run(struct drm_sched_job *sched_job) ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); break; case ATTACH_DEBUG_BO: + case DETACH_DEBUG_BO: ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); break; default: @@ -515,7 +512,7 @@ static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) struct alloc_requests *xrs_req; int ret; - if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) { + if (AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_TEMPORAL_ONLY)) { hwctx->num_unused_col = xdna->dev_handle->total_col - hwctx->num_col; hwctx->num_col = xdna->dev_handle->total_col; return aie2_create_context(xdna->dev_handle, hwctx); @@ -552,7 +549,7 @@ static void aie2_release_resource(struct amdxdna_hwctx *hwctx) struct amdxdna_dev *xdna = hwctx->client->xdna; int ret; - if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) { + if (AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_TEMPORAL_ONLY)) { ret = aie2_destroy_context(xdna->dev_handle, hwctx); if (ret && ret != -ENODEV) XDNA_ERR(xdna, "Destroy temporal only context failed, ret %d", ret); @@ -818,7 +815,7 @@ static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size if (!hwctx->cus) return -ENOMEM; - ret = amdxdna_pm_resume_get_locked(xdna); + ret = amdxdna_pm_resume_get(xdna); if (ret) goto free_cus; @@ -893,6 +890,7 @@ static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl, aie2_cmd_wait(hwctx, seq); if (cmd.result) { XDNA_ERR(xdna, "Response failure 0x%x", cmd.result); + ret = -EINVAL; goto put_obj; } @@ -965,7 +963,7 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) found = false; down_write(&xdna->notifier_lock); list_for_each_entry(mapp, &abo->mem.umap_list, node) { - if (mapp->invalid) { + if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) { found = true; break; } @@ -976,11 +974,9 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) up_write(&xdna->notifier_lock); return 0; } - kref_get(&mapp->refcnt); + up_write(&xdna->notifier_lock); - XDNA_DBG(xdna, "populate memory range %lx %lx", - mapp->vma->vm_start, mapp->vma->vm_end); mm = mapp->notifier.mm; if (!mmget_not_zero(mm)) { amdxdna_umap_put(mapp); diff --git a/drivers/accel/amdxdna/aie2_error.c b/drivers/accel/amdxdna/aie2_error.c index 58abb59b61535..70007b4363cd0 100644 --- a/drivers/accel/amdxdna/aie2_error.c +++ b/drivers/accel/amdxdna/aie2_error.c @@ -249,12 +249,12 @@ static u32 aie2_error_backtrack(struct amdxdna_dev_hdl *ndev, void *err_info, u3 enum aie_error_category cat; cat = aie_get_error_category(err->row, err->event_id, err->mod_type); - XDNA_ERR(ndev->xdna, "Row: %d, Col: %d, module %d, event ID %d, category %d", + XDNA_ERR(ndev->aie.xdna, "Row: %d, Col: %d, module %d, event ID %d, category %d", err->row, err->col, err->mod_type, err->event_id, cat); if (err->col >= 32) { - XDNA_WARN(ndev->xdna, "Invalid column number"); + XDNA_WARN(ndev->aie.xdna, "Invalid column number"); break; } @@ -294,7 +294,7 @@ static void aie2_error_worker(struct work_struct *err_work) e = container_of(err_work, struct async_event, work); - xdna = e->ndev->xdna; + xdna = e->ndev->aie.xdna; if (e->resp.status == MAX_AIE2_STATUS_CODE) return; @@ -329,7 +329,7 @@ static void aie2_error_worker(struct work_struct *err_work) void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev) { - struct amdxdna_dev *xdna = ndev->xdna; + struct amdxdna_dev *xdna = ndev->aie.xdna; struct async_events *events; events = ndev->async_events; @@ -344,7 +344,7 @@ void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev) int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev) { - struct amdxdna_dev *xdna = ndev->xdna; + struct amdxdna_dev *xdna = ndev->aie.xdna; u32 total_col = ndev->total_col; u32 total_size = ASYNC_BUF_SIZE * total_col; struct async_events *events; @@ -402,12 +402,15 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev) int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, struct amdxdna_drm_get_array *args) { - struct amdxdna_dev *xdna = ndev->xdna; + struct amdxdna_dev *xdna = ndev->aie.xdna; drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + if (!args->num_element) + return -EINVAL; + args->num_element = 1; - args->element_size = sizeof(ndev->last_async_err); + args->element_size = min(args->element_size, sizeof(ndev->last_async_err)); if (copy_to_user(u64_to_user_ptr(args->buffer), &ndev->last_async_err, args->element_size)) return -EFAULT; diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c index a1c546c3e81c3..3028968066064 100644 --- a/drivers/accel/amdxdna/aie2_message.c +++ b/drivers/accel/amdxdna/aie2_message.c @@ -16,6 +16,7 @@ #include #include +#include "aie.h" #include "aie2_msg_priv.h" #include "aie2_pci.h" #include "amdxdna_ctx.h" @@ -24,38 +25,12 @@ #include "amdxdna_mailbox_helper.h" #include "amdxdna_pci_drv.h" -#define DECLARE_AIE2_MSG(name, op) \ - DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE) - #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops) -static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev, - struct xdna_mailbox_msg *msg) -{ - struct amdxdna_dev *xdna = ndev->xdna; - struct xdna_notify *hdl = msg->handle; - int ret; - - if (!ndev->mgmt_chann) - return -ENODEV; - - ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg); - if (ret == -ETIME) - aie2_destroy_mgmt_chann(ndev); - - if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) { - XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", - msg->opcode, *hdl->data); - ret = -EINVAL; - } - - return ret; -} - void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size, dma_addr_t *dma_addr) { - struct amdxdna_dev *xdna = ndev->xdna; + struct amdxdna_dev *xdna = ndev->aie.xdna; void *vaddr; int order; @@ -79,7 +54,7 @@ void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size, void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size, void *cpu_addr, dma_addr_t dma_addr) { - struct amdxdna_dev *xdna = ndev->xdna; + struct amdxdna_dev *xdna = ndev->aie.xdna; if (amdxdna_iova_on(xdna)) { amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr); @@ -91,12 +66,12 @@ void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size, int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) { - DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND); + DECLARE_AIE_MSG(suspend, MSG_OP_SUSPEND); int ret; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) { - XDNA_ERR(ndev->xdna, "Failed to suspend fw, ret %d", ret); + XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret); return ret; } @@ -105,22 +80,22 @@ int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) { - DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME); + DECLARE_AIE_MSG(suspend, MSG_OP_RESUME); - return aie2_send_mgmt_msg_wait(ndev, &msg); + return aie_send_mgmt_msg_wait(&ndev->aie, &msg); } int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) { - DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); + DECLARE_AIE_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); int ret; req.type = type; req.value = value; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) { - XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret); + XDNA_ERR(ndev->aie.xdna, "Failed to set runtime config, ret %d", ret); return ret; } @@ -129,13 +104,13 @@ int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) { - DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); + DECLARE_AIE_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); int ret; req.type = type; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) { - XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret); + XDNA_ERR(ndev->aie.xdna, "Failed to get runtime config, ret %d", ret); return ret; } @@ -145,20 +120,20 @@ int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) { - DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); + DECLARE_AIE_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); req.pasid = pasid; - return aie2_send_mgmt_msg_wait(ndev, &msg); + return aie_send_mgmt_msg_wait(&ndev->aie, &msg); } int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version) { - DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); - struct amdxdna_dev *xdna = ndev->xdna; + DECLARE_AIE_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); + struct amdxdna_dev *xdna = ndev->aie.xdna; int ret; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) return ret; @@ -173,10 +148,10 @@ int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *ver int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata) { - DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); + DECLARE_AIE_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); int ret; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) return ret; @@ -211,10 +186,10 @@ int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *m int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, struct amdxdna_fw_ver *fw_ver) { - DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); + DECLARE_AIE_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); int ret; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) return ret; @@ -228,12 +203,12 @@ int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) { - DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); - struct amdxdna_dev *xdna = ndev->xdna; + DECLARE_AIE_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); + struct amdxdna_dev *xdna = ndev->aie.xdna; int ret; req.context_id = id; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret && ret != -ENODEV) XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); else if (ret == -ENODEV) @@ -245,7 +220,7 @@ static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) { - if (!AIE2_FEATURE_ON(ndev, AIE2_PREEMPT)) + if (!AIE_FEATURE_ON(&ndev->aie, AIE2_PREEMPT)) return PRIORITY_HIGH; switch (hwctx->qos.priority) { @@ -264,8 +239,8 @@ static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev, int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) { - DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); - struct amdxdna_dev *xdna = ndev->xdna; + DECLARE_AIE_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); + struct amdxdna_dev *xdna = ndev->aie.xdna; struct xdna_mailbox_chann_res x2i; struct xdna_mailbox_chann_res i2x; struct cq_pair *cq_pair; @@ -280,7 +255,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0; req.context_priority = aie2_get_context_priority(ndev, hwctx); - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) return ret; @@ -344,7 +319,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) { - struct amdxdna_dev *xdna = ndev->xdna; + struct amdxdna_dev *xdna = ndev->aie.xdna; int ret; if (!hwctx->priv->mbox_chann) @@ -363,14 +338,14 @@ int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwc int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) { - DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); - struct amdxdna_dev *xdna = ndev->xdna; + DECLARE_AIE_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); + struct amdxdna_dev *xdna = ndev->aie.xdna; int ret; req.context_id = context_id; req.buf_addr = addr; req.buf_size = size; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) return ret; @@ -392,14 +367,15 @@ static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg) int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, u32 *cols_filled) { - DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); - struct amdxdna_dev *xdna = ndev->xdna; - u32 buf_sz = size, aie_bitmap = 0; + DECLARE_AIE_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); + struct amdxdna_dev *xdna = ndev->aie.xdna; + u32 buf_sz, aie_bitmap = 0; struct amdxdna_client *client; dma_addr_t dma_addr; u8 *buff_addr; int ret; + buf_sz = ndev->metadata.cols * ndev->metadata.size; buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr); if (IS_ERR(buff_addr)) return PTR_ERR(buff_addr); @@ -414,8 +390,8 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, req.num_cols = hweight32(aie_bitmap); req.aie_bitmap = aie_bitmap; - drm_clflush_virt_range(buff_addr, size); /* device can access */ - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + drm_clflush_virt_range(buff_addr, req.dump_buff_size); /* device can access */ + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) { XDNA_ERR(xdna, "Error during NPU query, status %d", ret); goto fail; @@ -423,13 +399,14 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, XDNA_DBG(xdna, "Query NPU status completed"); - if (size < resp.size) { + if (buf_sz < resp.size) { ret = -EINVAL; - XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); + XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); goto fail; } - if (copy_to_user(buf, buff_addr, resp.size)) { + size = min(size, resp.size); + if (copy_to_user(buf, buff_addr, size)) { ret = -EFAULT; XDNA_ERR(xdna, "Failed to copy NPU status to user space"); goto fail; @@ -446,16 +423,17 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, struct amdxdna_drm_query_telemetry_header *header) { - DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); - struct amdxdna_dev *xdna = ndev->xdna; + DECLARE_AIE_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); + struct amdxdna_dev *xdna = ndev->aie.xdna; dma_addr_t dma_addr; - u32 buf_sz = size; + u32 buf_sz; u8 *addr; int ret; if (header->type >= MAX_TELEMETRY_TYPE) return -EINVAL; + buf_sz = min(size, SZ_4M); addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr); if (IS_ERR(addr)) return PTR_ERR(addr); @@ -464,20 +442,21 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, req.buf_size = buf_sz; req.type = header->type; - drm_clflush_virt_range(addr, size); /* device can access */ - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + drm_clflush_virt_range(addr, req.buf_size); /* device can access */ + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) { XDNA_ERR(xdna, "Query telemetry failed, status %d", ret); goto free_buf; } - if (size < resp.size) { + if (buf_sz < resp.size) { ret = -EINVAL; - XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); + XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); goto free_buf; } - if (copy_to_user(buf, addr, resp.size)) { + size = min(size, resp.size); + if (copy_to_user(buf, addr, size)) { ret = -EFAULT; XDNA_ERR(xdna, "Failed to copy telemetry to user space"); goto free_buf; @@ -506,8 +485,8 @@ int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, req.buf_addr = addr; req.buf_size = size; - XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size); - return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT); + XDNA_DBG(ndev->aie.xdna, "Register addr 0x%llx size 0x%x", addr, size); + return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg, TX_TIMEOUT); } int aie2_config_cu(struct amdxdna_hwctx *hwctx, @@ -866,7 +845,6 @@ static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo, int ret; u32 op; - op = amdxdna_cmd_get_op(cmd_abo); switch (op) { case ERT_START_CU: @@ -915,12 +893,12 @@ aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size); break; case ERT_START_NPU_PREEMPT: - if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT)) + if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) return -EOPNOTSUPP; ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size); break; case ERT_START_NPU_PREEMPT_ELF: - if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT)) + if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) return -EOPNOTSUPP; ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size); break; @@ -935,26 +913,12 @@ aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, void aie2_msg_init(struct amdxdna_dev_hdl *ndev) { - if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND)) + if (AIE_FEATURE_ON(&ndev->aie, AIE2_NPU_COMMAND)) ndev->exec_msg_ops = &npu_exec_message_ops; else ndev->exec_msg_ops = &legacy_exec_message_ops; } -void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev) -{ - struct amdxdna_dev *xdna = ndev->xdna; - - drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); - - if (!ndev->mgmt_chann) - return; - - xdna_mailbox_stop_channel(ndev->mgmt_chann); - xdna_mailbox_free_channel(ndev->mgmt_chann); - ndev->mgmt_chann = NULL; -} - static inline struct amdxdna_gem_obj * aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) { @@ -1199,14 +1163,14 @@ int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job * int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, struct app_health_report *report) { - DECLARE_AIE2_MSG(get_app_health, MSG_OP_GET_APP_HEALTH); - struct amdxdna_dev *xdna = ndev->xdna; + DECLARE_AIE_MSG(get_app_health, MSG_OP_GET_APP_HEALTH); + struct amdxdna_dev *xdna = ndev->aie.xdna; struct app_health_report *buf; dma_addr_t dma_addr; u32 buf_size; int ret; - if (!AIE2_FEATURE_ON(ndev, AIE2_APP_HEALTH)) { + if (!AIE_FEATURE_ON(&ndev->aie, AIE2_APP_HEALTH)) { XDNA_DBG(xdna, "App health feature not supported"); return -EOPNOTSUPP; } @@ -1222,8 +1186,8 @@ int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, req.context_id = context_id; req.buf_size = buf_size; - drm_clflush_virt_range(buf, sizeof(*report)); - ret = aie2_send_mgmt_msg_wait(ndev, &msg); + drm_clflush_virt_range(buf, req.buf_size); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); if (ret) { XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status); goto free_buf; diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h index f18e89a39e357..4a01d73209bbf 100644 --- a/drivers/accel/amdxdna/aie2_msg_priv.h +++ b/drivers/accel/amdxdna/aie2_msg_priv.h @@ -460,7 +460,7 @@ struct fatal_error_info { __u32 exception_pc; /* Program Counter at the time of the exception */ __u32 app_module; /* Error module name */ __u32 task_index; /* Index of the task in which the error occurred */ - __u32 reserved[128]; + __u32 reserved[127]; }; struct app_health_report { diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c index 4500b9ccb02e0..a7b923005ab7c 100644 --- a/drivers/accel/amdxdna/aie2_pci.c +++ b/drivers/accel/amdxdna/aie2_pci.c @@ -60,45 +60,6 @@ struct mgmt_mbox_chann_info { __u32 rsvd[4]; }; -static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor) -{ - const struct aie2_fw_feature_tbl *feature; - bool found = false; - - for (feature = ndev->priv->fw_feature_tbl; feature->major; feature++) { - if (feature->major != fw_major) - continue; - if (fw_minor < feature->min_minor) - continue; - if (feature->max_minor > 0 && fw_minor > feature->max_minor) - continue; - - ndev->feature_mask |= feature->features; - - /* firmware version matches one of the driver support entry */ - found = true; - } - - return found ? 0 : -EOPNOTSUPP; -} - -static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev) -{ - struct amdxdna_dev *xdna = ndev->xdna; - - XDNA_DBG(xdna, "i2x tail 0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg); - XDNA_DBG(xdna, "i2x head 0x%x", ndev->mgmt_i2x.mb_head_ptr_reg); - XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr); - XDNA_DBG(xdna, "i2x rsize 0x%x", ndev->mgmt_i2x.rb_size); - XDNA_DBG(xdna, "x2i tail 0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg); - XDNA_DBG(xdna, "x2i head 0x%x", ndev->mgmt_x2i.mb_head_ptr_reg); - XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr); - XDNA_DBG(xdna, "x2i rsize 0x%x", ndev->mgmt_x2i.rb_size); - XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx); - XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major); - XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor); -} - static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev) { struct mgmt_mbox_chann_info info_regs; @@ -128,13 +89,13 @@ static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev) reg[i] = readl(ndev->sram_base + off + i * sizeof(u32)); if (info_regs.magic != MGMT_MBOX_MAGIC) { - XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic); + XDNA_ERR(ndev->aie.xdna, "Invalid mbox magic 0x%x", info_regs.magic); ret = -EINVAL; goto done; } - i2x = &ndev->mgmt_i2x; - x2i = &ndev->mgmt_x2i; + i2x = &ndev->aie.mgmt_i2x; + x2i = &ndev->aie.mgmt_x2i; i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head); i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail); @@ -146,14 +107,15 @@ static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev) x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf); x2i->rb_size = info_regs.x2i_buf_sz; - ndev->mgmt_chan_idx = info_regs.msi_id; - ndev->mgmt_prot_major = info_regs.prot_major; - ndev->mgmt_prot_minor = info_regs.prot_minor; + ndev->aie.mgmt_chan_idx = info_regs.msi_id; + ndev->aie.mgmt_prot_major = info_regs.prot_major; + ndev->aie.mgmt_prot_minor = info_regs.prot_minor; - ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor); + ret = aie_check_protocol(&ndev->aie, ndev->aie.mgmt_prot_major, + ndev->aie.mgmt_prot_minor); done: - aie2_dump_chann_info_debug(ndev); + aie_dump_mgmt_chann_debug(&ndev->aie); /* Must clear address at FW_ALIVE_OFF */ writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF)); @@ -173,13 +135,14 @@ int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, continue; if (cfg->feature_mask && - bitmap_subset(&cfg->feature_mask, &ndev->feature_mask, AIE2_FEATURE_MAX)) + bitmap_subset(&cfg->feature_mask, &ndev->aie.feature_mask, + AIE2_FEATURE_MAX)) continue; value = val ? *val : cfg->value; ret = aie2_set_runtime_cfg(ndev, cfg->type, value); if (ret) { - XDNA_ERR(ndev->xdna, "Set type %d value %d failed", + XDNA_ERR(ndev->aie.xdna, "Set type %d value %d failed", cfg->type, value); return ret; } @@ -194,13 +157,13 @@ static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev) ret = aie2_suspend_fw(ndev); if (ret) { - XDNA_ERR(ndev->xdna, "Suspend firmware failed"); + XDNA_ERR(ndev->aie.xdna, "Suspend firmware failed"); return ret; } ret = aie2_resume_fw(ndev); if (ret) { - XDNA_ERR(ndev->xdna, "Resume firmware failed"); + XDNA_ERR(ndev->aie.xdna, "Resume firmware failed"); return ret; } @@ -213,19 +176,19 @@ static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev) ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL); if (ret) { - XDNA_ERR(ndev->xdna, "Runtime config failed"); + XDNA_ERR(ndev->aie.xdna, "Runtime config failed"); return ret; } ret = aie2_assign_mgmt_pasid(ndev, 0); if (ret) { - XDNA_ERR(ndev->xdna, "Can not assign PASID"); + XDNA_ERR(ndev->aie.xdna, "Can not assign PASID"); return ret; } ret = aie2_xdna_reset(ndev); if (ret) { - XDNA_ERR(ndev->xdna, "Reset firmware failed"); + XDNA_ERR(ndev->aie.xdna, "Reset firmware failed"); return ret; } @@ -236,21 +199,21 @@ static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev) { int ret; - ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver); + ret = aie2_query_firmware_version(ndev, &ndev->aie.xdna->fw_ver); if (ret) { - XDNA_ERR(ndev->xdna, "query firmware version failed"); + XDNA_ERR(ndev->aie.xdna, "query firmware version failed"); return ret; } ret = aie2_query_aie_version(ndev, &ndev->version); if (ret) { - XDNA_ERR(ndev->xdna, "Query AIE version failed"); + XDNA_ERR(ndev->aie.xdna, "Query AIE version failed"); return ret; } ret = aie2_query_aie_metadata(ndev, &ndev->metadata); if (ret) { - XDNA_ERR(ndev->xdna, "Query AIE metadata failed"); + XDNA_ERR(ndev->aie.xdna, "Query AIE metadata failed"); return ret; } @@ -262,8 +225,8 @@ static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev) static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev) { if (aie2_suspend_fw(ndev)) - XDNA_ERR(ndev->xdna, "Suspend_fw failed"); - XDNA_DBG(ndev->xdna, "Firmware suspended"); + XDNA_ERR(ndev->aie.xdna, "Suspend_fw failed"); + XDNA_DBG(ndev->aie.xdna, "Firmware suspended"); } static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action) @@ -331,7 +294,7 @@ static void aie2_hw_stop(struct amdxdna_dev *xdna) aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL); aie2_mgmt_fw_fini(ndev); - aie2_destroy_mgmt_chann(ndev); + aie_destroy_chann(&ndev->aie, &ndev->aie.mgmt_chann); drmm_kfree(&xdna->ddev, ndev->mbox); ndev->mbox = NULL; aie2_psp_stop(ndev->psp_hdl); @@ -374,8 +337,8 @@ static int aie2_hw_start(struct amdxdna_dev *xdna) goto disable_dev; } - ndev->mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox); - if (!ndev->mgmt_chann) { + ndev->aie.mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox); + if (!ndev->aie.mgmt_chann) { XDNA_ERR(xdna, "failed to alloc channel"); ret = -ENODEV; goto disable_dev; @@ -399,17 +362,17 @@ static int aie2_hw_start(struct amdxdna_dev *xdna) goto stop_psp; } - mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx); + mgmt_mb_irq = pci_irq_vector(pdev, ndev->aie.mgmt_chan_idx); if (mgmt_mb_irq < 0) { ret = mgmt_mb_irq; XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret); goto stop_psp; } - xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4; - ret = xdna_mailbox_start_channel(ndev->mgmt_chann, - &ndev->mgmt_x2i, - &ndev->mgmt_i2x, + xdna_mailbox_intr_reg = ndev->aie.mgmt_i2x.mb_head_ptr_reg + 4; + ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann, + &ndev->aie.mgmt_x2i, + &ndev->aie.mgmt_i2x, xdna_mailbox_intr_reg, mgmt_mb_irq); if (ret) { @@ -448,14 +411,14 @@ static int aie2_hw_start(struct amdxdna_dev *xdna) stop_fw: aie2_suspend_fw(ndev); - xdna_mailbox_stop_channel(ndev->mgmt_chann); + xdna_mailbox_stop_channel(ndev->aie.mgmt_chann); stop_psp: aie2_psp_stop(ndev->psp_hdl); fini_smu: aie2_smu_fini(ndev); free_channel: - xdna_mailbox_free_channel(ndev->mgmt_chann); - ndev->mgmt_chann = NULL; + xdna_mailbox_free_channel(ndev->aie.mgmt_chann); + ndev->aie.mgmt_chann = NULL; disable_dev: pci_disable_device(pdev); @@ -521,7 +484,7 @@ static int aie2_init(struct amdxdna_dev *xdna) return -ENOMEM; ndev->priv = xdna->dev_info->dev_priv; - ndev->xdna = xdna; + ndev->aie.xdna = xdna; for (i = 0; i < ARRAY_SIZE(npu_fw); i++) { fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]); @@ -644,23 +607,19 @@ static void aie2_fini(struct amdxdna_dev *xdna) static int aie2_get_aie_status(struct amdxdna_client *client, struct amdxdna_drm_get_info *args) { - struct amdxdna_drm_query_aie_status status; + struct amdxdna_drm_query_aie_status status = {}; struct amdxdna_dev *xdna = client->xdna; struct amdxdna_dev_hdl *ndev; + u32 buf_sz; int ret; ndev = xdna->dev_handle; - if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) { + buf_sz = min(args->buffer_size, sizeof(status)); + if (copy_from_user(&status, u64_to_user_ptr(args->buffer), buf_sz)) { XDNA_ERR(xdna, "Failed to copy AIE request into kernel"); return -EFAULT; } - if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) { - XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.", - status.buffer_size, ndev->metadata.cols * ndev->metadata.size); - return -EINVAL; - } - ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer), status.buffer_size, &status.cols_filled); if (ret) { @@ -668,7 +627,7 @@ static int aie2_get_aie_status(struct amdxdna_client *client, return ret; } - if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) { + if (copy_to_user(u64_to_user_ptr(args->buffer), &status, buf_sz)) { XDNA_ERR(xdna, "Failed to copy AIE request info to user space"); return -EFAULT; } @@ -683,6 +642,7 @@ static int aie2_get_aie_metadata(struct amdxdna_client *client, struct amdxdna_dev *xdna = client->xdna; struct amdxdna_dev_hdl *ndev; int ret = 0; + u32 buf_sz; ndev = xdna->dev_handle; meta = kzalloc_obj(*meta); @@ -714,7 +674,8 @@ static int aie2_get_aie_metadata(struct amdxdna_client *client, meta->shim.lock_count = ndev->metadata.shim.lock_count; meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count; - if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta))) + buf_sz = min(args->buffer_size, sizeof(*meta)); + if (copy_to_user(u64_to_user_ptr(args->buffer), meta, buf_sz)) ret = -EFAULT; kfree(meta); @@ -727,12 +688,14 @@ static int aie2_get_aie_version(struct amdxdna_client *client, struct amdxdna_drm_query_aie_version version; struct amdxdna_dev *xdna = client->xdna; struct amdxdna_dev_hdl *ndev; + u32 buf_sz; ndev = xdna->dev_handle; version.major = ndev->version.major; version.minor = ndev->version.minor; - if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version))) + buf_sz = min(args->buffer_size, sizeof(version)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &version, buf_sz)) return -EFAULT; return 0; @@ -743,13 +706,15 @@ static int aie2_get_firmware_version(struct amdxdna_client *client, { struct amdxdna_drm_query_firmware_version version; struct amdxdna_dev *xdna = client->xdna; + u32 buf_sz; version.major = xdna->fw_ver.major; version.minor = xdna->fw_ver.minor; version.patch = xdna->fw_ver.sub; version.build = xdna->fw_ver.build; - if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version))) + buf_sz = min(args->buffer_size, sizeof(version)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &version, buf_sz)) return -EFAULT; return 0; @@ -761,11 +726,13 @@ static int aie2_get_power_mode(struct amdxdna_client *client, struct amdxdna_drm_get_power_mode mode = {}; struct amdxdna_dev *xdna = client->xdna; struct amdxdna_dev_hdl *ndev; + u32 buf_sz; ndev = xdna->dev_handle; mode.power_mode = ndev->pw_mode; - if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode))) + buf_sz = min(args->buffer_size, sizeof(mode)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, buf_sz)) return -EFAULT; return 0; @@ -778,6 +745,7 @@ static int aie2_get_clock_metadata(struct amdxdna_client *client, struct amdxdna_dev *xdna = client->xdna; struct amdxdna_dev_hdl *ndev; int ret = 0; + u32 buf_sz; ndev = xdna->dev_handle; clock = kzalloc_obj(*clock); @@ -790,7 +758,8 @@ static int aie2_get_clock_metadata(struct amdxdna_client *client, snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock"); clock->h_clock.freq_mhz = ndev->hclk_freq; - if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock))) + buf_sz = min(args->buffer_size, sizeof(*clock)); + if (copy_to_user(u64_to_user_ptr(args->buffer), clock, buf_sz)) ret = -EFAULT; kfree(clock); @@ -816,12 +785,14 @@ static int aie2_get_sensors(struct amdxdna_client *client, scnprintf(sensor.label, sizeof(sensor.label), "Total Power"); scnprintf(sensor.units, sizeof(sensor.units), "mW"); + if (args->buffer_size < sizeof(sensor)) + goto out; + if (copy_to_user(u64_to_user_ptr(args->buffer), &sensor, sizeof(sensor))) return -EFAULT; + args->buffer_size -= sizeof(sensor); sensors_count++; - if (args->buffer_size <= sensors_count * sizeof(sensor)) - goto out; for (i = 0; i < min_t(u32, ndev->total_col, 8); i++) { memset(&sensor, 0, sizeof(sensor)); @@ -831,13 +802,15 @@ static int aie2_get_sensors(struct amdxdna_client *client, scnprintf(sensor.label, sizeof(sensor.label), "Column %d Utilization", i); scnprintf(sensor.units, sizeof(sensor.units), "%%"); + if (args->buffer_size < sizeof(sensor)) + goto out; + if (copy_to_user(u64_to_user_ptr(args->buffer) + sensors_count * sizeof(sensor), &sensor, sizeof(sensor))) return -EFAULT; + args->buffer_size -= sizeof(sensor); sensors_count++; - if (args->buffer_size <= sensors_count * sizeof(sensor)) - goto out; } out: @@ -933,6 +906,7 @@ static int aie2_query_resource_info(struct amdxdna_client *client, const struct amdxdna_dev_priv *priv; struct amdxdna_dev_hdl *ndev; struct amdxdna_dev *xdna; + u32 buf_sz; xdna = client->xdna; ndev = xdna->dev_handle; @@ -944,7 +918,8 @@ static int aie2_query_resource_info(struct amdxdna_client *client, res_info.npu_tops_curr = ndev->curr_tops; res_info.npu_task_curr = ndev->hwctx_num; - if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, sizeof(res_info))) + buf_sz = min(args->buffer_size, sizeof(res_info)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, buf_sz)) return -EFAULT; return 0; @@ -980,12 +955,7 @@ static int aie2_get_telemetry(struct amdxdna_client *client, XDNA_ERR(xdna, "Invalid buffer size"); return -EINVAL; } - telemetry_data_sz = args->buffer_size - header_sz; - if (telemetry_data_sz > SZ_4M) { - XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz); - return -EINVAL; - } header = kzalloc(header_sz, GFP_KERNEL); if (!header) @@ -1026,6 +996,7 @@ static int aie2_get_preempt_state(struct amdxdna_client *client, struct amdxdna_drm_attribute_state state = {}; struct amdxdna_dev *xdna = client->xdna; struct amdxdna_dev_hdl *ndev; + u32 buf_sz; ndev = xdna->dev_handle; if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE) @@ -1033,7 +1004,8 @@ static int aie2_get_preempt_state(struct amdxdna_client *client, else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE) state.state = ndev->frame_boundary_preempt; - if (copy_to_user(u64_to_user_ptr(args->buffer), &state, sizeof(state))) + buf_sz = min(args->buffer_size, sizeof(state)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &state, buf_sz)) return -EFAULT; return 0; diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h index efcf4be035f04..90fb0aafaf406 100644 --- a/drivers/accel/amdxdna/aie2_pci.h +++ b/drivers/accel/amdxdna/aie2_pci.h @@ -10,6 +10,7 @@ #include #include +#include "aie.h" #include "aie2_msg_priv.h" #include "amdxdna_mailbox.h" @@ -20,7 +21,7 @@ #define AIE2_DEVM_BASE 0x4000000 #define AIE2_DEVM_SIZE SZ_64M -#define NDEV2PDEV(ndev) (to_pci_dev((ndev)->xdna->ddev.dev)) +#define NDEV2PDEV(ndev) (to_pci_dev((ndev)->aie.xdna->ddev.dev)) #define AIE2_SRAM_OFF(ndev, addr) ((addr) - (ndev)->priv->sram_dev_addr) #define AIE2_MBOX_OFF(ndev, addr) ((addr) - (ndev)->priv->mbox_dev_addr) @@ -45,7 +46,7 @@ ({ \ typeof(ndev) _ndev = (ndev); \ ((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \ - pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->xdna->dev_info->mbox_bar); \ + pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->aie.xdna->dev_info->mbox_bar); \ }) #if IS_ENABLED(CONFIG_AMD_PMF) @@ -203,23 +204,16 @@ struct aie2_exec_msg_ops { }; struct amdxdna_dev_hdl { - struct amdxdna_dev *xdna; + struct aie_device aie; const struct amdxdna_dev_priv *priv; void __iomem *sram_base; void __iomem *smu_base; void __iomem *mbox_base; struct psp_device *psp_hdl; - struct xdna_mailbox_chann_res mgmt_x2i; - struct xdna_mailbox_chann_res mgmt_i2x; - u32 mgmt_chan_idx; - u32 mgmt_prot_major; - u32 mgmt_prot_minor; - u32 total_col; struct aie_version version; struct aie_metadata metadata; - unsigned long feature_mask; struct aie2_exec_msg_ops *exec_msg_ops; /* power management and clock*/ @@ -237,7 +231,6 @@ struct amdxdna_dev_hdl { /* Mailbox and the management channel */ struct mailbox *mbox; - struct mailbox_channel *mgmt_chann; struct async_events *async_events; enum aie2_dev_status dev_status; @@ -266,21 +259,12 @@ enum aie2_fw_feature { AIE2_FEATURE_MAX }; -struct aie2_fw_feature_tbl { - u64 features; - u32 major; - u32 max_minor; - u32 min_minor; -}; - #define AIE2_ALL_FEATURES GENMASK_ULL(AIE2_FEATURE_MAX - 1, AIE2_NPU_COMMAND) -#define AIE2_FEATURE_ON(ndev, feature) test_bit(feature, &(ndev)->feature_mask) struct amdxdna_dev_priv { const char *fw_path; const struct rt_config *rt_config; const struct dpm_clk_freq *dpm_clk_tbl; - const struct aie2_fw_feature_tbl *fw_feature_tbl; #define COL_ALIGN_NONE 0 #define COL_ALIGN_NATURE 1 @@ -306,7 +290,7 @@ extern const struct dpm_clk_freq npu1_dpm_clk_table[]; extern const struct dpm_clk_freq npu4_dpm_clk_table[]; extern const struct rt_config npu1_default_rt_cfg[]; extern const struct rt_config npu4_default_rt_cfg[]; -extern const struct aie2_fw_feature_tbl npu4_fw_feature_table[]; +extern const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[]; /* aie2_smu.c */ int aie2_smu_init(struct amdxdna_dev_hdl *ndev); diff --git a/drivers/accel/amdxdna/aie2_pm.c b/drivers/accel/amdxdna/aie2_pm.c index 29bd4403a94d4..5ec6728d04fd5 100644 --- a/drivers/accel/amdxdna/aie2_pm.c +++ b/drivers/accel/amdxdna/aie2_pm.c @@ -31,14 +31,14 @@ int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) { int ret; - ret = amdxdna_pm_resume_get_locked(ndev->xdna); + ret = amdxdna_pm_resume_get_locked(ndev->aie.xdna); if (ret) return ret; ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level); if (!ret) ndev->dpm_level = dpm_level; - amdxdna_pm_suspend_put(ndev->xdna); + amdxdna_pm_suspend_put(ndev->aie.xdna); return ret; } @@ -81,7 +81,7 @@ int aie2_pm_init(struct amdxdna_dev_hdl *ndev) int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target) { - struct amdxdna_dev *xdna = ndev->xdna; + struct amdxdna_dev *xdna = ndev->aie.xdna; u32 clk_gating, dpm_level; int ret; diff --git a/drivers/accel/amdxdna/aie2_smu.c b/drivers/accel/amdxdna/aie2_smu.c index d8c31924e501b..727637dac3a8d 100644 --- a/drivers/accel/amdxdna/aie2_smu.c +++ b/drivers/accel/amdxdna/aie2_smu.c @@ -46,7 +46,7 @@ static int aie2_smu_exec(struct amdxdna_dev_hdl *ndev, u32 reg_cmd, ret = readx_poll_timeout(readl, SMU_REG(ndev, SMU_RESP_REG), resp, resp, AIE2_INTERVAL, AIE2_TIMEOUT); if (ret) { - XDNA_ERR(ndev->xdna, "smu cmd %d timed out", reg_cmd); + XDNA_ERR(ndev->aie.xdna, "smu cmd %d timed out", reg_cmd); return ret; } @@ -54,7 +54,7 @@ static int aie2_smu_exec(struct amdxdna_dev_hdl *ndev, u32 reg_cmd, *out = readl(SMU_REG(ndev, SMU_OUT_REG)); if (resp != SMU_RESULT_OK) { - XDNA_ERR(ndev->xdna, "smu cmd %d failed, 0x%x", reg_cmd, resp); + XDNA_ERR(ndev->aie.xdna, "smu cmd %d failed, 0x%x", reg_cmd, resp); return -EINVAL; } @@ -69,7 +69,7 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ret = aie2_smu_exec(ndev, AIE2_SMU_SET_MPNPUCLK_FREQ, ndev->priv->dpm_clk_tbl[dpm_level].npuclk, &freq); if (ret) { - XDNA_ERR(ndev->xdna, "Set npu clock to %d failed, ret %d\n", + XDNA_ERR(ndev->aie.xdna, "Set npu clock to %d failed, ret %d\n", ndev->priv->dpm_clk_tbl[dpm_level].npuclk, ret); return ret; } @@ -78,7 +78,7 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ret = aie2_smu_exec(ndev, AIE2_SMU_SET_HCLK_FREQ, ndev->priv->dpm_clk_tbl[dpm_level].hclk, &freq); if (ret) { - XDNA_ERR(ndev->xdna, "Set h clock to %d failed, ret %d\n", + XDNA_ERR(ndev->aie.xdna, "Set h clock to %d failed, ret %d\n", ndev->priv->dpm_clk_tbl[dpm_level].hclk, ret); return ret; } @@ -87,7 +87,7 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ndev->max_tops = 2 * ndev->total_col; ndev->curr_tops = ndev->max_tops * freq / 1028; - XDNA_DBG(ndev->xdna, "MP-NPU clock %d, H clock %d\n", + XDNA_DBG(ndev->aie.xdna, "MP-NPU clock %d, H clock %d\n", ndev->npuclk_freq, ndev->hclk_freq); return 0; @@ -99,14 +99,14 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ret = aie2_smu_exec(ndev, AIE2_SMU_SET_HARD_DPMLEVEL, dpm_level, NULL); if (ret) { - XDNA_ERR(ndev->xdna, "Set hard dpm level %d failed, ret %d ", + XDNA_ERR(ndev->aie.xdna, "Set hard dpm level %d failed, ret %d ", dpm_level, ret); return ret; } ret = aie2_smu_exec(ndev, AIE2_SMU_SET_SOFT_DPMLEVEL, dpm_level, NULL); if (ret) { - XDNA_ERR(ndev->xdna, "Set soft dpm level %d failed, ret %d", + XDNA_ERR(ndev->aie.xdna, "Set soft dpm level %d failed, ret %d", dpm_level, ret); return ret; } @@ -116,7 +116,7 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ndev->max_tops = NPU4_DPM_TOPS(ndev, ndev->max_dpm_level); ndev->curr_tops = NPU4_DPM_TOPS(ndev, dpm_level); - XDNA_DBG(ndev->xdna, "MP-NPU clock %d, H clock %d\n", + XDNA_DBG(ndev->aie.xdna, "MP-NPU clock %d, H clock %d\n", ndev->npuclk_freq, ndev->hclk_freq); return 0; @@ -132,13 +132,13 @@ int aie2_smu_init(struct amdxdna_dev_hdl *ndev) */ ret = aie2_smu_exec(ndev, AIE2_SMU_POWER_OFF, 0, NULL); if (ret) { - XDNA_ERR(ndev->xdna, "Access power failed, ret %d", ret); + XDNA_ERR(ndev->aie.xdna, "Access power failed, ret %d", ret); return ret; } ret = aie2_smu_exec(ndev, AIE2_SMU_POWER_ON, 0, NULL); if (ret) { - XDNA_ERR(ndev->xdna, "Power on failed, ret %d", ret); + XDNA_ERR(ndev->aie.xdna, "Power on failed, ret %d", ret); return ret; } @@ -152,5 +152,5 @@ void aie2_smu_fini(struct amdxdna_dev_hdl *ndev) ndev->priv->hw_ops.set_dpm(ndev, 0); ret = aie2_smu_exec(ndev, AIE2_SMU_POWER_OFF, 0, NULL); if (ret) - XDNA_ERR(ndev->xdna, "Power off failed, ret %d", ret); + XDNA_ERR(ndev->aie.xdna, "Power off failed, ret %d", ret); } diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c index ff6c3e8e5a15b..5315466f137b5 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.c +++ b/drivers/accel/amdxdna/amdxdna_ctx.c @@ -356,16 +356,25 @@ int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct dr return -EINVAL; } - guard(mutex)(&xdna->dev_lock); + ret = amdxdna_pm_resume_get(xdna); + if (ret) { + XDNA_ERR(xdna, "Resume failed, ret %d", ret); + goto free_buf; + } + + mutex_lock(&xdna->dev_lock); hwctx = xa_load(&client->hwctx_xa, args->handle); if (!hwctx) { XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle); ret = -EINVAL; - goto free_buf; + goto unlock; } ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size); +unlock: + mutex_unlock(&xdna->dev_lock); + amdxdna_pm_suspend_put(xdna); free_buf: kfree(buf); return ret; @@ -386,16 +395,25 @@ int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl) if (!gobj) return -EINVAL; + ret = amdxdna_pm_resume_get(xdna); + if (ret) { + XDNA_ERR(xdna, "Resume failed, ret %d", ret); + goto put_obj; + } + abo = to_xdna_obj(gobj); - guard(mutex)(&xdna->dev_lock); + mutex_lock(&xdna->dev_lock); hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx); if (!hwctx) { ret = -EINVAL; - goto put_obj; + goto unlock; } ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl); +unlock: + mutex_unlock(&xdna->dev_lock); + amdxdna_pm_suspend_put(xdna); put_obj: drm_gem_object_put(gobj); return ret; @@ -478,6 +496,10 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, int ret, idx; XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt); + + if (!xdna->dev_info->ops->cmd_submit) + return -EOPNOTSUPP; + job = kzalloc_flex(*job, bos, arg_bo_cnt); if (!job) return -ENOMEM; @@ -491,6 +513,16 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, ret = -EINVAL; goto free_job; } + } else if (!drv_cmd) { + /* + * Only internal driver commands (drv_cmd != NULL) may omit a + * command BO. A user command submission with the invalid handle + * would leave job->cmd_bo NULL and later fault when the scheduler + * dereferences it in amdxdna_cmd_set_state(). + */ + XDNA_DBG(xdna, "Command BO handle required for user submission"); + ret = -EINVAL; + goto free_job; } ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt); diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c index 6e367ddb9e1be..cdf113dadc4db 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -200,7 +200,7 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni, xdna = to_xdna_dev(to_gobj(abo)->dev); XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d", - mapp->vma->vm_start, mapp->vma->vm_end, abo->type); + mapp->range.start, mapp->range.end, abo->type); if (!mmu_notifier_range_blockable(range)) return false; @@ -229,15 +229,23 @@ static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = { .invalidate = amdxdna_hmm_invalidate, }; +static inline bool compare_range(struct amdxdna_umap *mapp, + struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + return (!mapp->unmapped && mapp->notifier.mm == mm && + mapp->range.start == start && mapp->range.end == end); +} + static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, struct vm_area_struct *vma) { struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); struct amdxdna_umap *mapp; - down_read(&xdna->notifier_lock); + down_write(&xdna->notifier_lock); list_for_each_entry(mapp, &abo->mem.umap_list, node) { - if (!vma || mapp->vma == vma) { + if (!vma || compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) { if (!mapp->unmapped) { queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); mapp->unmapped = true; @@ -246,19 +254,16 @@ static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, break; } } - up_read(&xdna->notifier_lock); + up_write(&xdna->notifier_lock); } static void amdxdna_umap_release(struct kref *ref) { struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt); struct amdxdna_gem_obj *abo = mapp->abo; - struct vm_area_struct *vma = mapp->vma; struct amdxdna_dev *xdna; mmu_interval_notifier_remove(&mapp->notifier); - if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping) - mapping_clear_unevictable(vma->vm_file->f_mapping); xdna = to_xdna_dev(to_gobj(mapp->abo)->dev); down_write(&xdna->notifier_lock); @@ -291,12 +296,21 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, unsigned long len = vma->vm_end - vma->vm_start; unsigned long addr = vma->vm_start; struct amdxdna_umap *mapp; - u32 nr_pages; + unsigned long nr_pages; int ret; if (!xdna->dev_info->ops->hmm_invalidate) return 0; + down_read(&xdna->notifier_lock); + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + if (compare_range(mapp, current->mm, addr, addr + len)) { + up_read(&xdna->notifier_lock); + return 0; + } + } + up_read(&xdna->notifier_lock); + mapp = kzalloc_obj(*mapp); if (!mapp) return -ENOMEM; @@ -322,13 +336,10 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, mapp->range.start = vma->vm_start; mapp->range.end = vma->vm_end; mapp->range.default_flags = HMM_PFN_REQ_FAULT; - mapp->vma = vma; mapp->abo = abo; kref_init(&mapp->refcnt); INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work); - if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping) - mapping_set_unevictable(vma->vm_file->f_mapping); down_write(&xdna->notifier_lock); if (list_empty(&abo->mem.umap_list)) @@ -469,6 +480,7 @@ static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struc close_vma: vma->vm_ops->close(vma); + return ret; put_obj: drm_gem_object_put(gobj); return ret; @@ -1027,6 +1039,7 @@ int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { + struct amdxdna_client *client = filp->driver_priv; struct amdxdna_dev *xdna = to_xdna_dev(dev); struct amdxdna_drm_sync_bo *args = data; struct amdxdna_gem_obj *abo; @@ -1061,7 +1074,7 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, args->handle, args->offset, args->size); if (args->direction == SYNC_DIRECT_FROM_DEVICE) - ret = amdxdna_hwctx_sync_debug_bo(abo->client, args->handle); + ret = amdxdna_hwctx_sync_debug_bo(client, args->handle); put_obj: drm_gem_object_put(gobj); diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h index 957305ccb485a..10242f4a5ca98 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.h +++ b/drivers/accel/amdxdna/amdxdna_gem.h @@ -12,7 +12,6 @@ #include "amdxdna_pci_drv.h" struct amdxdna_umap { - struct vm_area_struct *vma; struct mmu_interval_notifier notifier; struct hmm_range range; struct work_struct hmm_unreg_work; diff --git a/drivers/accel/amdxdna/amdxdna_iommu.c b/drivers/accel/amdxdna/amdxdna_iommu.c index 4626434d4180c..77443872beb2d 100644 --- a/drivers/accel/amdxdna/amdxdna_iommu.c +++ b/drivers/accel/amdxdna/amdxdna_iommu.c @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -40,7 +41,7 @@ int amdxdna_iommu_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo) struct sg_table *sgt; dma_addr_t dma_addr; struct iova *iova; - size_t size; + ssize_t size; if (abo->type != AMDXDNA_BO_DEV_HEAP && abo->type != AMDXDNA_BO_SHMEM) return 0; @@ -65,7 +66,14 @@ int amdxdna_iommu_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo) size = iommu_map_sgtable(xdna->domain, dma_addr, sgt, IOMMU_READ | IOMMU_WRITE); + if (size < 0) { + XDNA_ERR(xdna, "iommu_map_sgtable failed: %zd", size); + __free_iova(&xdna->iovad, iova); + return size; + } + if (size < abo->mem.size) { + iommu_unmap(xdna->domain, dma_addr, size); __free_iova(&xdna->iovad, iova); return -ENXIO; } @@ -110,10 +118,12 @@ void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma iova_align(&xdna->iovad, size), IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); if (ret) - goto free_iova; + goto free_cpu_addr; return cpu_addr; +free_cpu_addr: + free_pages((unsigned long)cpu_addr, get_order(size)); free_iova: __free_iova(&xdna->iovad, iova); return ERR_PTR(ret); @@ -127,10 +137,30 @@ void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size, free_pages((unsigned long)cpu_addr, get_order(size)); } +static void amdxdna_cleanup_force_iova(struct drm_device *dev, void *res) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev); + + if (xdna->domain) { + iommu_detach_group(xdna->domain, xdna->group); + put_iova_domain(&xdna->iovad); + iova_cache_put(); + iommu_domain_free(xdna->domain); + } + + iommu_group_put(xdna->group); +} + +void amdxdna_iommu_fini(struct amdxdna_dev *xdna) +{ + if (xdna->group && !xdna->domain) + iommu_group_put(xdna->group); +} + int amdxdna_iommu_init(struct amdxdna_dev *xdna) { unsigned long order; - int ret; + int ret = 0; xdna->group = iommu_group_get(xdna->ddev.dev); if (!xdna->group || !force_iova) @@ -156,8 +186,14 @@ int amdxdna_iommu_init(struct amdxdna_dev *xdna) if (ret) goto put_iova; + ret = drmm_add_action(&xdna->ddev, amdxdna_cleanup_force_iova, NULL); + if (ret) + goto detach_group; + return 0; +detach_group: + iommu_detach_group(xdna->domain, xdna->group); put_iova: put_iova_domain(&xdna->iovad); iova_cache_put(); @@ -165,20 +201,8 @@ int amdxdna_iommu_init(struct amdxdna_dev *xdna) iommu_domain_free(xdna->domain); put_group: iommu_group_put(xdna->group); + xdna->group = NULL; xdna->domain = NULL; return ret; } - -void amdxdna_iommu_fini(struct amdxdna_dev *xdna) -{ - if (xdna->domain) { - iommu_detach_group(xdna->domain, xdna->group); - put_iova_domain(&xdna->iovad); - iova_cache_put(); - iommu_domain_free(xdna->domain); - } - - if (xdna->group) - iommu_group_put(xdna->group); -} diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c index e681a090752df..13c20bb3890c9 100644 --- a/drivers/accel/amdxdna/amdxdna_mailbox.c +++ b/drivers/accel/amdxdna/amdxdna_mailbox.c @@ -485,6 +485,9 @@ struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb) void xdna_mailbox_free_channel(struct mailbox_channel *mb_chann) { + if (!mb_chann) + return; + destroy_workqueue(mb_chann->work_q); kfree(mb_chann); } @@ -531,6 +534,9 @@ void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann) struct mailbox_msg *mb_msg; unsigned long msg_id; + if (!mb_chann) + return; + /* Disable an irq and wait. This might sleep. */ free_irq(mb_chann->msix_irq, mb_chann); @@ -538,7 +544,9 @@ void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann) drain_workqueue(mb_chann->work_q); /* We can clean up and release resources */ - xa_for_each(&mb_chann->chan_xa, msg_id, mb_msg) + xa_for_each_start(&mb_chann->chan_xa, msg_id, mb_msg, mb_chann->next_msgid) + mailbox_release_msg(mb_chann, mb_msg); + xa_for_each_range(&mb_chann->chan_xa, msg_id, mb_msg, 0, mb_chann->next_msgid - 1) mailbox_release_msg(mb_chann, mb_msg); xa_destroy(&mb_chann->chan_xa); diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h index 0661749917d68..5e0bf565a1ae6 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.h +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -66,6 +66,13 @@ struct amdxdna_dev_ops { int (*get_array)(struct amdxdna_client *client, struct amdxdna_drm_get_array *args); }; +struct amdxdna_fw_feature_tbl { + u64 features; + u32 major; + u32 max_minor; + u32 min_minor; +}; + /* * struct amdxdna_dev_info - Device hardware information * Record device static information, like reg, mbox, PSP, SMU bar index @@ -83,6 +90,7 @@ struct amdxdna_dev_info { size_t dev_mem_size; char *vbnv; const struct amdxdna_dev_priv *dev_priv; + const struct amdxdna_fw_feature_tbl *fw_feature_tbl; const struct amdxdna_dev_ops *ops; }; diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c index 85390e3cc9f98..0e0cd69cd1fbf 100644 --- a/drivers/accel/amdxdna/amdxdna_ubuf.c +++ b/drivers/accel/amdxdna/amdxdna_ubuf.c @@ -146,13 +146,17 @@ struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, ret = pin_user_pages_fast(va_ent[i].vaddr, npages, FOLL_WRITE | FOLL_LONGTERM, &ubuf->pages[start]); - if (ret < 0 || ret != npages) { - ret = -ENOMEM; + if (ret >= 0) { + start += ret; + if (ret != npages) { + XDNA_ERR(xdna, "Partially pinned pages %d/%u", ret, npages); + ret = -ENOMEM; + goto destroy_pages; + } + } else { XDNA_ERR(xdna, "Failed to pin pages ret %d", ret); goto destroy_pages; } - - start += ret; } exp_info.ops = &amdxdna_ubuf_dmabuf_ops; diff --git a/drivers/accel/amdxdna/npu1_regs.c b/drivers/accel/amdxdna/npu1_regs.c index 1320e924e5482..2ea7568a2e995 100644 --- a/drivers/accel/amdxdna/npu1_regs.c +++ b/drivers/accel/amdxdna/npu1_regs.c @@ -65,7 +65,7 @@ const struct dpm_clk_freq npu1_dpm_clk_table[] = { { 0 } }; -static const struct aie2_fw_feature_tbl npu1_fw_feature_table[] = { +static const struct amdxdna_fw_feature_tbl npu1_fw_feature_table[] = { { .major = 5, .min_minor = 7 }, { .features = BIT_U64(AIE2_NPU_COMMAND), .major = 5, .min_minor = 8 }, { 0 } @@ -75,7 +75,6 @@ static const struct amdxdna_dev_priv npu1_dev_priv = { .fw_path = "amdnpu/1502_00/", .rt_config = npu1_default_rt_cfg, .dpm_clk_tbl = npu1_dpm_clk_table, - .fw_feature_tbl = npu1_fw_feature_table, .col_align = COL_ALIGN_NONE, .mbox_dev_addr = NPU1_MBOX_BAR_BASE, .mbox_size = 0, /* Use BAR size */ @@ -120,5 +119,6 @@ const struct amdxdna_dev_info dev_npu1_info = { .vbnv = "RyzenAI-npu1", .device_type = AMDXDNA_DEV_TYPE_KMQ, .dev_priv = &npu1_dev_priv, + .fw_feature_tbl = npu1_fw_feature_table, .ops = &aie2_ops, }; diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c index 619bff042e52c..9689c56c83beb 100644 --- a/drivers/accel/amdxdna/npu4_regs.c +++ b/drivers/accel/amdxdna/npu4_regs.c @@ -88,7 +88,7 @@ const struct dpm_clk_freq npu4_dpm_clk_table[] = { { 0 } }; -const struct aie2_fw_feature_tbl npu4_fw_feature_table[] = { +const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[] = { { .major = 6, .min_minor = 12 }, { .features = BIT_U64(AIE2_NPU_COMMAND), .major = 6, .min_minor = 15 }, { .features = BIT_U64(AIE2_PREEMPT), .major = 6, .min_minor = 12 }, @@ -102,7 +102,6 @@ static const struct amdxdna_dev_priv npu4_dev_priv = { .fw_path = "amdnpu/17f0_10/", .rt_config = npu4_default_rt_cfg, .dpm_clk_tbl = npu4_dpm_clk_table, - .fw_feature_tbl = npu4_fw_feature_table, .col_align = COL_ALIGN_NATURE, .mbox_dev_addr = NPU4_MBOX_BAR_BASE, .mbox_size = 0, /* Use BAR size */ @@ -147,5 +146,6 @@ const struct amdxdna_dev_info dev_npu4_info = { .vbnv = "RyzenAI-npu4", .device_type = AMDXDNA_DEV_TYPE_KMQ, .dev_priv = &npu4_dev_priv, + .fw_feature_tbl = npu4_fw_feature_table, .ops = &aie2_ops, /* NPU4 can share NPU1's callback */ }; diff --git a/drivers/accel/amdxdna/npu5_regs.c b/drivers/accel/amdxdna/npu5_regs.c index c0ac5daf32ee5..98ee8780f3f5c 100644 --- a/drivers/accel/amdxdna/npu5_regs.c +++ b/drivers/accel/amdxdna/npu5_regs.c @@ -66,7 +66,6 @@ static const struct amdxdna_dev_priv npu5_dev_priv = { .fw_path = "amdnpu/17f0_11/", .rt_config = npu4_default_rt_cfg, .dpm_clk_tbl = npu4_dpm_clk_table, - .fw_feature_tbl = npu4_fw_feature_table, .col_align = COL_ALIGN_NATURE, .mbox_dev_addr = NPU5_MBOX_BAR_BASE, .mbox_size = 0, /* Use BAR size */ @@ -111,5 +110,6 @@ const struct amdxdna_dev_info dev_npu5_info = { .vbnv = "RyzenAI-npu5", .device_type = AMDXDNA_DEV_TYPE_KMQ, .dev_priv = &npu5_dev_priv, + .fw_feature_tbl = npu4_fw_feature_table, .ops = &aie2_ops, }; diff --git a/drivers/accel/amdxdna/npu6_regs.c b/drivers/accel/amdxdna/npu6_regs.c index ce591ed0d4832..31400cca5ec4c 100644 --- a/drivers/accel/amdxdna/npu6_regs.c +++ b/drivers/accel/amdxdna/npu6_regs.c @@ -66,7 +66,6 @@ static const struct amdxdna_dev_priv npu6_dev_priv = { .fw_path = "amdnpu/17f0_10/", .rt_config = npu4_default_rt_cfg, .dpm_clk_tbl = npu4_dpm_clk_table, - .fw_feature_tbl = npu4_fw_feature_table, .col_align = COL_ALIGN_NATURE, .mbox_dev_addr = NPU6_MBOX_BAR_BASE, .mbox_size = 0, /* Use BAR size */ @@ -112,5 +111,6 @@ const struct amdxdna_dev_info dev_npu6_info = { .vbnv = "RyzenAI-npu6", .device_type = AMDXDNA_DEV_TYPE_KMQ, .dev_priv = &npu6_dev_priv, + .fw_feature_tbl = npu4_fw_feature_table, .ops = &aie2_ops, }; diff --git a/drivers/accel/ivpu/ivpu_fw_log.c b/drivers/accel/ivpu/ivpu_fw_log.c index 275baf844b562..716467aa31567 100644 --- a/drivers/accel/ivpu/ivpu_fw_log.c +++ b/drivers/accel/ivpu/ivpu_fw_log.c @@ -43,6 +43,10 @@ static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *off ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", log->header_size); return -EINVAL; } + if (log->size < log->header_size) { + ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); + return -EINVAL; + } if ((char *)log + log->size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) { ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); return -EINVAL; diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c index 521931d1f7fca..b24f31a8b5672 100644 --- a/drivers/accel/ivpu/ivpu_job.c +++ b/drivers/accel/ivpu/ivpu_job.c @@ -208,9 +208,9 @@ static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id, priority); if (ret) - return ret; + ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id); - return 0; + return ret; } static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) @@ -281,10 +281,10 @@ static int ivpu_cmdq_register(struct ivpu_file_priv *file_priv, struct ivpu_cmdq } ret = ivpu_register_db(file_priv, cmdq); - if (ret) - return ret; + if (ret && vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) + ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id); - return 0; + return ret; } static int ivpu_cmdq_unregister(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) diff --git a/drivers/acpi/acpi_ipmi.c b/drivers/acpi/acpi_ipmi.c index 8f1aeae8b72ed..2dbed92d54b30 100644 --- a/drivers/acpi/acpi_ipmi.c +++ b/drivers/acpi/acpi_ipmi.c @@ -490,7 +490,7 @@ static void ipmi_bmc_gone(int iface) mutex_lock(&driver_data.ipmi_lock); list_for_each_entry_safe(iter, temp, &driver_data.ipmi_devices, head) { - if (iter->ipmi_ifnum != iface) { + if (iter->ipmi_ifnum == iface) { ipmi_device = iter; __ipmi_dev_kill(iter); break; @@ -550,7 +550,6 @@ acpi_ipmi_space_handler(u32 function, acpi_physical_address address, return AE_TYPE; } - acpi_ipmi_msg_get(tx_msg); mutex_lock(&driver_data.ipmi_lock); /* Do not add a tx_msg that can not be flushed. */ if (ipmi_device->dead) { @@ -558,6 +557,7 @@ acpi_ipmi_space_handler(u32 function, acpi_physical_address address, ipmi_msg_release(tx_msg); return AE_NOT_EXIST; } + acpi_ipmi_msg_get(tx_msg); spin_lock_irqsave(&ipmi_device->tx_msg_lock, flags); list_add_tail(&tx_msg->head, &ipmi_device->tx_msg_list); spin_unlock_irqrestore(&ipmi_device->tx_msg_lock, flags); diff --git a/drivers/acpi/acpi_pad.c b/drivers/acpi/acpi_pad.c index ec94b09bb7471..bff702835cfeb 100644 --- a/drivers/acpi/acpi_pad.c +++ b/drivers/acpi/acpi_pad.c @@ -435,12 +435,12 @@ static int acpi_pad_probe(struct platform_device *pdev) static void acpi_pad_remove(struct platform_device *pdev) { + acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev), + ACPI_DEVICE_NOTIFY, acpi_pad_notify); + mutex_lock(&isolated_cpus_lock); acpi_pad_idle_cpus(0); mutex_unlock(&isolated_cpus_lock); - - acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev), - ACPI_DEVICE_NOTIFY, acpi_pad_notify); } static const struct acpi_device_id pad_device_ids[] = { diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c index 386fc1abcbdcb..fc43df083738c 100644 --- a/drivers/acpi/acpi_tad.c +++ b/drivers/acpi/acpi_tad.c @@ -856,7 +856,7 @@ static int acpi_tad_probe(struct platform_device *pdev) * runtime suspend. Everything else should be taken care of by the ACPI * PM domain callbacks. */ - if (ACPI_TAD_AC_WAKE) { + if (caps & ACPI_TAD_AC_WAKE) { device_init_wakeup(dev, true); dev_pm_set_driver_flags(dev, DPM_FLAG_SMART_SUSPEND | DPM_FLAG_MAY_SKIP_RESUME); diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h index 3990d509bbabf..394ad8d92ff90 100644 --- a/drivers/acpi/acpica/acutils.h +++ b/drivers/acpi/acpica/acutils.h @@ -626,8 +626,6 @@ void acpi_ut_repair_name(char *name); #if defined (ACPI_DEBUGGER) || defined (ACPI_APPLICATION) || defined (ACPI_DEBUG_OUTPUT) u8 acpi_ut_safe_strcpy(char *dest, acpi_size dest_size, char *source); -void acpi_ut_safe_strncpy(char *dest, char *source, acpi_size dest_size); - u8 acpi_ut_safe_strcat(char *dest, acpi_size dest_size, char *source); u8 diff --git a/drivers/acpi/acpica/utnonansi.c b/drivers/acpi/acpica/utnonansi.c index 3a7952be65457..a465e5a1d3093 100644 --- a/drivers/acpi/acpica/utnonansi.c +++ b/drivers/acpi/acpica/utnonansi.c @@ -164,11 +164,4 @@ acpi_ut_safe_strncat(char *dest, return (FALSE); } -void acpi_ut_safe_strncpy(char *dest, char *source, acpi_size dest_size) -{ - /* Always terminate destination string */ - - strscpy_pad(dest, source, dest_size); -} - #endif diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c index d80276368b810..5df470eea7540 100644 --- a/drivers/acpi/button.c +++ b/drivers/acpi/button.c @@ -182,7 +182,6 @@ struct acpi_button { bool gpe_enabled; }; -static struct acpi_device *lid_device; static long lid_init_state = -1; static unsigned long lid_report_interval __read_mostly = 500; @@ -378,9 +377,29 @@ static int acpi_button_remove_fs(struct acpi_button *button) return 0; } +static struct acpi_device *lid_device; +static DEFINE_MUTEX(acpi_lid_lock); + +static void acpi_lid_save(struct acpi_device *adev) +{ + guard(mutex)(&acpi_lid_lock); + + lid_device = adev; +} + +static void acpi_lid_forget(struct acpi_device *adev) +{ + guard(mutex)(&acpi_lid_lock); + + if (lid_device == adev) + lid_device = NULL; +} + /* Driver Interface */ int acpi_lid_open(void) { + guard(mutex)(&acpi_lid_lock); + if (!lid_device) return -ENODEV; @@ -674,7 +693,7 @@ static int acpi_button_probe(struct platform_device *pdev) * This assumes there's only one lid device, or if there are * more we only care about the last one... */ - lid_device = device; + acpi_lid_save(device); } pr_info("%s [%s]\n", name, acpi_device_bid(device)); @@ -696,6 +715,9 @@ static void acpi_button_remove(struct platform_device *pdev) struct acpi_button *button = platform_get_drvdata(pdev); struct acpi_device *adev = button->adev; + if (button->type == ACPI_BUTTON_TYPE_LID) + acpi_lid_forget(adev); + switch (adev->device_type) { case ACPI_BUS_TYPE_POWER_BUTTON: acpi_remove_fixed_event_handler(ACPI_EVENT_POWER_BUTTON, diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index f370be8715ae2..34edec0f2bde2 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -185,8 +185,13 @@ show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_freq); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time); -/* Check for valid access_width, otherwise, fallback to using bit_width */ -#define GET_BIT_WIDTH(reg) ((reg)->access_width ? (8 << ((reg)->access_width - 1)) : (reg)->bit_width) +/* + * PCC reuses the access_width field as the subspace id, so only decode access + * size for non-PCC registers. Otherwise, use the bit_width. + */ +#define GET_BIT_WIDTH(reg) (((reg)->access_width && \ + (reg)->space_id != ACPI_ADR_SPACE_PLATFORM_COMM) ? \ + (8 << ((reg)->access_width - 1)) : (reg)->bit_width) /* Shift and apply the mask for CPC reads/writes */ #define MASK_VAL_READ(reg, val) (((val) >> (reg)->bit_offset) & \ @@ -1045,7 +1050,6 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) * by the bit width field; the access size is used to indicate * the PCC subspace id. */ - size = reg->bit_width; vaddr = GET_PCC_VADDR(reg->address, pcc_ss_id); } else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) @@ -1118,7 +1122,6 @@ static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val) * by the bit width field; the access size is used to indicate * the PCC subspace id. */ - size = reg->bit_width; vaddr = GET_PCC_VADDR(reg->address, pcc_ss_id); } else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index 9304ac996d41a..7f554ce6345f7 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -3069,6 +3069,8 @@ static void acpi_nfit_unregister(void *data) struct acpi_nfit_desc *acpi_desc = data; nvdimm_bus_unregister(acpi_desc->nvdimm_bus); + /* The nvdimm_bus object may have been freed, so clear the pointer. */ + acpi_desc->nvdimm_bus = NULL; } int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) @@ -3309,7 +3311,10 @@ static void acpi_nfit_remove_notify_handler(void *data) void acpi_nfit_shutdown(void *data) { struct acpi_nfit_desc *acpi_desc = data; - struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); + struct device *bus_dev; + + if (!acpi_desc || !acpi_desc->nvdimm_bus) + return; /* * Destruct under acpi_desc_lock so that nfit_handle_mce does not @@ -3324,6 +3329,7 @@ void acpi_nfit_shutdown(void *data) mutex_unlock(&acpi_desc->init_mutex); cancel_delayed_work_sync(&acpi_desc->dwork); + bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); /* * Bounce the nvdimm bus lock to make sure any in-flight * acpi_nfit_ars_rescan() submissions have had a chance to @@ -3406,9 +3412,14 @@ static int acpi_nfit_probe(struct platform_device *pdev) sz - sizeof(struct acpi_table_nfit)); if (rc) - return rc; + acpi_nfit_shutdown(acpi_desc); + + return rc; +} - return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc); +static void acpi_nfit_remove(struct platform_device *pdev) +{ + acpi_nfit_shutdown(platform_get_drvdata(pdev)); } static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle) @@ -3460,6 +3471,9 @@ static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle) { struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); + if (!acpi_desc) + return; + if (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG); else @@ -3489,6 +3503,7 @@ MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); static struct platform_driver acpi_nfit_driver = { .probe = acpi_nfit_probe, + .remove = acpi_nfit_remove, .driver = { .name = "acpi-nfit", .acpi_match_table = acpi_nfit_ids, diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index ee5facccbe10c..4482cf28f56ae 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -1143,7 +1143,7 @@ static int acpi_processor_get_lpi_info(struct acpi_processor *pr) return 0; } -int __weak acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) +int __weak __cpuidle acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) { return -ENODEV; } @@ -1156,7 +1156,7 @@ int __weak acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) * * Return: 0 for success or negative value for error */ -static int acpi_idle_lpi_enter(struct cpuidle_device *dev, +static int __cpuidle acpi_idle_lpi_enter(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) { struct acpi_processor *pr; @@ -1355,6 +1355,15 @@ void acpi_processor_register_idle_driver(void) int ret = -ENODEV; int cpu; + /* + * If a cpuidle driver is already registered, there is no need to + * evaluate _CST or attempt to register the ACPI idle driver. + */ + if (cpuidle_get_driver()) { + pr_debug("cpuidle driver %pS already registered.\n", cpuidle_get_driver()); + return; + } + acpi_processor_update_max_cstate(); /* diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index bc8050d8a6f51..56df4599d3604 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -871,7 +871,7 @@ bool acpi_dev_resource_interrupt(struct acpi_resource *ares, int index, EXPORT_SYMBOL_GPL(acpi_dev_resource_interrupt); /** - * acpi_dev_free_resource_list - Free resource from %acpi_dev_get_resources(). + * acpi_dev_free_resource_list - Free resource from acpi_dev_get_resources(). * @list: The head of the resource list to free. */ void acpi_dev_free_resource_list(struct list_head *list) @@ -991,7 +991,7 @@ static int __acpi_dev_get_resources(struct acpi_device *adev, * * The resultant struct resource objects are put on the list pointed to by * @list, that must be empty initially, as members of struct resource_entry - * objects. Callers of this routine should use %acpi_dev_free_resource_list() to + * objects. Callers of this routine should use acpi_dev_free_resource_list() to * free that list. * * The number of resources in the output list is returned on success, an error @@ -1032,7 +1032,7 @@ static int is_memory(struct acpi_resource *ares, void *not_used) * The resultant struct resource objects are put on the list pointed to * by @list, that must be empty initially, as members of struct * resource_entry objects. Callers of this routine should use - * %acpi_dev_free_resource_list() to free that list. + * acpi_dev_free_resource_list() to free that list. * * The number of resources in the output list is returned on success, * an error code reflecting the error condition is returned otherwise. diff --git a/drivers/acpi/riscv/cpuidle.c b/drivers/acpi/riscv/cpuidle.c index 624f9bbdb58c4..c76dbabff702e 100644 --- a/drivers/acpi/riscv/cpuidle.c +++ b/drivers/acpi/riscv/cpuidle.c @@ -66,7 +66,7 @@ int acpi_processor_ffh_lpi_probe(unsigned int cpu) return acpi_cpu_init_idle(cpu); } -int acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) +int __cpuidle acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) { u32 state = lpi->address; diff --git a/drivers/acpi/riscv/rimt.c b/drivers/acpi/riscv/rimt.c index 906282b0e63c0..e4538fa6c2c82 100644 --- a/drivers/acpi/riscv/rimt.c +++ b/drivers/acpi/riscv/rimt.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -257,11 +258,11 @@ static int rimt_iommu_xlate(struct device *dev, struct acpi_rimt_node *node, u32 rimt_fwnode = rimt_get_fwnode(node); /* - * The IOMMU drivers may not be probed yet. - * Defer the IOMMU configuration + * The IOMMU drivers may not be probed yet. Defer the IOMMU + * configuration if it's still in initialization stage. */ if (!rimt_fwnode) - return -EPROBE_DEFER; + return driver_deferred_probe_check_state(dev); /* * EPROBE_DEFER ensures IOMMU is probed before the devices that diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 6d479caf89cbd..d721d64a98580 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -82,33 +82,6 @@ static void amba_put_disable_pclk(struct amba_device *pcdev) } -static ssize_t driver_override_show(struct device *_dev, - struct device_attribute *attr, char *buf) -{ - struct amba_device *dev = to_amba_device(_dev); - ssize_t len; - - device_lock(_dev); - len = sprintf(buf, "%s\n", dev->driver_override); - device_unlock(_dev); - return len; -} - -static ssize_t driver_override_store(struct device *_dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct amba_device *dev = to_amba_device(_dev); - int ret; - - ret = driver_set_override(_dev, &dev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} -static DEVICE_ATTR_RW(driver_override); - #define amba_attr_func(name,fmt,arg...) \ static ssize_t name##_show(struct device *_dev, \ struct device_attribute *attr, char *buf) \ @@ -126,7 +99,6 @@ amba_attr_func(resource, "\t%016llx\t%016llx\t%016lx\n", static struct attribute *amba_dev_attrs[] = { &dev_attr_id.attr, &dev_attr_resource.attr, - &dev_attr_driver_override.attr, NULL, }; ATTRIBUTE_GROUPS(amba_dev); @@ -209,10 +181,11 @@ static int amba_match(struct device *dev, const struct device_driver *drv) { struct amba_device *pcdev = to_amba_device(dev); const struct amba_driver *pcdrv = to_amba_driver(drv); + int ret; mutex_lock(&pcdev->periphid_lock); if (!pcdev->periphid) { - int ret = amba_read_periphid(pcdev); + ret = amba_read_periphid(pcdev); /* * Returning any error other than -EPROBE_DEFER from bus match @@ -230,8 +203,9 @@ static int amba_match(struct device *dev, const struct device_driver *drv) mutex_unlock(&pcdev->periphid_lock); /* When driver_override is set, only bind to the matching driver */ - if (pcdev->driver_override) - return !strcmp(pcdev->driver_override, drv->name); + ret = device_match_driver_override(dev, drv); + if (ret >= 0) + return ret; return amba_lookup(pcdrv->id_table, pcdev) != NULL; } @@ -436,6 +410,7 @@ static const struct dev_pm_ops amba_pm = { const struct bus_type amba_bustype = { .name = "amba", .dev_groups = amba_dev_groups, + .driver_override = true, .match = amba_match, .uevent = amba_uevent, .probe = amba_probe, diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 9e61942245936..8f2ef1bd539f1 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1658,7 +1658,20 @@ static void binder_txn_latency_free(struct binder_transaction *t) static void binder_free_transaction(struct binder_transaction *t) { - struct binder_proc *target_proc = t->to_proc; + struct binder_thread *target_thread; + struct binder_proc *target_proc; + + spin_lock(&t->lock); + target_proc = t->to_proc; + target_thread = t->to_thread; + /* + * Pin target_thread to keep target_proc alive. Undelivered + * transactions with !target_thread are safe, as target_proc + * can only be the current context there. + */ + if (target_thread) + atomic_inc(&target_thread->tmp_ref); + spin_unlock(&t->lock); if (target_proc) { binder_inner_proc_lock(target_proc); @@ -1672,6 +1685,10 @@ static void binder_free_transaction(struct binder_transaction *t) t->buffer->transaction = NULL; binder_inner_proc_unlock(target_proc); } + + if (target_thread) + binder_thread_dec_tmpref(target_thread); + if (trace_binder_txn_latency_free_enabled()) binder_txn_latency_free(t); /* @@ -3080,12 +3097,11 @@ static void binder_transaction(struct binder_proc *proc, int t_debug_id = atomic_inc_return(&binder_last_id); ktime_t t_start_time = ktime_get(); struct lsm_context lsmctx = { }; - struct list_head sgc_head; - struct list_head pf_head; + size_t lsmctx_aligned_size = 0; + LIST_HEAD(sgc_head); + LIST_HEAD(pf_head); const void __user *user_buffer = (const void __user *) (uintptr_t)tr->data.ptr.buffer; - INIT_LIST_HEAD(&sgc_head); - INIT_LIST_HEAD(&pf_head); e = binder_transaction_log_add(&binder_transaction_log); e->debug_id = t_debug_id; @@ -3348,7 +3364,6 @@ static void binder_transaction(struct binder_proc *proc, if (target_node && target_node->txn_security_ctx) { u32 secid; - size_t added_size; security_cred_getsecid(proc->cred, &secid); ret = security_secid_to_secctx(secid, &lsmctx); @@ -3360,9 +3375,9 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_get_secctx_failed; } - added_size = ALIGN(lsmctx.len, sizeof(u64)); - extra_buffers_size += added_size; - if (extra_buffers_size < added_size) { + lsmctx_aligned_size = ALIGN(lsmctx.len, sizeof(u64)); + extra_buffers_size += lsmctx_aligned_size; + if (extra_buffers_size < lsmctx_aligned_size) { binder_txn_error("%d:%d integer overflow of extra_buffers_size\n", thread->pid, proc->pid); return_error = BR_FAILED_REPLY; @@ -3399,7 +3414,7 @@ static void binder_transaction(struct binder_proc *proc, size_t buf_offset = ALIGN(tr->data_size, sizeof(void *)) + ALIGN(tr->offsets_size, sizeof(void *)) + ALIGN(extra_buffers_size, sizeof(void *)) - - ALIGN(lsmctx.len, sizeof(u64)); + lsmctx_aligned_size; t->security_ctx = t->buffer->user_data + buf_offset; err = binder_alloc_copy_to_buffer(&target_proc->alloc, @@ -3454,7 +3469,7 @@ static void binder_transaction(struct binder_proc *proc, off_end_offset = off_start_offset + tr->offsets_size; sg_buf_offset = ALIGN(off_end_offset, sizeof(void *)); sg_buf_end_offset = sg_buf_offset + extra_buffers_size - - ALIGN(lsmctx.len, sizeof(u64)); + lsmctx_aligned_size; off_min = 0; for (buffer_offset = off_start_offset; buffer_offset < off_end_offset; buffer_offset += sizeof(binder_size_t)) { diff --git a/drivers/android/binder/allocation.rs b/drivers/android/binder/allocation.rs index b7b05e72970a2..ea5846e4da16a 100644 --- a/drivers/android/binder/allocation.rs +++ b/drivers/android/binder/allocation.rs @@ -259,7 +259,7 @@ impl Drop for Allocation { if let Some(offsets) = info.offsets.clone() { let view = AllocationView::new(self, offsets.start); - for i in offsets.step_by(size_of::()) { + for i in offsets.step_by(size_of::()) { if view.cleanup_object(i).is_err() { pr_warn!("Error cleaning up object at offset {}\n", i) } @@ -420,7 +420,8 @@ impl<'a> AllocationView<'a> { } fn cleanup_object(&self, index_offset: usize) -> Result { - let offset = self.alloc.read(index_offset)?; + let offset = self.alloc.read::(index_offset)?; + let offset: usize = offset.try_into().map_err(|_| EINVAL)?; let header = self.read::(offset)?; match header.type_ { BINDER_TYPE_WEAK_BINDER | BINDER_TYPE_BINDER => { diff --git a/drivers/android/binder/error.rs b/drivers/android/binder/error.rs index 45d85d4c28159..1296072c35d96 100644 --- a/drivers/android/binder/error.rs +++ b/drivers/android/binder/error.rs @@ -73,20 +73,17 @@ impl fmt::Debug for BinderError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.reply { BR_FAILED_REPLY => match self.source.as_ref() { - Some(source) => f - .debug_struct("BR_FAILED_REPLY") - .field("source", source) - .finish(), + Some(source) => source.fmt(f), None => f.pad("BR_FAILED_REPLY"), }, BR_DEAD_REPLY => f.pad("BR_DEAD_REPLY"), BR_FROZEN_REPLY => f.pad("BR_FROZEN_REPLY"), BR_TRANSACTION_PENDING_FROZEN => f.pad("BR_TRANSACTION_PENDING_FROZEN"), BR_TRANSACTION_COMPLETE => f.pad("BR_TRANSACTION_COMPLETE"), - _ => f - .debug_struct("BinderError") - .field("reply", &self.reply) - .finish(), + _ => match self.source.as_ref() { + Some(source) => source.fmt(f), + None => self.reply.fmt(f), + }, } } } diff --git a/drivers/android/binder/freeze.rs b/drivers/android/binder/freeze.rs index 53b60035639aa..f4df14568b252 100644 --- a/drivers/android/binder/freeze.rs +++ b/drivers/android/binder/freeze.rs @@ -154,10 +154,17 @@ impl DeliverToRead for FreezeMessage { } impl FreezeListener { - pub(crate) fn on_process_exit(&self, proc: &Arc) { + /// Called when this freeze listener is cleared abnormally. + /// + /// This occurs either because the process exited or because the process dropped its last + /// refcount on the node ref without explicitly removing the freeze listener first. + /// + /// The returned `KVVec` is just a value that should be dropped outside of the lock. + pub(crate) fn on_process_cleanup(&self, proc: &Process) -> KVVec> { if !self.is_clearing { - self.node.remove_freeze_listener(proc); + return self.node.remove_freeze_listener(proc); } + KVVec::new() } } diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs index 69f757ff7461e..c10148e9069f3 100644 --- a/drivers/android/binder/node.rs +++ b/drivers/android/binder/node.rs @@ -682,12 +682,13 @@ impl Node { } } - pub(crate) fn remove_freeze_listener(&self, p: &Arc) { - let _unused_capacity; + pub(crate) fn remove_freeze_listener(&self, p: &Process) -> KVVec> { let mut guard = self.owner.inner.lock(); let inner = self.inner.access_mut(&mut guard); let len = inner.freeze_list.len(); - inner.freeze_list.retain(|proc| !Arc::ptr_eq(proc, p)); + inner + .freeze_list + .retain(|proc| !core::ptr::eq::(&**proc, p)); if len == inner.freeze_list.len() { pr_warn!( "Could not remove freeze listener for {}\n", @@ -695,8 +696,9 @@ impl Node { ); } if inner.freeze_list.is_empty() { - _unused_capacity = mem::take(&mut inner.freeze_list); + return mem::take(&mut inner.freeze_list); } + KVVec::new() } pub(crate) fn freeze_list<'a>(&'a self, guard: &'a ProcessInner) -> &'a [Arc] { diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index 96b8440ceac62..cdd1a90797266 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -900,7 +900,11 @@ impl Process { pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult { // When handle is zero, try to get the context manager. if handle == 0 { - Ok(self.ctx.get_manager_node(true)?) + let node_ref = self.ctx.get_manager_node(true)?; + if core::ptr::eq(self, &*node_ref.node.owner) { + return Err(EINVAL.into()); + } + Ok(node_ref) } else { Ok(self.get_node_from_handle(handle, true)?) } @@ -942,6 +946,8 @@ impl Process { // To preserve original binder behaviour, we only fail requests where the manager tries to // increment references on itself. + let _to_free_freeze_listener; + let _to_free_freeze_listener_cleanup; let mut refs = self.node_refs.lock(); if let Some(info) = refs.by_handle.get_mut(&handle) { if info.node_ref().update(inc, strong) { @@ -957,6 +963,14 @@ impl Process { unsafe { info.node_ref2().node.remove_node_info(info) }; let id = info.node_ref().node.global_id(); + + if let Some(freeze) = *info.freeze() { + if let Some(fl) = refs.freeze_listeners.remove(&freeze) { + _to_free_freeze_listener_cleanup = fl.on_process_cleanup(&self); + _to_free_freeze_listener = fl; + } + } + refs.by_handle.remove(&handle); refs.by_node.remove(&id); refs.handle_is_present.release_id(handle as usize); @@ -1380,7 +1394,7 @@ impl Process { // Clean up freeze listeners. let freeze_listeners = take(&mut self.node_refs.lock().freeze_listeners); for listener in freeze_listeners.values() { - listener.on_process_exit(&self); + listener.on_process_cleanup(&self); } drop(freeze_listeners); diff --git a/drivers/android/binder/rust_binder_events.c b/drivers/android/binder/rust_binder_events.c index 488b1470060cc..5792aa59cc825 100644 --- a/drivers/android/binder/rust_binder_events.c +++ b/drivers/android/binder/rust_binder_events.c @@ -28,6 +28,9 @@ const char * const binder_command_strings[] = { "BC_DEAD_BINDER_DONE", "BC_TRANSACTION_SG", "BC_REPLY_SG", + "BC_REQUEST_FREEZE_NOTIFICATION", + "BC_CLEAR_FREEZE_NOTIFICATION", + "BC_FREEZE_NOTIFICATION_DONE", }; const char * const binder_return_strings[] = { @@ -51,7 +54,9 @@ const char * const binder_return_strings[] = { "BR_FAILED_REPLY", "BR_FROZEN_REPLY", "BR_ONEWAY_SPAM_SUSPECT", - "BR_TRANSACTION_PENDING_FROZEN" + "BR_TRANSACTION_PENDING_FROZEN", + "BR_FROZEN_BINDER", + "BR_CLEAR_FREEZE_NOTIFICATION_DONE", }; #define CREATE_TRACE_POINTS diff --git a/drivers/android/binder/stats.rs b/drivers/android/binder/stats.rs index ab75e9561cbf4..ec81dc7747db8 100644 --- a/drivers/android/binder/stats.rs +++ b/drivers/android/binder/stats.rs @@ -8,8 +8,8 @@ use crate::defs::*; use kernel::sync::atomic::{ordering::Relaxed, Atomic}; use kernel::{ioctl::_IOC_NR, seq_file::SeqFile, seq_print}; -const BC_COUNT: usize = _IOC_NR(BC_REPLY_SG) as usize + 1; -const BR_COUNT: usize = _IOC_NR(BR_TRANSACTION_PENDING_FROZEN) as usize + 1; +const BC_COUNT: usize = _IOC_NR(BC_FREEZE_NOTIFICATION_DONE) as usize + 1; +const BR_COUNT: usize = _IOC_NR(BR_CLEAR_FREEZE_NOTIFICATION_DONE) as usize + 1; pub(crate) static GLOBAL_STATS: BinderStats = BinderStats::new(); diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs index 97d5f31e8fe3c..3b85208139410 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -495,9 +495,16 @@ impl Thread { Ok(()) } + pub(crate) fn clear_extended_error(&self, debug_id: usize) { + self.inner.lock().extended_error = ExtendedError::new(debug_id as u32, BR_OK, 0); + } + pub(crate) fn get_extended_error(&self, data: UserSlice) -> Result { let mut writer = data.writer(); - let ee = self.inner.lock().extended_error; + let mut inner = self.inner.lock(); + let ee = inner.extended_error; + inner.extended_error = ExtendedError::new(0, BR_OK, 0); + drop(inner); writer.write(&ee)?; Ok(()) } @@ -1109,7 +1116,10 @@ impl Thread { inner.pop_transaction_to_reply(thread.as_ref()) } { let reply = Err(BR_DEAD_REPLY); - if !transaction.from.deliver_single_reply(reply, &transaction) { + if !transaction + .from + .deliver_single_reply(reply, &transaction, None) + { break; } @@ -1121,8 +1131,9 @@ impl Thread { &self, reply: Result, u32>, transaction: &DArc, + extended_error: Option, ) { - if self.deliver_single_reply(reply, transaction) { + if self.deliver_single_reply(reply, transaction, extended_error) { transaction.from.unwind_transaction_stack(); } } @@ -1136,6 +1147,7 @@ impl Thread { &self, reply: Result, u32>, transaction: &DArc, + extended_error: Option, ) -> bool { if let Ok(transaction) = &reply { crate::trace::trace_transaction(true, transaction, Some(&self.task)); @@ -1152,6 +1164,12 @@ impl Thread { return true; } + if let Some(ee) = extended_error { + if inner.extended_error.command == BR_OK { + inner.extended_error = ee; + } + } + match reply { Ok(work) => { inner.push_work(work); @@ -1222,6 +1240,9 @@ impl Thread { info.buffers_size = td.buffers_size as usize; // SAFETY: Above `read` call initializes all bytes, so this union read is ok. info.target_handle = unsafe { td.transaction_data.target.handle }; + + info.debug_id = super::next_debug_id(); + Ok(()) } @@ -1230,6 +1251,8 @@ impl Thread { let mut info = TransactionInfo::zeroed(); self.read_transaction_info(cmd, reader, &mut info)?; + self.clear_extended_error(info.debug_id); + let ret = if info.is_reply { self.reply_inner(&mut info) } else if info.is_oneway() { @@ -1239,23 +1262,21 @@ impl Thread { }; if let Err(err) = ret { - if err.reply != BR_TRANSACTION_COMPLETE { - info.reply = err.reply; - } - self.push_return_work(err.reply); - if let Some(source) = &err.source { - info.errno = source.to_errno(); + if err.reply != BR_TRANSACTION_COMPLETE { info.reply = err.reply; + if let Some(source) = &err.source { + info.errno = source.to_errno(); - { - let mut ee = self.inner.lock().extended_error; - ee.command = err.reply; - ee.param = source.to_errno(); + { + let mut inner = self.inner.lock(); + inner.extended_error = + ExtendedError::new(info.debug_id as u32, err.reply, source.to_errno()); + } } pr_warn!( - "{}:{} transaction to {} failed: {source:?}", + "{}:{} transaction to {} failed: {err:?}", info.from_pid, info.from_tid, info.to_pid @@ -1320,18 +1341,24 @@ impl Thread { let allow_fds = orig.flags & TF_ACCEPT_FDS != 0; let reply = Transaction::new_reply(self, process, info, allow_fds)?; self.inner.lock().push_work(completion); - orig.from.deliver_reply(Ok(reply), &orig); + orig.from.deliver_reply(Ok(reply), &orig, None); Ok(()) })() .map_err(|mut err| { // At this point we only return `BR_TRANSACTION_COMPLETE` to the caller, and we must let // the sender know that the transaction has completed (with an error in this case). + pr_warn!( - "Failure {:?} during reply - delivering BR_FAILED_REPLY to sender.", - err + "{}:{} reply to {} failed: {err:?}", + info.from_pid, + info.from_tid, + info.to_pid ); - let reply = Err(BR_FAILED_REPLY); - orig.from.deliver_reply(reply, &orig); + + let param = err.source.as_ref().map_or(0, |e| e.to_errno()); + let ee = ExtendedError::new(info.debug_id as u32, err.reply, param); + orig.from + .deliver_reply(Err(BR_FAILED_REPLY), &orig, Some(ee)); err.reply = BR_TRANSACTION_COMPLETE; err }); diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder/transaction.rs index 1d9b66920a216..0e5d07b7e6f0a 100644 --- a/drivers/android/binder/transaction.rs +++ b/drivers/android/binder/transaction.rs @@ -42,6 +42,7 @@ pub(crate) struct TransactionInfo { pub(crate) reply: u32, pub(crate) oneway_spam_suspect: bool, pub(crate) is_reply: bool, + pub(crate) debug_id: usize, } impl TransactionInfo { @@ -93,7 +94,6 @@ impl Transaction { from: &Arc, info: &mut TransactionInfo, ) -> BinderResult> { - let debug_id = super::next_debug_id(); let allow_fds = node_ref.node.flags & FLAT_BINDER_FLAG_ACCEPTS_FDS != 0; let txn_security_ctx = node_ref.node.flags & FLAT_BINDER_FLAG_TXN_SECURITY_CTX != 0; let mut txn_security_ctx_off = if txn_security_ctx { Some(0) } else { None }; @@ -101,7 +101,7 @@ impl Transaction { let mut alloc = match from.copy_transaction_data( to.clone(), info, - debug_id, + info.debug_id, allow_fds, txn_security_ctx_off.as_mut(), ) { @@ -128,7 +128,7 @@ impl Transaction { let data_address = alloc.ptr; Ok(DTRWrap::arc_pin_init(pin_init!(Transaction { - debug_id, + debug_id: info.debug_id, target_node: Some(target_node), from_parent, sender_euid: Kuid::current_euid(), @@ -152,9 +152,8 @@ impl Transaction { info: &mut TransactionInfo, allow_fds: bool, ) -> BinderResult> { - let debug_id = super::next_debug_id(); let mut alloc = - match from.copy_transaction_data(to.clone(), info, debug_id, allow_fds, None) { + match from.copy_transaction_data(to.clone(), info, info.debug_id, allow_fds, None) { Ok(alloc) => alloc, Err(err) => { pr_warn!("Failure in copy_transaction_data: {:?}", err); @@ -165,7 +164,7 @@ impl Transaction { alloc.set_info_clear_on_drop(); } Ok(DTRWrap::arc_pin_init(pin_init!(Transaction { - debug_id, + debug_id: info.debug_id, target_node: None, from_parent: None, sender_euid: Kuid::current_euid(), @@ -394,7 +393,7 @@ impl DeliverToRead for Transaction { let send_failed_reply = ScopeGuard::new(|| { if self.target_node.is_some() && self.flags & TF_ONE_WAY == 0 { let reply = Err(BR_FAILED_REPLY); - self.from.deliver_reply(reply, &self); + self.from.deliver_reply(reply, &self, None); } self.drop_outstanding_txn(); }); @@ -478,7 +477,7 @@ impl DeliverToRead for Transaction { // If this is not a reply or oneway transaction, then send a dead reply. if self.target_node.is_some() && self.flags & TF_ONE_WAY == 0 { let reply = Err(BR_DEAD_REPLY); - self.from.deliver_reply(reply, &self); + self.from.deliver_reply(reply, &self, None); } self.drop_outstanding_txn(); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 3d0027ec33c2a..80196326ff8ff 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -1338,7 +1338,7 @@ static int ata_hpa_resize(struct ata_device *dev) /* do we need to do it? */ if ((dev->class != ATA_DEV_ATA && dev->class != ATA_DEV_ZAC) || !ata_id_has_lba(dev->id) || !ata_id_hpa_enabled(dev->id) || - (dev->quirks & ATA_QUIRK_BROKEN_HPA)) + (dev->quirks & ATA_QUIRK_BROKEN_HPA) || ata_id_is_locked(dev->id)) return 0; /* read native max address */ @@ -1540,6 +1540,7 @@ unsigned int ata_exec_internal(struct ata_device *dev, struct ata_taskfile *tf, { struct ata_link *link = dev->link; struct ata_port *ap = link->ap; + const bool owns_eh_mutex = ap->host->eh_owner == current; u8 command = tf->command; struct ata_queued_cmd *qc; struct scatterlist sgl; @@ -1617,11 +1618,25 @@ unsigned int ata_exec_internal(struct ata_device *dev, struct ata_taskfile *tf, } } - ata_eh_release(ap); + if (owns_eh_mutex) { + /* + * To prevent that the compiler complains about the + * ata_eh_release() call below. + */ + __acquire(&ap->host->eh_mutex); + ata_eh_release(ap); + } rc = wait_for_completion_timeout(&wait, msecs_to_jiffies(timeout)); - ata_eh_acquire(ap); + if (owns_eh_mutex) { + ata_eh_acquire(ap); + /* + * To prevent that the compiler complains about the above + * ata_eh_acquire() call. + */ + __release(&ap->host->eh_mutex); + } ata_sff_flush_pio_task(ap); @@ -2832,6 +2847,24 @@ static void ata_dev_config_cpr(struct ata_device *dev) if (!nr_cpr) goto out; + /* + * The device reports the number of CPR descriptors independently of the + * log size, and that count is also used to emit VPD page B9h into the + * fixed-size rbuf. Reject a count larger than what that buffer can hold + * (ATA_DEV_MAX_CPR) or larger than the log the device actually returned. + */ + if (nr_cpr > ATA_DEV_MAX_CPR) { + ata_dev_warn(dev, + "Too many concurrent positioning ranges\n"); + goto out; + } + + if (buf_len < 64 + (size_t)nr_cpr * 32) { + ata_dev_warn(dev, + "Invalid number of concurrent positioning ranges\n"); + goto out; + } + cpr_log = kzalloc_flex(*cpr_log, cpr, nr_cpr); if (!cpr_log) goto out; @@ -3959,7 +3992,7 @@ int ata_dev_revalidate(struct ata_device *dev, unsigned int new_class, /* verify n_sectors hasn't changed */ if (dev->class != ATA_DEV_ATA || !n_sectors || - dev->n_sectors == n_sectors) + dev->n_sectors == n_sectors || ata_id_is_locked(dev->id)) return 0; /* n_sectors has changed */ @@ -4280,6 +4313,9 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { /* Apacer models with LPM issues */ { "Apacer AS340*", NULL, ATA_QUIRK_NOLPM }, + /* PNY CS900 (Phison PS3111-S11, DRAM-less) drops the link on DIPM */ + { "PNY CS900 1TB SSD", NULL, ATA_QUIRK_NOLPM }, + /* Silicon Motion models with LPM issues */ { "MD619HXCLDE3TC", "TCVAID", ATA_QUIRK_NOLPM }, { "MD619GXCLDE3TC", "TCV35D", ATA_QUIRK_NOLPM }, diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index d43207c6e4679..41b3b47327764 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -37,8 +37,6 @@ #include "libata.h" #include "libata-transport.h" -#define ATA_SCSI_RBUF_SIZE 2048 - static DEFINE_SPINLOCK(ata_scsi_rbuf_lock); static u8 ata_scsi_rbuf[ATA_SCSI_RBUF_SIZE]; @@ -1931,8 +1929,13 @@ static void ata_scsi_rbuf_fill(struct ata_device *dev, struct scsi_cmnd *cmd, memset(ata_scsi_rbuf, 0, ATA_SCSI_RBUF_SIZE); len = actor(dev, cmd, ata_scsi_rbuf); if (len) { + if (WARN_ON(len > ATA_SCSI_RBUF_SIZE)) { + ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND, 0, 0); + spin_unlock_irqrestore(&ata_scsi_rbuf_lock, flags); + return; + } sg_copy_from_buffer(scsi_sglist(cmd), scsi_sg_count(cmd), - ata_scsi_rbuf, ATA_SCSI_RBUF_SIZE); + ata_scsi_rbuf, len); cmd->result = SAM_STAT_GOOD; if (scsi_bufflen(cmd) > len) scsi_set_resid(cmd, scsi_bufflen(cmd) - len); diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index b5423b6e97de1..329e9c5776f0d 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -148,6 +148,15 @@ static inline bool ata_acpi_dev_manage_restart(struct ata_device *dev) { return #endif /* libata-scsi.c */ +#define ATA_SCSI_RBUF_SIZE 2048 + +/* + * Maximum number of concurrent positioning ranges (CPR) supported. The ACS + * specifications allow up to 255, but we limit this to the number of CPR + * descriptors that fit in the rbuf buffer used to emit VPD page B9h. + */ +#define ATA_DEV_MAX_CPR min(255, ((ATA_SCSI_RBUF_SIZE - 64) / 32)) + extern struct ata_device *ata_scsi_find_dev(struct ata_port *ap, const struct scsi_device *scsidev); extern int ata_scsi_add_hosts(struct ata_host *host, diff --git a/drivers/ata/pata_pxa.c b/drivers/ata/pata_pxa.c index 03dbaf4a13a75..9f63bdfb8576e 100644 --- a/drivers/ata/pata_pxa.c +++ b/drivers/ata/pata_pxa.c @@ -286,6 +286,7 @@ static int pxa_ata_probe(struct platform_device *pdev) ret = dmaengine_slave_config(data->dma_chan, &config); if (ret < 0) { dev_err(&pdev->dev, "dma configuration failed: %d\n", ret); + dma_release_channel(data->dma_chan); return ret; } diff --git a/drivers/ata/sata_gemini.c b/drivers/ata/sata_gemini.c index 530ee26b30121..56ae2820df588 100644 --- a/drivers/ata/sata_gemini.c +++ b/drivers/ata/sata_gemini.c @@ -353,7 +353,7 @@ static int gemini_sata_probe(struct platform_device *pdev) if (sg->ide_pins) { ret = gemini_setup_ide_pins(dev); if (ret) - return ret; + goto out_unprep_clk; } dev_info(dev, "set up the Gemini IDE/SATA nexus\n"); diff --git a/drivers/base/bus.c b/drivers/base/bus.c index 8b6722ff8590d..d17bd91490ee4 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -544,10 +544,10 @@ static const struct attribute_group driver_override_dev_group = { */ int bus_add_device(struct device *dev) { - struct subsys_private *sp = bus_to_subsys(dev->bus); + struct subsys_private *sp; int error; - if (!sp) { + if (!dev->bus) { /* * This is a normal operation for many devices that do not * have a bus assigned to them, just say that all went @@ -556,6 +556,13 @@ int bus_add_device(struct device *dev) return 0; } + sp = bus_to_subsys(dev->bus); + if (!sp) { + pr_err("%s: cannot add device '%s' to unregistered bus '%s'\n", + __func__, dev_name(dev), dev->bus->name); + return -EINVAL; + } + /* * Reference in sp is now incremented and will be dropped when * the device is removed from the bus diff --git a/drivers/base/core.c b/drivers/base/core.c index bd2ddf2aab505..478aa3fbf1e83 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -235,6 +235,79 @@ static void __fw_devlink_pickup_dangling_consumers(struct fwnode_handle *fwnode, __fw_devlink_pickup_dangling_consumers(child, new_sup); } +static void fw_devlink_pickup_dangling_consumers(struct device *dev) +{ + struct fwnode_handle *child; + + guard(mutex)(&fwnode_link_lock); + + fwnode_for_each_available_child_node(dev->fwnode, child) + __fw_devlink_pickup_dangling_consumers(child, dev->fwnode); + __fw_devlink_link_to_consumers(dev); +} + +/** + * fw_devlink_refresh_fwnode - Recheck the tree under this firmware node + * @fwnode: The fwnode under which the fwnode tree has changed + * + * This function is mainly meant to adjust the supplier/consumer dependencies + * after a fwnode tree overlay has occurred. + */ +void fw_devlink_refresh_fwnode(struct fwnode_handle *fwnode) +{ + struct device *dev; + + /* + * Find the closest ancestor fwnode that has been converted to a device + * that can bind to a driver (bus device). + */ + fwnode_handle_get(fwnode); + do { + if (fwnode_test_flag(fwnode, FWNODE_FLAG_NOT_DEVICE)) + continue; + + dev = get_dev_from_fwnode(fwnode); + if (!dev) + continue; + + if (dev->bus) + break; + + put_device(dev); + } while ((fwnode = fwnode_get_next_parent(fwnode))); + + /* + * If none of the ancestor fwnodes have (yet) been converted to a device + * that can bind to a driver, there's nothing to fix up. + */ + if (!fwnode) + return; + + WARN(device_is_bound(dev) && dev->links.status != DL_DEV_DRIVER_BOUND, + "Don't multithread overlaying and probing the same device!\n"); + + /* + * If the device has already bound to a driver, then we need to redo + * some of the work that was done after the device was bound to a + * driver. If the device hasn't bound to a driver, running things too + * soon would incorrectly pick up consumers that it shouldn't. + */ + if (dev->links.status == DL_DEV_DRIVER_BOUND) { + fw_devlink_pickup_dangling_consumers(dev); + /* + * Some of dangling consumers could have been put previously in + * the deferred probe list due to the unavailability of their + * suppliers. Those consumers have been picked up and some of + * their suppliers links have been updated. Time to re-try their + * probe sequence. + */ + driver_deferred_probe_trigger(); + } + + put_device(dev); + fwnode_handle_put(fwnode); +} + static DEFINE_MUTEX(device_links_lock); DEFINE_STATIC_SRCU(device_links_srcu); @@ -1312,16 +1385,8 @@ void device_links_driver_bound(struct device *dev) * child firmware node. */ if (dev->fwnode && dev->fwnode->dev == dev) { - struct fwnode_handle *child; - fwnode_links_purge_suppliers(dev->fwnode); - - guard(mutex)(&fwnode_link_lock); - - fwnode_for_each_available_child_node(dev->fwnode, child) - __fw_devlink_pickup_dangling_consumers(child, - dev->fwnode); - __fw_devlink_link_to_consumers(dev); + fw_devlink_pickup_dangling_consumers(dev); } device_remove_file(dev, &dev_attr_waiting_for_supplier); diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 1dc1e3528043c..51c7132e98a07 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -323,12 +323,11 @@ void deferred_probe_extend_timeout(void) * If the work hasn't been queued yet or if the work expired, don't * start a new one. */ - if (cancel_delayed_work(&deferred_probe_timeout_work)) { - schedule_delayed_work(&deferred_probe_timeout_work, - driver_deferred_probe_timeout * HZ); + if (delayed_work_pending(&deferred_probe_timeout_work) && + mod_delayed_work(system_percpu_wq, &deferred_probe_timeout_work, + secs_to_jiffies(driver_deferred_probe_timeout))) pr_debug("Extended deferred probe timeout by %d secs\n", driver_deferred_probe_timeout); - } } /** diff --git a/drivers/base/faux.c b/drivers/base/faux.c index fb3e42f213624..3d1d1eafb4733 100644 --- a/drivers/base/faux.c +++ b/drivers/base/faux.c @@ -133,6 +133,9 @@ struct faux_device *faux_device_create_with_groups(const char *name, struct device *dev; int ret; + if (!faux_bus_root) + return NULL; + faux_obj = kzalloc_obj(*faux_obj); if (!faux_obj) return NULL; @@ -232,19 +235,12 @@ EXPORT_SYMBOL_GPL(faux_device_destroy); int __init faux_bus_init(void) { + struct device *root; int ret; - faux_bus_root = kzalloc_obj(*faux_bus_root); - if (!faux_bus_root) - return -ENOMEM; - - dev_set_name(faux_bus_root, "faux"); - - ret = device_register(faux_bus_root); - if (ret) { - put_device(faux_bus_root); - return ret; - } + root = root_device_register("faux"); + if (IS_ERR(root)) + return PTR_ERR(root); ret = bus_register(&faux_bus_type); if (ret) @@ -254,12 +250,14 @@ int __init faux_bus_init(void) if (ret) goto error_driver; + faux_bus_root = root; + return ret; error_driver: bus_unregister(&faux_bus_type); error_bus: - device_unregister(faux_bus_root); + root_device_unregister(root); return ret; } diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index a11b30dda23be..24213a0ea8317 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -1132,6 +1132,7 @@ EXPORT_SYMBOL(release_firmware); /* Async support */ struct firmware_work { struct work_struct work; + struct list_head list; struct module *module; const char *name; struct device *device; @@ -1140,6 +1141,17 @@ struct firmware_work { u32 opt_flags; }; +static LIST_HEAD(firmware_work_list); +static DEFINE_SPINLOCK(firmware_work_lock); + +static void firmware_work_free(struct firmware_work *fw_work) +{ + put_device(fw_work->device); /* taken in request_firmware_nowait() */ + module_put(fw_work->module); + kfree_const(fw_work->name); + kfree(fw_work); +} + static void request_firmware_work_func(struct work_struct *work) { struct firmware_work *fw_work; @@ -1150,11 +1162,15 @@ static void request_firmware_work_func(struct work_struct *work) _request_firmware(&fw, fw_work->name, fw_work->device, NULL, 0, 0, fw_work->opt_flags); fw_work->cont(fw, fw_work->context); - put_device(fw_work->device); /* taken in request_firmware_nowait() */ - module_put(fw_work->module); - kfree_const(fw_work->name); - kfree(fw_work); + spin_lock_irq(&firmware_work_lock); + if (!list_empty(&fw_work->list)) { + list_del_init(&fw_work->list); + spin_unlock_irq(&firmware_work_lock); + firmware_work_free(fw_work); + return; + } + spin_unlock_irq(&firmware_work_lock); } @@ -1164,6 +1180,7 @@ static int _request_firmware_nowait( void (*cont)(const struct firmware *fw, void *context), bool nowarn) { struct firmware_work *fw_work; + unsigned long flags; fw_work = kzalloc_obj(struct firmware_work, gfp); if (!fw_work) @@ -1196,7 +1213,12 @@ static int _request_firmware_nowait( get_device(fw_work->device); INIT_WORK(&fw_work->work, request_firmware_work_func); + + spin_lock_irqsave(&firmware_work_lock, flags); + list_add_tail(&fw_work->list, &firmware_work_list); schedule_work(&fw_work->work); + spin_unlock_irqrestore(&firmware_work_lock, flags); + return 0; } @@ -1259,6 +1281,44 @@ int firmware_request_nowait_nowarn( } EXPORT_SYMBOL_GPL(firmware_request_nowait_nowarn); +/** + * request_firmware_nowait_cancel() - cancel an async firmware request + * @device: device for which the firmware is being loaded + * @context: context passed to request_firmware_nowait() + * @cont: callback passed to request_firmware_nowait() + * + * Cancel a pending request_firmware_nowait() request for @device, @context + * and @cont. If the associated work has already started, this function waits + * until the callback has returned. If the callback has already completed, this + * function does nothing. + * + * This function may sleep. + */ +void request_firmware_nowait_cancel(struct device *device, void *context, + void (*cont)(const struct firmware *fw, + void *context)) +{ + struct firmware_work *fw_work = NULL; + struct firmware_work *tmp; + + spin_lock_irq(&firmware_work_lock); + list_for_each_entry_reverse(tmp, &firmware_work_list, list) { + if (tmp->device == device && tmp->context == context && + tmp->cont == cont) { + fw_work = tmp; + list_del_init(&fw_work->list); + break; + } + } + spin_unlock_irq(&firmware_work_lock); + + if (!fw_work) + return; + cancel_work_sync(&fw_work->work); + firmware_work_free(fw_work); +} +EXPORT_SYMBOL_GPL(request_firmware_nowait_cancel); + #ifdef CONFIG_FW_CACHE static ASYNC_DOMAIN_EXCLUSIVE(fw_cache_domain); @@ -1503,9 +1563,10 @@ static void device_cache_fw_images(void) mutex_lock(&fw_lock); fwc->state = FW_LOADER_START_CACHE; - dpm_for_each_dev(NULL, dev_cache_fw_image); mutex_unlock(&fw_lock); + dpm_for_each_dev(NULL, dev_cache_fw_image); + /* wait for completion of caching firmware for all devices */ async_synchronize_full_domain(&fw_cache_domain); diff --git a/drivers/base/firmware_loader/sysfs_upload.c b/drivers/base/firmware_loader/sysfs_upload.c index f59a7856934ce..efc33294212fd 100644 --- a/drivers/base/firmware_loader/sysfs_upload.c +++ b/drivers/base/firmware_loader/sysfs_upload.c @@ -343,7 +343,6 @@ firmware_upload_register(struct module *module, struct device *parent, goto free_fw_upload_priv; } fw_upload->priv = fw_sysfs; - fw_sysfs->fw_upload_priv = fw_upload_priv; fw_dev = &fw_sysfs->dev; ret = alloc_lookup_fw_priv(name, &fw_cache, &fw_priv, NULL, 0, 0, @@ -351,10 +350,12 @@ firmware_upload_register(struct module *module, struct device *parent, if (ret != 0) { if (ret > 0) ret = -EINVAL; - goto free_fw_sysfs; + put_device(fw_dev); + goto free_fw_upload_priv; } fw_priv->is_paged_buf = true; fw_sysfs->fw_priv = fw_priv; + fw_sysfs->fw_upload_priv = fw_upload_priv; ret = device_add(fw_dev); if (ret) { @@ -365,9 +366,6 @@ firmware_upload_register(struct module *module, struct device *parent, return fw_upload; -free_fw_sysfs: - kfree(fw_sysfs); - free_fw_upload_priv: kfree(fw_upload_priv); diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 6981b55d582ac..843f85d3e3f9b 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -807,7 +807,6 @@ static int add_memory_block(unsigned long block_id, int nid, unsigned long state mem->start_section_nr = block_id * sections_per_block; mem->state = state; mem->nid = nid; - mem->altmap = altmap; INIT_LIST_HEAD(&mem->group_next); #ifndef CONFIG_NUMA @@ -825,6 +824,8 @@ static int add_memory_block(unsigned long block_id, int nid, unsigned long state if (ret) return ret; + mem->altmap = altmap; + if (group) { mem->group = group; list_add(&mem->group_next, &group->memory_blocks); diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index e1b550664babb..ed48c292f5757 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -115,7 +115,7 @@ void device_pm_sleep_init(struct device *dev) dev->power.is_noirq_suspended = false; dev->power.is_late_suspended = false; init_completion(&dev->power.completion); - complete_all(&dev->power.completion); + complete(&dev->power.completion); dev->power.wakeup = NULL; INIT_LIST_HEAD(&dev->power.entry); } @@ -252,6 +252,10 @@ static void dpm_wait(struct device *dev, bool async) if (!dev) return; + /* Devices with no PM support don't use the completion. */ + if (dev->power.no_pm) + return; + if (async || (pm_async_enabled && dev->power.async_suspend)) wait_for_completion(&dev->power.completion); } diff --git a/drivers/base/property.c b/drivers/base/property.c index 8e0148a37fffb..e08eadd66f4f9 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -1277,8 +1277,10 @@ fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode, if (fwnode_ep.port != port) continue; - if (fwnode_ep.id == endpoint) + if (fwnode_ep.id == endpoint) { + fwnode_handle_put(best_ep); return ep; + } if (!endpoint_next) continue; diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index 27616b05111cc..aa8f2efed7798 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -245,7 +245,7 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) if (map->cache_ops->exit) { dev_dbg(map->dev, "Destroying %s cache\n", map->cache_ops->name); map->lock(map->lock_arg); - ret = map->cache_ops->exit(map); + map->cache_ops->exit(map); map->unlock(map->lock_arg); } err_free_reg_defaults: diff --git a/drivers/base/regmap/regmap-i2c.c b/drivers/base/regmap/regmap-i2c.c index 31e30dfced192..51a04961faf7b 100644 --- a/drivers/base/regmap/regmap-i2c.c +++ b/drivers/base/regmap/regmap-i2c.c @@ -337,7 +337,7 @@ static int regmap_smbus_word_write_reg16(void *context, const void *data, val = ((u8 *)data)[2]; return i2c_smbus_write_word_data(i2c, addr_hi, - cpu_to_le16(((u16)val << 8) | addr_lo)); + ((u16)val << 8) | addr_lo); } static const struct regmap_bus regmap_smbus_byte_word_reg16 = { diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 58b95bf4bdca6..2135c14354a85 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -1810,6 +1810,11 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req data_size -= digest_size; } + if (data_size < 0) { + drbd_err(peer_device, "Invalid data reply size\n"); + return -EIO; + } + /* optimistically update recv_cnt. if receiving fails below, * we disconnect anyways, and counters will be reset. */ peer_device->device->recv_cnt += data_size>>9; diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index c5d45cf91f882..fc1b37044a9b0 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -502,7 +502,7 @@ config BT_NXPUART config BT_INTEL_PCIE tristate "Intel HCI PCIe driver" - depends on PCI + depends on PCI && ACPI select BT_INTEL select FW_LOADER help diff --git a/drivers/bluetooth/bpa10x.c b/drivers/bluetooth/bpa10x.c index 2ae38a321c4b3..e63d1af250ec9 100644 --- a/drivers/bluetooth/bpa10x.c +++ b/drivers/bluetooth/bpa10x.c @@ -255,9 +255,13 @@ static int bpa10x_setup(struct hci_dev *hdev) if (IS_ERR(skb)) return PTR_ERR(skb); - bt_dev_info(hdev, "%s", (char *)(skb->data + 1)); + /* Bounded print: the device controls skb->len. */ + if (skb->len > 1) { + int len = skb->len - 1; - hci_set_fw_info(hdev, "%s", skb->data + 1); + bt_dev_info(hdev, "%.*s", len, (char *)(skb->data + 1)); + hci_set_fw_info(hdev, "%.*s", len, skb->data + 1); + } kfree_skb(skb); return 0; diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index dcaaa4ca02b99..5e9cac090bd8f 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -67,9 +67,10 @@ static struct { u32 fw_build_num; } coredump_info; -static const guid_t btintel_guid_dsm = +const guid_t btintel_guid_dsm = GUID_INIT(0xaa10f4e0, 0x81ac, 0x4233, 0xab, 0xf6, 0x3b, 0x2a, 0xc5, 0x0e, 0x28, 0xd9); +EXPORT_SYMBOL_GPL(btintel_guid_dsm); int btintel_check_bdaddr(struct hci_dev *hdev) { @@ -2624,7 +2625,7 @@ static void btintel_set_ppag(struct hci_dev *hdev, struct intel_version_tlv *ver kfree_skb(skb); } -static int btintel_acpi_reset_method(struct hci_dev *hdev) +int btintel_acpi_reset_method(struct hci_dev *hdev) { int ret = 0; acpi_status status; @@ -2632,14 +2633,14 @@ static int btintel_acpi_reset_method(struct hci_dev *hdev) struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; status = acpi_evaluate_object(ACPI_HANDLE(GET_HCIDEV_DEV(hdev)), "_PRR", NULL, &buffer); - if (ACPI_FAILURE(status)) { + if (ACPI_FAILURE(status) || !buffer.pointer) { bt_dev_err(hdev, "Failed to run _PRR method"); ret = -ENODEV; return ret; } p = buffer.pointer; - if (p->package.count != 1 || p->type != ACPI_TYPE_PACKAGE) { + if (p->type != ACPI_TYPE_PACKAGE || p->package.count != 1) { bt_dev_err(hdev, "Invalid arguments"); ret = -EINVAL; goto exit_on_error; @@ -2663,6 +2664,7 @@ static int btintel_acpi_reset_method(struct hci_dev *hdev) kfree(buffer.pointer); return ret; } +EXPORT_SYMBOL_GPL(btintel_acpi_reset_method); static void btintel_set_dsm_reset_method(struct hci_dev *hdev, struct intel_version_tlv *ver_tlv) diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h index 0e9ca99aaaaea..37d93abdd5a37 100644 --- a/drivers/bluetooth/btintel.h +++ b/drivers/bluetooth/btintel.h @@ -53,6 +53,7 @@ struct intel_tlv { } __packed; #define BTINTEL_HCI_OP_RESET 0xfc01 +#define BTINTEL_HCI_OP_DEBUG 0xfcd9 #define BTINTEL_CNVI_BLAZARI 0x900 /* BlazarI - Lunar Lake */ #define BTINTEL_CNVI_BLAZARIW 0x901 /* BlazarIW - Wildcat Lake */ @@ -77,8 +78,11 @@ struct intel_tlv { #define BTINTEL_HWID_BZRU 0x1d /* BlazarU - Meteor Lake */ #define BTINTEL_HWID_SCP 0x1f /* Scorpius Peak - Panther Lake */ #define BTINTEL_HWID_SCP2 0x20 /* Scorpius Peak2 - Nova Lake */ +#define BTINTEL_HWID_SCP2F 0x21 /* Scorpius Peak2-F - Nova Lake */ #define BTINTEL_HWID_BZRIW 0x22 /* BlazarIW - Wildcat Lake */ +extern const guid_t btintel_guid_dsm; + struct intel_version_tlv { u32 cnvi_top; u32 cnvr_top; @@ -289,6 +293,7 @@ int btintel_bootloader_setup_tlv(struct hci_dev *hdev, int btintel_shutdown_combined(struct hci_dev *hdev); void btintel_hw_error(struct hci_dev *hdev, u8 code); void btintel_print_fseq_info(struct hci_dev *hdev); +int btintel_acpi_reset_method(struct hci_dev *hdev); #else static inline int btintel_check_bdaddr(struct hci_dev *hdev) @@ -422,4 +427,8 @@ static inline void btintel_hw_error(struct hci_dev *hdev, u8 code) static inline void btintel_print_fseq_info(struct hci_dev *hdev) { } +static inline int btintel_acpi_reset_method(struct hci_dev *hdev) +{ + return -ENODEV; +} #endif diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index 37e0507636330..be9348960348f 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -102,6 +103,22 @@ enum { BTINTEL_PCIE_D3 }; +enum { + BTINTEL_PCIE_DSM_SET_RESET_TIMING = 1, + BTINTEL_PCIE_DSM_GET_RESET_TIMING = 2, + BTINTEL_PCIE_DSM_BT_PLDR_CONFIG = 3, + BTINTEL_PCIE_DSM_GET_RESET_TYPE = 4, + BTINTEL_PCIE_DSM_DYNAMIC_PLDR = 5, + BTINTEL_PCIE_DSM_GET_RESET_METHOD = 6, + BTINTEL_PCIE_DSM_SET_PLDR_DELAY = 7, +}; + +enum btintel_dsm_internal_product_reset_mode { + BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET = BIT(0), + BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR = BIT(1), + BTINTEL_PCIE_DSM_PLDR_MODE_EN_BT_OFF_ON = BIT(2), +}; + /* Structure for dbgc fragment buffer * @buf_addr_lsb: LSB of the buffer's physical address * @buf_addr_msb: MSB of the buffer's physical address @@ -128,10 +145,21 @@ struct btintel_pcie_dbgc_ctxt { struct btintel_pcie_dbgc_ctxt_buf bufs[BTINTEL_PCIE_DBGC_BUFFER_COUNT]; }; -struct btintel_pcie_removal { - struct pci_dev *pdev; - struct work_struct work; -}; +struct btintel_pcie_trigger_evt { + u8 type; + u8 len; + __le32 addr; + __le32 size; +} __packed; + +struct btintel_pcie_fwtrigger_evt { + __le32 reserved; + u8 type; /* Debug Trigger event */ + __le16 len; + u8 event_type; + __le16 event_id; + __le16 reserved2; +} __packed; static LIST_HEAD(btintel_pcie_recovery_list); static DEFINE_SPINLOCK(btintel_pcie_recovery_lock); @@ -626,9 +654,10 @@ static void *btintel_pcie_copy_tlv(void *dest, enum btintel_pcie_tlv_type type, static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data) { u32 offset, prev_size, wr_ptr_status, dump_size, data_len; + u32 status_reg, wrap_reg; struct btintel_pcie_dbgc *dbgc = &data->dbgc; struct hci_dev *hdev = data->hdev; - u8 *pdata, *p, buf_idx; + u8 *pdata, *p, buf_idx, hw_variant; struct intel_tlv *tlv; struct timespec64 now; struct tm tm_now; @@ -641,7 +670,28 @@ static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data) return -EOPNOTSUPP; - wr_ptr_status = btintel_pcie_rd_dev_mem(data, BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS); + hw_variant = INTEL_HW_VARIANT(data->cnvi); + switch (hw_variant) { + case BTINTEL_HWID_BZRI: + case BTINTEL_HWID_BZRIW: + status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS; + wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND; + break; + case BTINTEL_HWID_SCP: + case BTINTEL_HWID_SCP2: + case BTINTEL_HWID_SCP2F: + status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS_SCP; + wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND_SCP; + break; + default: + bt_dev_err(hdev, "Unsupported Intel hardware variant (0x%2.2x)", + hw_variant); + return -EINVAL; + } + + wr_ptr_status = btintel_pcie_rd_dev_mem(data, status_reg); + data->dmp_hdr.wrap_ctr = btintel_pcie_rd_dev_mem(data, wrap_reg); + offset = wr_ptr_status & BTINTEL_PCIE_DBG_OFFSET_BIT_MASK; buf_idx = BTINTEL_PCIE_DBGC_DBG_BUF_IDX(wr_ptr_status); @@ -684,6 +734,11 @@ static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data) sizeof(*tlv) + strlen(vendor) + sizeof(*tlv) + strlen(driver); + if (data->dmp_hdr.event_type && data->dmp_hdr.event_id) { + data_len += sizeof(*tlv) + sizeof(data->dmp_hdr.event_type); + data_len += sizeof(*tlv) + sizeof(data->dmp_hdr.event_id); + } + /* * sizeof(u32) - signature * sizeof(data_len) - to store tlv data size @@ -718,10 +773,6 @@ static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data) sizeof(data->dmp_hdr.write_ptr)); p = btintel_pcie_copy_tlv(p, BTINTEL_WRAP_CTR, &data->dmp_hdr.wrap_ctr, sizeof(data->dmp_hdr.wrap_ctr)); - - data->dmp_hdr.wrap_ctr = btintel_pcie_rd_dev_mem(data, - BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND); - p = btintel_pcie_copy_tlv(p, BTINTEL_TRIGGER_REASON, &data->dmp_hdr.trigger_reason, sizeof(data->dmp_hdr.trigger_reason)); p = btintel_pcie_copy_tlv(p, BTINTEL_FW_SHA, &data->dmp_hdr.fw_git_sha1, @@ -731,6 +782,17 @@ static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data) p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_TOP, &data->dmp_hdr.cnvi_top, sizeof(data->dmp_hdr.cnvi_top)); + if (data->dmp_hdr.event_type && data->dmp_hdr.event_id) { + p = btintel_pcie_copy_tlv(p, BTINTEL_EVENT_TYPE, + &data->dmp_hdr.event_type, + sizeof(data->dmp_hdr.event_type)); + p = btintel_pcie_copy_tlv(p, BTINTEL_EVENT_ID, + &data->dmp_hdr.event_id, + sizeof(data->dmp_hdr.event_id)); + data->dmp_hdr.event_type = 0; + data->dmp_hdr.event_id = 0; + } + memcpy(p, dbgc->bufs[0].data, dbgc->count * BTINTEL_PCIE_DBGC_BUFFER_SIZE); dev_coredumpv(&hdev->dev, pdata, dump_size, GFP_KERNEL); return 0; @@ -1318,6 +1380,73 @@ static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data) kfree(buf); } +static int btintel_pcie_dump_fwtrigger_event(struct btintel_pcie_data *data) +{ + struct btintel_pcie_fwtrigger_evt *evt; + struct sk_buff *skb; + unsigned int len; + int err; + u8 *buf; + + if (!data->debug_evt_size || !data->debug_evt_addr) + return -EINVAL; + + len = data->debug_evt_size; + + len = ALIGN_DOWN(len, 4); + + if (len < sizeof(*evt) || len > HCI_MAX_EVENT_SIZE) { + bt_dev_err(data->hdev, "Invalid FW trigger data size (%u bytes)", len); + return -EINVAL; + } + + buf = kzalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + btintel_pcie_mac_init(data); + + err = btintel_pcie_read_device_mem(data, buf, data->debug_evt_addr, + len); + if (err) + goto exit_on_error; + + evt = (void *)buf; + data->dmp_hdr.event_type = evt->event_type; + data->dmp_hdr.event_id = le16_to_cpu(evt->event_id); + + bt_dev_dbg(data->hdev, "event type: 0x%2.2x event id: 0x%4.4x len: %u", + data->dmp_hdr.event_type, data->dmp_hdr.event_id, len); + + skb = bt_skb_alloc(len, GFP_KERNEL); + if (!skb) { + err = -ENOMEM; + goto exit_on_error; + } + skb_put_data(skb, buf, len); + + hci_recv_diag(data->hdev, skb); + err = 0; + +exit_on_error: + kfree(buf); + return err; +} + +static void btintel_pcie_msix_fw_trigger_handler(struct btintel_pcie_data *data) +{ + bt_dev_dbg(data->hdev, "Received firmware smart trigger cause"); + + if (test_and_set_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags)) + return; + + /* Trigger device core dump when there is FW assert */ + if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) + data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT; + + queue_work(data->coredump_workqueue, &data->coredump_work); +} + static void btintel_pcie_msix_hw_exp_handler(struct btintel_pcie_data *data) { bt_dev_err(data->hdev, "Received hw exception interrupt"); @@ -1332,14 +1461,27 @@ static void btintel_pcie_msix_hw_exp_handler(struct btintel_pcie_data *data) if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT; - queue_work(data->workqueue, &data->rx_work); + queue_work(data->coredump_workqueue, &data->coredump_work); } -static void btintel_pcie_rx_work(struct work_struct *work) +static void btintel_pcie_coredump_worker(struct work_struct *work) { struct btintel_pcie_data *data = container_of(work, - struct btintel_pcie_data, rx_work); - struct sk_buff *skb; + struct btintel_pcie_data, coredump_work); + int err; + + /* hdev is NULL until setup_hdev() succeeds, and is cleared on + * teardown after disable_work_sync() drains us; bail in that case. + */ + if (!data->hdev) + return; + + if (test_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags)) { + err = btintel_pcie_dump_fwtrigger_event(data); + if (err) + bt_dev_warn(data->hdev, "failed to log fwtrigger event"); + clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags); + } if (test_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) { btintel_pcie_dump_traces(data->hdev); @@ -1357,6 +1499,13 @@ static void btintel_pcie_rx_work(struct work_struct *work) btintel_pcie_read_hwexp(data); clear_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags); } +} + +static void btintel_pcie_rx_work(struct work_struct *work) +{ + struct btintel_pcie_data *data = container_of(work, + struct btintel_pcie_data, rx_work); + struct sk_buff *skb; /* Process the sk_buf in queue and send to the HCI layer */ while ((skb = skb_dequeue(&data->rx_skb_q))) { @@ -1497,6 +1646,9 @@ static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id) btintel_pcie_msix_tx_handle(data); } + if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_FWTRIG) + btintel_pcie_msix_fw_trigger_handler(data); + /* This interrupt is triggered by the firmware after updating * boot_stage register and image_response register */ @@ -1571,6 +1723,7 @@ static struct btintel_pcie_causes_list causes_list[] = { { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x01 }, { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x20 }, { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x23 }, + { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_FWTRIG, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x25 }, }; /* This function configures the interrupt masks for both HW_INT_CAUSES and @@ -1961,6 +2114,9 @@ static int btintel_pcie_send_frame(struct hci_dev *hdev, if (test_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags)) return -ENODEV; + if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags)) + return -ENODEV; + /* Due to the fw limitation, the type header of the packet should be * 4 bytes unlike 1 byte for UART. In UART, the firmware can read * the first byte to get the packet type and redirect the rest of data @@ -2027,9 +2183,11 @@ static int btintel_pcie_send_frame(struct hci_dev *hdev, static void btintel_pcie_release_hdev(struct btintel_pcie_data *data) { - struct hci_dev *hdev; + struct hci_dev *hdev = data->hdev; + + if (!hdev) + return; - hdev = data->hdev; hci_unregister_dev(hdev); hci_free_dev(hdev); data->hdev = NULL; @@ -2057,6 +2215,55 @@ static void btintel_pcie_synchronize_irqs(struct btintel_pcie_data *data) synchronize_irq(data->msix_entries[i].vector); } +static int btintel_pcie_get_debug_info_addr(struct hci_dev *hdev) +{ + struct btintel_pcie_data *data = hci_get_drvdata(hdev); + struct btintel_pcie_trigger_evt *evt; + u8 param[1] = {0x10}; + struct sk_buff *skb; + int err = 0; + + skb = __hci_cmd_sync(hdev, BTINTEL_HCI_OP_DEBUG, 1, param, + HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) { + bt_dev_err(hdev, "Reading Intel read debug info address command failed (%ld)", + PTR_ERR(skb)); + /* Not all Intel products supports this command */ + if (PTR_ERR(skb) == -EOPNOTSUPP) + return 0; + return PTR_ERR(skb); + } + + if (skb->len < (1 + sizeof(*evt))) { + bt_dev_err(hdev, "Debug info response too short (%u bytes)", skb->len); + err = -EIO; + goto exit_error; + } + + /* Check the status */ + if (skb->data[0]) { + bt_dev_err(hdev, "Reading Intel read debug info command failed (0x%2.2x)", + skb->data[0]); + err = -EIO; + goto exit_error; + } + + /* Consume Command Complete Status field */ + skb_pull(skb, 1); + + evt = (void *)skb->data; + + data->debug_evt_addr = le32_to_cpu(evt->addr); + data->debug_evt_size = le32_to_cpu(evt->size); + + bt_dev_dbg(hdev, "config type: %u config len: %u debug event addr: 0x%8.8x size: 0x%8.8x", + evt->type, evt->len, data->debug_evt_addr, + data->debug_evt_size); +exit_error: + kfree_skb(skb); + return err; +} + static int btintel_pcie_setup_internal(struct hci_dev *hdev) { struct btintel_pcie_data *data = hci_get_drvdata(hdev); @@ -2156,6 +2363,10 @@ static int btintel_pcie_setup_internal(struct hci_dev *hdev) if (ver_tlv.img_type == 0x02 || ver_tlv.img_type == 0x03) data->dmp_hdr.fw_git_sha1 = ver_tlv.git_sha1; + err = btintel_pcie_get_debug_info_addr(hdev); + if (err) + goto exit_error; + btintel_print_fseq_info(hdev); exit_error: kfree_skb(skb); @@ -2264,65 +2475,202 @@ static void btintel_pcie_inc_recovery_count(struct pci_dev *pdev, } } -static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data); +static void btintel_pcie_reset(struct hci_dev *hdev); -static void btintel_pcie_removal_work(struct work_struct *wk) +static int btintel_pcie_acpi_reset_method(struct btintel_pcie_data *data) { - struct btintel_pcie_removal *removal = - container_of(wk, struct btintel_pcie_removal, work); - struct pci_dev *pdev = removal->pdev; - struct btintel_pcie_data *data; + union acpi_object *obj, argv4; + acpi_handle handle; + int ret; + struct pldr_mode { + __le16 cmd_type; + __le16 cmd_payload; + } __packed; + + /* set 1 for _PRR mode + * Product Reset (PLDR Abort flow) + */ + static const struct pldr_mode mode = { + .cmd_type = cpu_to_le16(1), + .cmd_payload = cpu_to_le16(BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET | + BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR), + }; + struct hci_dev *hdev = data->hdev; + + handle = ACPI_HANDLE(GET_HCIDEV_DEV(data->hdev)); + if (!handle) { + bt_dev_err(data->hdev, "No support for bluetooth device in ACPI firmware"); + return -EACCES; + } + + if (!acpi_has_method(handle, "_PRR")) { + bt_dev_err(data->hdev, "No support for _PRR ACPI method, cold boot"); + return -ENODEV; + } + + argv4.buffer.type = ACPI_TYPE_BUFFER; + argv4.buffer.length = sizeof(mode); + argv4.buffer.pointer = (void *)&mode; + + obj = acpi_evaluate_dsm(handle, &btintel_guid_dsm, 0, + BTINTEL_PCIE_DSM_DYNAMIC_PLDR, &argv4); + if (!obj) { + bt_dev_err(data->hdev, "Failed to call dsm to set reset method"); + return -EIO; + } + ACPI_FREE(obj); + + pci_dev_lock(data->pdev); + pci_save_state(data->pdev); + ret = btintel_acpi_reset_method(hdev); + if (ret) + bt_dev_err(data->hdev, "ACPI _PRR reset failed (%d), PLDR incomplete", + ret); + pci_restore_state(data->pdev); + pci_dev_unlock(data->pdev); + return ret; +} + +static void btintel_pcie_perform_pldr(struct btintel_pcie_data *data) +{ + struct pci_dev *pdev = data->pdev; + struct pci_dev *wifi = NULL; + struct pci_bus *bus; + int ret; + /* on integrated we have to look up by ID (same bus) */ + static const struct pci_device_id wifi_device_ids[] = { + #define WIFI_DEV(_id) { PCI_DEVICE(PCI_VENDOR_ID_INTEL, _id) } + WIFI_DEV(0xA840), /* LNL */ + WIFI_DEV(0xE440), /* PTL-P */ + WIFI_DEV(0xE340), /* PTL-H */ + WIFI_DEV(0xD340), /* NVL-H */ + WIFI_DEV(0x6E70), /* NVL-S */ + WIFI_DEV(0x4D40), /* WCL */ + {} + }; + struct pci_dev *tmp = NULL; + + bus = pdev->bus; + if (!bus) + return; + + list_for_each_entry(tmp, &bus->devices, bus_list) { + if (pci_match_id(wifi_device_ids, tmp)) { + wifi = pci_dev_get(tmp); + break; + } + } + + if (wifi) + device_release_driver(&wifi->dev); + + /* Wi-Fi is fully unbound before the reset and fully reprobed after + * the normal PCI probe path handles all state setup from scratch. + * BT needs pci_save_state()/pci_restore_state() because the BT driver + * is still partially attached when the _PRR runs (it hasn't been unbound yet). + * The PCI device needs to remain minimally functional so that + * device_reprobe(&pdev->dev) can work afterward + */ + ret = btintel_pcie_acpi_reset_method(data); + + if (wifi) { + if (device_reprobe(&wifi->dev)) + BT_ERR("WiFi reprobe failed for BDF:%s", pci_name(wifi)); + pci_dev_put(wifi); + } + + if (!ret) { + if (device_reprobe(&pdev->dev)) + BT_ERR("BT reprobe failed for BDF:%s", pci_name(pdev)); + } +} + +/* + * Issue a Function Level Reset and hand teardown/re-init off to the PCI + * core via device_reprobe(), mirroring the PLDR path's contract. + * + * Caller must hold pci_lock_rescan_remove() and must have already + * disabled interrupts and drained both rx_work and coredump_work. + */ +static int btintel_pcie_perform_flr(struct btintel_pcie_data *data) +{ + struct pci_dev *pdev = data->pdev; int err; + /* pci_try_reset_function() avoids the device_lock ABBA against + * btintel_pcie_remove(): .remove() runs with device_lock held and + * then waits for this work via disable_work_sync(); the blocking + * pci_reset_function() would deadlock by trying to re-acquire + * device_lock here. + */ + err = pci_try_reset_function(pdev); + if (err) { + BT_ERR("Failed resetting the pcie device (%d)", err); + return err; + } + + /* device_reprobe() always detaches the driver first (running + * .remove(), which frees 'data'); any re-probe failure leaves the + * device unbound but 'data' is already gone, so just log it. + */ + if (device_reprobe(&pdev->dev)) + BT_ERR("BT reprobe failed for BDF:%s", pci_name(pdev)); + + return 0; +} + +static void btintel_pcie_reset_work(struct work_struct *wk) +{ + struct btintel_pcie_data *data = + container_of(wk, struct btintel_pcie_data, reset_work); + struct pci_dev *pdev = data->pdev; + pci_lock_rescan_remove(); if (!pdev->bus) - goto error; + goto out; - data = pci_get_drvdata(pdev); + if (!data) + goto out; btintel_pcie_disable_interrupts(data); btintel_pcie_synchronize_irqs(data); flush_work(&data->rx_work); + /* Drain any in-flight coredump and block new ones across reset. + * Safe from self-deadlock: coredump_work runs on a separate wq. + */ + disable_work_sync(&data->coredump_work); bt_dev_dbg(data->hdev, "Release bluetooth interface"); - btintel_pcie_release_hdev(data); - - err = pci_reset_function(pdev); - if (err) { - BT_ERR("Failed resetting the pcie device (%d)", err); - goto error; - } - - btintel_pcie_enable_interrupts(data); - btintel_pcie_config_msix(data); - err = btintel_pcie_enable_bt(data); - if (err) { - BT_ERR("Failed to enable bluetooth hardware after reset (%d)", - err); - goto error; + /* Both reset paths follow the same contract: on success they + * destroy 'data' via device_reprobe() (a fresh probe re-INIT_WORKs + * the coredump_work with disable count 0), so enable_work() must + * NOT be called on the success path. Only the FLR path can fail + * with 'data' still alive, in which case we balance the + * disable_work_sync() above so a later successful reset is not + * permanently blocked. + * + * pci_lock_rescan_remove() (held above) serializes against PCI + * device addition/removal (hotplug), so no device can be added to + * or removed from the bus list while this code runs. + */ + if (data->reset_type == BTINTEL_PCIE_IOSF_PRR_PLDR) { + btintel_pcie_perform_pldr(data); + goto out; } - btintel_pcie_reset_ia(data); - btintel_pcie_start_rx(data); - data->flags = 0; + if (btintel_pcie_perform_flr(data)) + enable_work(&data->coredump_work); - err = btintel_pcie_setup_hdev(data); - if (err) { - BT_ERR("Failed registering hdev (%d)", err); - goto error; - } -error: +out: pci_dev_put(pdev); pci_unlock_rescan_remove(); - kfree(removal); } static void btintel_pcie_reset(struct hci_dev *hdev) { - struct btintel_pcie_removal *removal; struct btintel_pcie_data *data; data = hci_get_drvdata(hdev); @@ -2333,14 +2681,8 @@ static void btintel_pcie_reset(struct hci_dev *hdev) if (test_and_set_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags)) return; - removal = kzalloc_obj(*removal, GFP_ATOMIC); - if (!removal) - return; - - removal->pdev = data->pdev; - INIT_WORK(&removal->work, btintel_pcie_removal_work); - pci_dev_get(removal->pdev); - schedule_work(&removal->work); + pci_dev_get(data->pdev); + schedule_work(&data->reset_work); } static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code) @@ -2350,15 +2692,19 @@ static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code) struct pci_dev *pdev = dev_data->pdev; time64_t retry_window; - if (code == 0x13) { - bt_dev_err(hdev, "Encountered top exception"); - return; - } + btintel_pcie_dump_debug_registers(hdev); data = btintel_pcie_get_recovery(pdev, &hdev->dev); if (!data) return; + if (code == 0x13) + dev_data->reset_type = BTINTEL_PCIE_IOSF_PRR_PLDR; + else + dev_data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR; + + bt_dev_err(hdev, "Encountered exception err:0x%x triggering: %s", code, + dev_data->reset_type == BTINTEL_PCIE_IOSF_PRR_PLDR ? "PLDR" : "FLR"); retry_window = ktime_get_boottime_seconds() - data->last_error; if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS && @@ -2451,7 +2797,6 @@ static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data) hdev->bus = HCI_PCI; hci_set_drvdata(hdev, data); - data->hdev = hdev; SET_HCIDEV_DEV(hdev, &data->pdev->dev); hdev->manufacturer = 2; @@ -2470,15 +2815,17 @@ static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data) err = hci_register_dev(hdev); if (err < 0) { BT_ERR("Failed to register to hdev (%d)", err); - goto exit_error; + hci_free_dev(hdev); + return err; } + /* Publish hdev only after successful registration; the coredump + * worker bails on !data->hdev, so it never observes a half-set-up + * device. + */ + data->hdev = hdev; data->dmp_hdr.driver_name = KBUILD_MODNAME; return 0; - -exit_error: - hci_free_dev(hdev); - return err; } static int btintel_pcie_probe(struct pci_dev *pdev, @@ -2509,12 +2856,23 @@ static int btintel_pcie_probe(struct pci_dev *pdev, if (!data->workqueue) return -ENOMEM; + data->coredump_workqueue = alloc_ordered_workqueue(KBUILD_MODNAME "_cd", 0); + if (!data->coredump_workqueue) { + destroy_workqueue(data->workqueue); + return -ENOMEM; + } + skb_queue_head_init(&data->rx_skb_q); INIT_WORK(&data->rx_work, btintel_pcie_rx_work); + INIT_WORK(&data->reset_work, btintel_pcie_reset_work); + INIT_WORK(&data->coredump_work, btintel_pcie_coredump_worker); data->boot_stage_cache = 0x00; data->img_resp_cache = 0x00; - + /* FLR can be invoked by echoing to debugfs path, so explicitly + * initialized + */ + data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR; err = btintel_pcie_config_pcie(pdev, data); if (err) goto exit_error; @@ -2550,6 +2908,8 @@ static int btintel_pcie_probe(struct pci_dev *pdev, /* reset device before exit */ btintel_pcie_reset_bt(data); + destroy_workqueue(data->coredump_workqueue); + pci_clear_master(pdev); pci_set_drvdata(pdev, NULL); @@ -2562,6 +2922,25 @@ static void btintel_pcie_remove(struct pci_dev *pdev) struct btintel_pcie_data *data; data = pci_get_drvdata(pdev); + if (!data) { + BT_WARN("PCI driver data is NULL, aborting remove"); + return; + } + + /* Permanently block coredump triggers and drain the worker before + * tearing down. Must run before cancel_work_sync(&reset_work) so + * the disable counter stays >= 1 even after reset_work()'s + * balanced enable_work() (counter 2 -> 1, never reaching 0). + */ + disable_work_sync(&data->coredump_work); + + /* Cancel pending reset work. Skip only when remove() is called from + * within the reset work itself (PLDR device_reprobe path) to avoid + * deadlock. current_work() returns the work_struct of the caller if + * we are in a workqueue context. + */ + if (current_work() != &data->reset_work) + disable_work_sync(&data->reset_work); btintel_pcie_disable_interrupts(data); @@ -2581,6 +2960,7 @@ static void btintel_pcie_remove(struct pci_dev *pdev) btintel_pcie_release_hdev(data); + destroy_workqueue(data->coredump_workqueue); destroy_workqueue(data->workqueue); btintel_pcie_free(data); @@ -2596,11 +2976,19 @@ static void btintel_pcie_coredump(struct device *dev) struct pci_dev *pdev = to_pci_dev(dev); struct btintel_pcie_data *data = pci_get_drvdata(pdev); + if (!data) + return; + if (test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) return; data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER; - queue_work(data->workqueue, &data->rx_work); + /* queue_work() returns false if the work is disabled (reset or + * remove in progress); clear the in-progress bit so a later + * trigger can succeed once the work is re-enabled. + */ + if (!queue_work(data->coredump_workqueue, &data->coredump_work)) + clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags); } #endif @@ -2712,6 +3100,7 @@ static int btintel_pcie_resume(struct device *dev) if (data->pm_sx_event == PM_EVENT_FREEZE || data->pm_sx_event == PM_EVENT_HIBERNATE) { set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); + data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR; btintel_pcie_reset(data->hdev); return 0; } @@ -2740,7 +3129,7 @@ static int btintel_pcie_resume(struct device *dev) &data->flags)) { data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT; - queue_work(data->workqueue, &data->rx_work); + queue_work(data->coredump_workqueue, &data->coredump_work); } set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); btintel_pcie_reset(data->hdev); diff --git a/drivers/bluetooth/btintel_pcie.h b/drivers/bluetooth/btintel_pcie.h index 13efef499e4e8..7caee093e3164 100644 --- a/drivers/bluetooth/btintel_pcie.h +++ b/drivers/bluetooth/btintel_pcie.h @@ -68,6 +68,10 @@ #define BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS (BTINTEL_PCIE_DBGC_BASE_ADDR + 0x1C) #define BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND (BTINTEL_PCIE_DBGC_BASE_ADDR + 0x2C) +#define BTINTEL_PCIE_DBGC_BASE_ADDR_SCP (0xf0d5d500) +#define BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS_SCP (BTINTEL_PCIE_DBGC_BASE_ADDR_SCP + 0x1C) +#define BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND_SCP (BTINTEL_PCIE_DBGC_BASE_ADDR_SCP + 0x2C) + #define BTINTEL_PCIE_DBG_IDX_BIT_MASK 0x0F #define BTINTEL_PCIE_DBGC_DBG_BUF_IDX(data) (((data) >> 24) & BTINTEL_PCIE_DBG_IDX_BIT_MASK) #define BTINTEL_PCIE_DBG_OFFSET_BIT_MASK 0xFFFFFF @@ -98,6 +102,7 @@ enum msix_hw_int_causes { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0 = BIT(0), /* cause 32 */ BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP1 = BIT(1), /* cause 33 */ BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP = BIT(3), /* cause 35 */ + BTINTEL_PCIE_MSIX_HW_INT_CAUSES_FWTRIG = BIT(5), /* cause 37 */ }; /* PCIe device states @@ -115,6 +120,7 @@ enum { BTINTEL_PCIE_CORE_HALTED, BTINTEL_PCIE_HWEXP_INPROGRESS, BTINTEL_PCIE_COREDUMP_INPROGRESS, + BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, BTINTEL_PCIE_RECOVERY_IN_PROGRESS, BTINTEL_PCIE_SETUP_DONE }; @@ -130,7 +136,9 @@ enum btintel_pcie_tlv_type { BTINTEL_DUMP_TIME, BTINTEL_FW_BUILD, BTINTEL_VENDOR, - BTINTEL_DRIVER + BTINTEL_DRIVER, + BTINTEL_EVENT_TYPE, + BTINTEL_EVENT_ID }; /* causes for the MBOX interrupts */ @@ -141,6 +149,11 @@ enum msix_mbox_int_causes { BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4 = BIT(3), /* cause MBOX4 */ }; +enum btintel_pcie_reset_type { + BTINTEL_PCIE_IOSF_PRR_FLR = 0, + BTINTEL_PCIE_IOSF_PRR_PLDR = 1, +}; + #define BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE BIT(7) /* Minimum and Maximum number of MSI-X Vector @@ -425,6 +438,8 @@ struct btintel_pcie_dump_header { u32 wrap_ctr; u16 trigger_reason; int state; + u8 event_type; + u16 event_id; }; /* struct btintel_pcie_data @@ -451,6 +466,8 @@ struct btintel_pcie_dump_header { * @workqueue: workqueue for RX work * @rx_skb_q: SKB queue for RX packet * @rx_work: RX work struct to process the RX packet in @rx_skb_q + * @coredump_workqueue: dedicated workqueue for coredump collection + * @coredump_work: work struct for coredump trace collection * @dma_pool: DMA pool for descriptors, index array and ci * @dma_p_addr: DMA address for pool * @dma_v_addr: address of pool @@ -497,6 +514,10 @@ struct btintel_pcie_data { struct workqueue_struct *workqueue; struct sk_buff_head rx_skb_q; struct work_struct rx_work; + struct work_struct reset_work; + + struct workqueue_struct *coredump_workqueue; + struct work_struct coredump_work; struct dma_pool *dma_pool; dma_addr_t dma_p_addr; @@ -508,9 +529,12 @@ struct btintel_pcie_data { struct txq txq; struct rxq rxq; u32 alive_intr_ctxt; + enum btintel_pcie_reset_type reset_type; struct btintel_pcie_dbgc dbgc; struct btintel_pcie_dump_header dmp_hdr; u8 pm_sx_event; + u32 debug_evt_addr; + u32 debug_evt_size; }; static inline u32 btintel_pcie_rd_reg32(struct btintel_pcie_data *data, diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c index 8ff66b276af03..c29e1841b2b3f 100644 --- a/drivers/bluetooth/btmtk.c +++ b/drivers/bluetooth/btmtk.c @@ -1074,8 +1074,10 @@ struct urb *alloc_mtk_intr_urb(struct hci_dev *hdev, struct sk_buff *skb, if (!urb) return ERR_PTR(-ENOMEM); - if (btmtk_isopkt_pad(hdev, skb)) + if (btmtk_isopkt_pad(hdev, skb)) { + usb_free_urb(urb); return ERR_PTR(-EINVAL); + } pipe = usb_sndintpipe(btmtk_data->udev, btmtk_data->isopkt_tx_ep->bEndpointAddress); diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index 5b0fab7b89b5a..c6f80c419e901 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -620,7 +620,7 @@ static void btmtksdio_txrx_work(struct work_struct *work) if (btmtksdio_rx_packet(bdev, rx_size) < 0) bdev->hdev->stat.err_rx++; } - } while (int_status || time_is_before_jiffies(txrx_timeout)); + } while (int_status && time_is_after_jiffies(txrx_timeout)); /* Enable interrupt */ if (bdev->func->irq_handler) diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c index e7036a48ce486..6a1cffe08d5f7 100644 --- a/drivers/bluetooth/btnxpuart.c +++ b/drivers/bluetooth/btnxpuart.c @@ -1267,6 +1267,12 @@ static int nxp_recv_fw_req_v3(struct hci_dev *hdev, struct sk_buff *skb) } nxpdev->fw_dnld_v3_offset = offset - nxpdev->fw_v3_offset_correction; + if (nxpdev->fw_dnld_v3_offset >= nxpdev->fw->size || + len > nxpdev->fw->size - nxpdev->fw_dnld_v3_offset) { + bt_dev_err(hdev, "FW download out of bounds, ignoring request"); + len = 0; + goto free_skb; + } serdev_device_write_buf(nxpdev->serdev, nxpdev->fw->data + nxpdev->fw_dnld_v3_offset, len); diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c index 62f9d4df3a4f8..03fa9409e3ee4 100644 --- a/drivers/bluetooth/btrtl.c +++ b/drivers/bluetooth/btrtl.c @@ -797,8 +797,9 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev, } BT_DBG("length=%x offset=%x index %d", patch_length, patch_offset, i); - min_size = patch_offset + patch_length; - if (btrtl_dev->fw_len < min_size) + if (patch_length < sizeof(epatch_info->fw_version) || + patch_offset > btrtl_dev->fw_len || + patch_length > btrtl_dev->fw_len - patch_offset) return -EINVAL; /* Copy the firmware into a new buffer and write the version at diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 830fefb342c6b..cc6392f8d98cd 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -531,6 +531,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x7392, 0xe611), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x2c4e, 0x0128), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8852AE Bluetooth devices */ { USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK | @@ -2994,6 +2996,11 @@ static int marvell_config_oob_wake(struct hci_dev *hdev) return 0; } +#else +static inline int marvell_config_oob_wake(struct hci_dev *hdev) +{ + return 0; +} #endif static int btusb_set_bdaddr_marvell(struct hci_dev *hdev, @@ -3836,6 +3843,11 @@ static int btusb_config_oob_wake(struct hci_dev *hdev) bt_dev_info(hdev, "OOB Wake-on-BT configured at IRQ %u", irq); return 0; } +#else +static inline int btusb_config_oob_wake(struct hci_dev *hdev) +{ + return 0; +} #endif static void btusb_check_needs_reset_resume(struct usb_interface *intf) @@ -4172,7 +4184,6 @@ static int btusb_probe(struct usb_interface *intf, hdev->wakeup = btusb_wakeup; hdev->hci_drv = &btusb_hci_drv; -#ifdef CONFIG_PM err = btusb_config_oob_wake(hdev); if (err) goto out_free_dev; @@ -4181,9 +4192,9 @@ static int btusb_probe(struct usb_interface *intf, if (id->driver_info & BTUSB_MARVELL && data->oob_wake_irq) { err = marvell_config_oob_wake(hdev); if (err) - goto out_free_dev; + goto err_disable_wakeup; } -#endif + if (id->driver_info & BTUSB_CW6622) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_STORED_LINK_KEY); @@ -4218,7 +4229,7 @@ static int btusb_probe(struct usb_interface *intf, if (id->driver_info & BTUSB_INTEL_COMBINED) { err = btintel_configure_setup(hdev, btusb_driver.name); if (err) - goto out_free_dev; + goto err_kill_tx_urbs; /* Transport specific configuration */ hdev->send = btusb_send_frame_intel; @@ -4381,7 +4392,7 @@ static int btusb_probe(struct usb_interface *intf, err = usb_set_interface(data->udev, 0, 0); if (err < 0) { BT_ERR("failed to set interface 0, alt 0 %d", err); - goto out_free_dev; + goto err_kill_tx_urbs; } } @@ -4389,7 +4400,7 @@ static int btusb_probe(struct usb_interface *intf, err = usb_driver_claim_interface(&btusb_driver, data->isoc, data); if (err < 0) - goto out_free_dev; + goto err_kill_tx_urbs; } if (IS_ENABLED(CONFIG_BT_HCIBTUSB_BCM) && data->diag) { @@ -4407,7 +4418,7 @@ static int btusb_probe(struct usb_interface *intf, err = hci_register_dev(hdev); if (err < 0) - goto out_free_dev; + goto err_release_siblings; usb_set_intfdata(intf, data); @@ -4416,6 +4427,20 @@ static int btusb_probe(struct usb_interface *intf, return 0; +err_release_siblings: + if (data->diag) { + usb_set_intfdata(data->diag, NULL); + usb_driver_release_interface(&btusb_driver, data->diag); + } + if (data->isoc) { + usb_set_intfdata(data->isoc, NULL); + usb_driver_release_interface(&btusb_driver, data->isoc); + } +err_kill_tx_urbs: + usb_kill_anchored_urbs(&data->tx_anchor); +err_disable_wakeup: + if (data->oob_wake_irq) + device_init_wakeup(&data->udev->dev, false); out_free_dev: if (data->reset_gpio) gpiod_put(data->reset_gpio); diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index 47f4902b40b47..2ad42c3bbaac7 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -239,6 +239,8 @@ static int hci_uart_flush(struct hci_dev *hdev) BT_DBG("hdev %p tty %p", hdev, tty); + disable_work_sync(&hu->write_work); + if (hu->tx_skb) { kfree_skb(hu->tx_skb); hu->tx_skb = NULL; } @@ -254,6 +256,14 @@ static int hci_uart_flush(struct hci_dev *hdev) percpu_up_read(&hu->proto_lock); + /* Resume TX. Also reschedule in case work was queued concurrently; + * this may schedule write_work although there's nothing to do. + */ + enable_work(&hu->write_work); + clear_bit(HCI_UART_SENDING, &hu->tx_state); + if (test_bit(HCI_UART_TX_WAKEUP, &hu->tx_state)) + hci_uart_tx_wakeup(hu); + return 0; } @@ -271,12 +281,8 @@ static int hci_uart_open(struct hci_dev *hdev) /* Close device */ static int hci_uart_close(struct hci_dev *hdev) { - struct hci_uart *hu = hci_get_drvdata(hdev); - BT_DBG("hdev %p", hdev); - cancel_work_sync(&hu->write_work); - hci_uart_flush(hdev); hdev->flush = NULL; return 0; diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 34500137df2c1..2444471956197 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -1028,7 +1028,7 @@ static void qca_dmp_hdr(struct hci_dev *hdev, struct sk_buff *skb) skb_put_data(skb, buf, strlen(buf)); snprintf(buf, sizeof(buf), "Driver: %s\n", - hu->serdev->dev.driver->name); + hu->serdev ? hu->serdev->dev.driver->name : "hci_ldisc_qca"); skb_put_data(skb, buf, strlen(buf)); } @@ -1916,9 +1916,12 @@ static int qca_setup(struct hci_uart *hu) const char *rampatch_name = qca_get_rampatch_name(hu); int ret; struct qca_btsoc_version ver; - struct qca_serdev *qcadev = serdev_device_get_drvdata(hu->serdev); + struct qca_serdev *qcadev = NULL; const char *soc_name; + if (hu->serdev) + qcadev = serdev_device_get_drvdata(hu->serdev); + ret = qca_check_speeds(hu); if (ret) return ret; @@ -1980,7 +1983,7 @@ static int qca_setup(struct hci_uart *hu) case QCA_WCN6750: case QCA_WCN6855: case QCA_WCN7850: - if (qcadev->bdaddr_property_broken) + if (qcadev && qcadev->bdaddr_property_broken) hci_set_quirk(hdev, HCI_QUIRK_BDADDR_PROPERTY_BROKEN); hci_set_aosp_capable(hdev); @@ -2073,7 +2076,7 @@ static int qca_setup(struct hci_uart *hu) else hu->hdev->set_bdaddr = qca_set_bdaddr; - if (qcadev->support_hfp_hw_offload) + if (qcadev && qcadev->support_hfp_hw_offload) qca_configure_hfp_offload(hdev); qca->fw_version = le16_to_cpu(ver.patch_ver); diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c index 2762eacf7f20e..eddb32b30539f 100644 --- a/drivers/bluetooth/hci_vhci.c +++ b/drivers/bluetooth/hci_vhci.c @@ -337,7 +337,17 @@ static ssize_t force_devcd_write(struct file *file, const char __user *user_buf, if (copy_from_user(&dump_data, user_buf, count)) return -EFAULT; + switch (dump_data.state) { + case HCI_DEVCOREDUMP_DONE: + case HCI_DEVCOREDUMP_ABORT: + case HCI_DEVCOREDUMP_TIMEOUT: + break; + default: + return -EINVAL; + } + data_size = count - offsetof(struct devcoredump_test_data, data); + skb = alloc_skb(data_size, GFP_ATOMIC); if (!skb) return -ENOMEM; diff --git a/drivers/bus/imx-weim.c b/drivers/bus/imx-weim.c index f735e0462c55e..87070155b0572 100644 --- a/drivers/bus/imx-weim.c +++ b/drivers/bus/imx-weim.c @@ -327,12 +327,6 @@ static int of_weim_notify(struct notifier_block *nb, unsigned long action, "Failed to setup timing for '%pOF'\n", rd->dn); if (!of_node_check_flag(rd->dn, OF_POPULATED)) { - /* - * Clear the flag before adding the device so that - * fw_devlink doesn't skip adding consumers to this - * device. - */ - fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); if (!of_platform_device_create(rd->dn, NULL, &pdev->dev)) { dev_err(&pdev->dev, "Failed to create child device '%pOF'\n", diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c index 0277e1ab11988..9db2a2a2c9134 100644 --- a/drivers/bus/mhi/ep/main.c +++ b/drivers/bus/mhi/ep/main.c @@ -232,7 +232,9 @@ static int mhi_ep_process_cmd_ring(struct mhi_ep_ring *ring, struct mhi_ring_ele ret = mhi_ep_create_device(mhi_cntrl, ch_id); if (ret) { dev_err(dev, "Error creating device for channel (%u)\n", ch_id); + mutex_lock(&mhi_cntrl->state_lock); mhi_ep_handle_syserr(mhi_cntrl); + mutex_unlock(&mhi_cntrl->state_lock); return ret; } } @@ -1087,11 +1089,12 @@ static void mhi_ep_reset_worker(struct work_struct *work) mhi_ep_power_down(mhi_cntrl); - mutex_lock(&mhi_cntrl->state_lock); - /* Reset MMIO to signal host that the MHI_RESET is completed in endpoint */ mhi_ep_mmio_reset(mhi_cntrl); + + mutex_lock(&mhi_cntrl->state_lock); cur_state = mhi_cntrl->mhi_state; + mutex_unlock(&mhi_cntrl->state_lock); /* * Only proceed further if the reset is due to SYS_ERR. The host will @@ -1100,8 +1103,6 @@ static void mhi_ep_reset_worker(struct work_struct *work) */ if (cur_state == MHI_STATE_SYS_ERR) mhi_ep_power_up(mhi_cntrl); - - mutex_unlock(&mhi_cntrl->state_lock); } /* @@ -1148,7 +1149,9 @@ int mhi_ep_power_up(struct mhi_ep_cntrl *mhi_cntrl) for (i = 0; i < mhi_cntrl->event_rings; i++) mhi_ep_ring_init(&mhi_cntrl->mhi_event[i].ring, RING_TYPE_ER, i); + mutex_lock(&mhi_cntrl->state_lock); mhi_cntrl->mhi_state = MHI_STATE_RESET; + mutex_unlock(&mhi_cntrl->state_lock); /* Set AMSS EE before signaling ready state */ mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS); diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c index 750da3dbb4c6e..a738333bfc758 100644 --- a/drivers/bus/mhi/host/pci_generic.c +++ b/drivers/bus/mhi/host/pci_generic.c @@ -1247,7 +1247,7 @@ static void mhi_pci_recovery_work(struct work_struct *work) dev_warn(&pdev->dev, "device recovery started\n"); - if (pdev->is_physfn) + if (!pdev->is_virtfn) timer_delete(&mhi_pdev->health_check_timer); pm_runtime_forbid(&pdev->dev); @@ -1277,7 +1277,7 @@ static void mhi_pci_recovery_work(struct work_struct *work) set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); - if (pdev->is_physfn) + if (!pdev->is_virtfn) mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); return; @@ -1368,7 +1368,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) mhi_cntrl_config = info->config; /* Initialize health check monitor only for Physical functions */ - if (pdev->is_physfn) + if (!pdev->is_virtfn) timer_setup(&mhi_pdev->health_check_timer, health_check, 0); mhi_cntrl = &mhi_pdev->mhi_cntrl; @@ -1390,7 +1390,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) mhi_cntrl->mru = info->mru_default; mhi_cntrl->name = info->name; - if (pdev->is_physfn) + if (!pdev->is_virtfn) mhi_pdev->reset_on_remove = info->reset_on_remove; if (info->edl_trigger) @@ -1439,7 +1439,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); /* start health check */ - if (pdev->is_physfn) + if (!pdev->is_virtfn) mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); /* Allow runtime suspend only if both PME from D3Hot and M3 are supported */ @@ -1468,7 +1468,7 @@ static void mhi_pci_remove(struct pci_dev *pdev) pm_runtime_forbid(&pdev->dev); pci_disable_sriov(pdev); - if (pdev->is_physfn) + if (!pdev->is_virtfn) timer_delete_sync(&mhi_pdev->health_check_timer); cancel_work_sync(&mhi_pdev->recovery_work); @@ -1500,7 +1500,7 @@ static void mhi_pci_reset_prepare(struct pci_dev *pdev) dev_info(&pdev->dev, "reset\n"); - if (pdev->is_physfn) + if (!pdev->is_virtfn) timer_delete(&mhi_pdev->health_check_timer); /* Clean up MHI state */ @@ -1546,7 +1546,7 @@ static void mhi_pci_reset_done(struct pci_dev *pdev) } set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); - if (pdev->is_physfn) + if (!pdev->is_virtfn) mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); } @@ -1612,7 +1612,7 @@ static int __maybe_unused mhi_pci_runtime_suspend(struct device *dev) if (test_and_set_bit(MHI_PCI_DEV_SUSPENDED, &mhi_pdev->status)) return 0; - if (pdev->is_physfn) + if (!pdev->is_virtfn) timer_delete(&mhi_pdev->health_check_timer); cancel_work_sync(&mhi_pdev->recovery_work); @@ -1665,7 +1665,7 @@ static int __maybe_unused mhi_pci_runtime_resume(struct device *dev) } /* Resume health check */ - if (pdev->is_physfn) + if (!pdev->is_virtfn) mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); /* It can be a remote wakeup (no mhi runtime_get), update access time */ diff --git a/drivers/bus/sunxi-rsb.c b/drivers/bus/sunxi-rsb.c index b4f2c64ac1810..daf0ea563e477 100644 --- a/drivers/bus/sunxi-rsb.c +++ b/drivers/bus/sunxi-rsb.c @@ -445,6 +445,9 @@ static int regmap_sunxi_rsb_reg_write(void *context, unsigned int reg, struct sunxi_rsb_ctx *ctx = context; struct sunxi_rsb_device *rdev = ctx->rdev; + if (reg > 0xff) + return -EINVAL; + return sunxi_rsb_write(rdev->rsb, rdev->rtaddr, reg, &val, ctx->size); } diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c index 9196dc50a48db..d3d2302472621 100644 --- a/drivers/cdx/cdx.c +++ b/drivers/cdx/cdx.c @@ -156,8 +156,6 @@ static int cdx_unregister_device(struct device *dev, } else { cdx_destroy_res_attr(cdx_dev, MAX_CDX_DEV_RESOURCES); debugfs_remove_recursive(cdx_dev->debugfs_dir); - kfree(cdx_dev->driver_override); - cdx_dev->driver_override = NULL; } /* @@ -268,6 +266,7 @@ static int cdx_bus_match(struct device *dev, const struct device_driver *drv) const struct cdx_driver *cdx_drv = to_cdx_driver(drv); const struct cdx_device_id *found_id = NULL; const struct cdx_device_id *ids; + int ret; if (cdx_dev->is_bus) return false; @@ -275,7 +274,8 @@ static int cdx_bus_match(struct device *dev, const struct device_driver *drv) ids = cdx_drv->match_id_table; /* When driver_override is set, only bind to the matching driver */ - if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name)) + ret = device_match_driver_override(dev, drv); + if (ret == 0) return false; found_id = cdx_match_id(ids, cdx_dev); @@ -289,7 +289,7 @@ static int cdx_bus_match(struct device *dev, const struct device_driver *drv) */ if (!found_id->override_only) return true; - if (cdx_dev->driver_override) + if (ret > 0) return true; ids = found_id + 1; @@ -453,36 +453,6 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(modalias); -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct cdx_device *cdx_dev = to_cdx_device(dev); - int ret; - - if (WARN_ON(dev->bus != &cdx_bus_type)) - return -EINVAL; - - ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} - -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct cdx_device *cdx_dev = to_cdx_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", cdx_dev->driver_override); - device_unlock(dev); - return len; -} -static DEVICE_ATTR_RW(driver_override); - static ssize_t enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -552,7 +522,6 @@ static struct attribute *cdx_dev_attrs[] = { &dev_attr_class.attr, &dev_attr_revision.attr, &dev_attr_modalias.attr, - &dev_attr_driver_override.attr, NULL, }; @@ -646,6 +615,7 @@ ATTRIBUTE_GROUPS(cdx_bus); const struct bus_type cdx_bus_type = { .name = "cdx", + .driver_override = true, .match = cdx_bus_match, .probe = cdx_probe, .remove = cdx_remove, diff --git a/drivers/char/agp/amd64-agp.c b/drivers/char/agp/amd64-agp.c index 2505df1f4e691..6741270e0a982 100644 --- a/drivers/char/agp/amd64-agp.c +++ b/drivers/char/agp/amd64-agp.c @@ -546,7 +546,7 @@ static int agp_amd64_probe(struct pci_dev *pdev, /* Fill in the mode register */ pci_read_config_dword(pdev, bridge->capndx+PCI_AGP_STATUS, &bridge->mode); - if (cache_nbs(pdev, cap_ptr) == -1) { + if (cache_nbs(pdev, cap_ptr) < 0) { agp_put_bridge(bridge); return -ENODEV; } diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c index 46c84e5df00fd..285c6037417ab 100644 --- a/drivers/char/hpet.c +++ b/drivers/char/hpet.c @@ -976,10 +976,14 @@ static acpi_status hpet_resources(struct acpi_resource *res, void *data) static int hpet_acpi_probe(struct platform_device *pdev) { - struct acpi_device *device = ACPI_COMPANION(&pdev->dev); + struct acpi_device *device; acpi_status result; struct hpet_data data; + device = ACPI_COMPANION(&pdev->dev); + if (!device) + return -ENODEV; + memset(&data, 0, sizeof(data)); result = diff --git a/drivers/char/hw_random/jh7110-trng.c b/drivers/char/hw_random/jh7110-trng.c index 9776f4daa0445..4712c3c530e42 100644 --- a/drivers/char/hw_random/jh7110-trng.c +++ b/drivers/char/hw_random/jh7110-trng.c @@ -256,19 +256,22 @@ static int starfive_trng_read(struct hwrng *rng, void *buf, size_t max, bool wai if (wait) { ret = starfive_trng_wait_idle(trng); - if (ret) - return -ETIMEDOUT; + if (ret) { + ret = -ETIMEDOUT; + goto out_put; + } } ret = starfive_trng_cmd(trng, STARFIVE_CTRL_GENE_RANDNUM, wait); if (ret) - return ret; + goto out_put; memcpy_fromio(buf, trng->base + STARFIVE_RAND0, max); + ret = max; +out_put: pm_runtime_put_sync_autosuspend(trng->dev); - - return max; + return ret; } static int starfive_trng_probe(struct platform_device *pdev) diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index 0ce02d7e5048e..5e83ffa105e41 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -69,8 +70,26 @@ static void request_entropy(struct virtrng_info *vi) static unsigned int copy_data(struct virtrng_info *vi, void *buf, unsigned int size) { - size = min_t(unsigned int, size, vi->data_avail); - memcpy(buf, vi->data + vi->data_idx, size); + unsigned int idx, avail; + + /* + * vi->data_avail was set from the device-reported used.len and + * vi->data_idx was advanced by previous copy_data() calls. A + * malicious or buggy virtio-rng backend can drive either past + * sizeof(vi->data). Clamp at point of use and harden the index + * with array_index_nospec() so the memcpy() below cannot be + * steered into adjacent slab memory, including under + * speculation. + */ + avail = min_t(unsigned int, vi->data_avail, sizeof(vi->data)); + if (vi->data_idx >= avail) { + vi->data_avail = 0; + request_entropy(vi); + return 0; + } + size = min_t(unsigned int, size, avail - vi->data_idx); + idx = array_index_nospec(vi->data_idx, sizeof(vi->data)); + memcpy(buf, vi->data + idx, size); vi->data_idx += size; vi->data_avail -= size; if (vi->data_avail == 0) diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 869ac87a4b6ae..3e0b7d60269d9 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -2347,6 +2347,10 @@ static int i_ipmi_request(struct ipmi_user *user, if (smi_msg == NULL) { if (!supplied_recv) ipmi_free_recv_msg(recv_msg); + else if (recv_msg->user) { + atomic_dec(&recv_msg->user->nr_msgs); + kref_put(&recv_msg->user->refcount, free_ipmi_user); + } return -ENOMEM; } } @@ -2420,6 +2424,10 @@ static int i_ipmi_request(struct ipmi_user *user, ipmi_free_smi_msg(smi_msg); if (!supplied_recv) ipmi_free_recv_msg(recv_msg); + else if (recv_msg->user) { + atomic_dec(&recv_msg->user->nr_msgs); + kref_put(&recv_msg->user->refcount, free_ipmi_user); + } } return rv; } @@ -4477,10 +4485,8 @@ static int handle_read_event_rsp(struct ipmi_smi *intf, mutex_unlock(&intf->users_mutex); list_for_each_entry_safe(recv_msg, recv_msg2, &msgs, link) { - user = recv_msg->user; list_del(&recv_msg->link); ipmi_free_recv_msg(recv_msg); - kref_put(&user->refcount, free_ipmi_user); } /* * We couldn't allocate memory for the diff --git a/drivers/char/sonypi.c b/drivers/char/sonypi.c index ccda997a9098a..24c1b26f34d6b 100644 --- a/drivers/char/sonypi.c +++ b/drivers/char/sonypi.c @@ -1117,7 +1117,11 @@ static int sonypi_disable(void) #ifdef CONFIG_ACPI static int sonypi_acpi_probe(struct platform_device *pdev) { - struct acpi_device *device = ACPI_COMPANION(&pdev->dev); + struct acpi_device *device; + + device = ACPI_COMPANION(&pdev->dev); + if (!device) + return -ENODEV; sonypi_acpi_device = device; strcpy(acpi_device_name(device), "Sony laptop hotkeys"); diff --git a/drivers/char/tlclk.c b/drivers/char/tlclk.c index 677d230a226cf..dd45fe5eb6f27 100644 --- a/drivers/char/tlclk.c +++ b/drivers/char/tlclk.c @@ -264,6 +264,7 @@ static ssize_t tlclk_read(struct file *filp, char __user *buf, size_t count, } static const struct file_operations tlclk_fops = { + .owner = THIS_MODULE, .read = tlclk_read, .open = tlclk_open, .release = tlclk_release, @@ -837,6 +838,9 @@ static void __exit tlclk_cleanup(void) misc_deregister(&tlclk_miscdev); unregister_chrdev(tlclk_major, "telco_clock"); + got_event = 1; + wake_up_all(&wq); + release_region(TLCLK_BASE, 8); timer_delete_sync(&switchover_timer); kfree(alarm_events); diff --git a/drivers/char/tpm/eventlog/tpm1.c b/drivers/char/tpm/eventlog/tpm1.c index e7913b2853d53..0397e3361020a 100644 --- a/drivers/char/tpm/eventlog/tpm1.c +++ b/drivers/char/tpm/eventlog/tpm1.c @@ -236,12 +236,12 @@ static int tpm1_binary_bios_measurements_show(struct seq_file *m, void *v) temp_ptr = (char *) &temp_event; - for (i = 0; i < (sizeof(struct tcpa_event) - 1) ; i++) + for (i = 0; i < sizeof(struct tcpa_event); i++) seq_putc(m, temp_ptr[i]); temp_ptr = (char *) v; - for (i = (sizeof(struct tcpa_event) - 1); + for (i = sizeof(struct tcpa_event); i < (sizeof(struct tcpa_event) + temp_event.event_size); i++) seq_putc(m, temp_ptr[i]); diff --git a/drivers/char/tpm/tpm-dev.c b/drivers/char/tpm/tpm-dev.c index 2779a8738c59e..74488f0a7b789 100644 --- a/drivers/char/tpm/tpm-dev.c +++ b/drivers/char/tpm/tpm-dev.c @@ -36,7 +36,7 @@ static int tpm_open(struct inode *inode, struct file *file) tpm_common_open(file, chip, priv, NULL); - return 0; + return nonseekable_open(inode, file); out: clear_bit(0, &chip->is_open); diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index b11e6fa8b740e..52ee350da8671 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -71,9 +71,9 @@ static const struct { {TPM2_CC_HIERARCHY_CHANGE_AUTH, 2000}, {TPM2_CC_GET_CAPABILITY, 750}, {TPM2_CC_NV_READ, 2000}, - {TPM2_CC_CREATE_PRIMARY, 30000}, - {TPM2_CC_CREATE, 30000}, - {TPM2_CC_CREATE_LOADED, 30000}, + {TPM2_CC_CREATE_PRIMARY, 300000}, + {TPM2_CC_CREATE, 300000}, + {TPM2_CC_CREATE_LOADED, 300000}, }; /** diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c index c4da6fde748f4..f44646b26b192 100644 --- a/drivers/char/tpm/tpm2-sessions.c +++ b/drivers/char/tpm/tpm2-sessions.c @@ -489,15 +489,17 @@ static void tpm2_KDFe(u8 z[EC_PT_SZ], const char *str, u8 *pt_u, u8 *pt_v, sha256_final(&sctx, out); } -static void tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip, - struct tpm2_auth *auth) +static int tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip, + struct tpm2_auth *auth) { struct crypto_kpp *kpp; struct kpp_request *req; + DECLARE_CRYPTO_WAIT(wait); struct scatterlist s[2], d[1]; struct ecdh p = {0}; u8 encoded_key[EC_PT_SZ], *x, *y; unsigned int buf_len; + int rc; /* secret is two sized points */ tpm_buf_append_u16(buf, (EC_PT_SZ + 2)*2); @@ -520,14 +522,15 @@ static void tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip, kpp = crypto_alloc_kpp("ecdh-nist-p256", CRYPTO_ALG_INTERNAL, 0); if (IS_ERR(kpp)) { dev_err(&chip->dev, "crypto ecdh allocation failed\n"); - return; + return PTR_ERR(kpp); } buf_len = crypto_ecdh_key_len(&p); if (sizeof(encoded_key) < buf_len) { dev_err(&chip->dev, "salt buffer too small needs %d\n", buf_len); - goto out; + rc = -EINVAL; + goto err_free_kpp; } crypto_ecdh_encode_key(encoded_key, buf_len, &p); /* this generates a random private key */ @@ -535,11 +538,17 @@ static void tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip, /* salt is now the public point of this private key */ req = kpp_request_alloc(kpp, GFP_KERNEL); - if (!req) - goto out; + if (!req) { + rc = -ENOMEM; + goto err_free_kpp; + } + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); kpp_request_set_input(req, NULL, 0); kpp_request_set_output(req, s, EC_PT_SZ*2); - crypto_kpp_generate_public_key(req); + rc = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait); + if (rc) + goto err_free_req; /* * we're not done: now we have to compute the shared secret * which is our private key multiplied by the tpm_key public @@ -551,8 +560,9 @@ static void tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip, kpp_request_set_input(req, s, EC_PT_SZ*2); sg_init_one(d, auth->salt, EC_PT_SZ); kpp_request_set_output(req, d, EC_PT_SZ); - crypto_kpp_compute_shared_secret(req); - kpp_request_free(req); + rc = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait); + if (rc) + goto err_free_req; /* * pass the shared secret through KDFe for salt. Note salt @@ -562,8 +572,16 @@ static void tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip, */ tpm2_KDFe(auth->salt, "SECRET", x, chip->null_ec_key_x, auth->salt); - out: + kpp_request_free(req); crypto_free_kpp(kpp); + return 0; + +err_free_req: + kpp_request_free(req); + +err_free_kpp: + crypto_free_kpp(kpp); + return rc; } /** @@ -1018,7 +1036,12 @@ int tpm2_start_auth_session(struct tpm_chip *chip) tpm_buf_append(&buf, auth->our_nonce, sizeof(auth->our_nonce)); /* append encrypted salt and squirrel away unencrypted in auth */ - tpm_buf_append_salt(&buf, chip, auth); + rc = tpm_buf_append_salt(&buf, chip, auth); + if (rc) { + tpm2_flush_context(chip, null_key); + tpm_buf_destroy(&buf); + goto out; + } /* session type (HMAC, audit or policy) */ tpm_buf_append_u8(&buf, TPM2_SE_HMAC); diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c index 7d1377e8e616a..ceb4100ba400f 100644 --- a/drivers/char/tpm/tpm_crb.c +++ b/drivers/char/tpm/tpm_crb.c @@ -786,8 +786,8 @@ static int crb_map_pluton(struct device *dev, struct crb_priv *priv, static int crb_acpi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct acpi_device *device = ACPI_COMPANION(dev); struct acpi_table_tpm2 *buf; + struct acpi_device *device; struct crb_priv *priv; struct tpm_chip *chip; struct tpm2_crb_smc *crb_smc; @@ -797,6 +797,10 @@ static int crb_acpi_probe(struct platform_device *pdev) u32 sm; int rc; + device = ACPI_COMPANION(dev); + if (!device) + return -ENODEV; + status = acpi_get_table(ACPI_SIG_TPM2, 1, (struct acpi_table_header **) &buf); if (ACPI_FAILURE(status) || buf->header.length < sizeof(*buf)) { diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index 21d79ad3b164e..153a57c792400 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -66,8 +66,8 @@ static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask, bool check_cancel) { struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + DEFINE_WAIT_FUNC(wait, woken_wake_function); unsigned long stop; - long rc; u8 status; bool canceled = false; u8 sts_mask; @@ -87,23 +87,30 @@ static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask, /* process status changes with irq support */ if (sts_mask) { ret = -ETIME; + add_wait_queue(queue, &wait); again: + if (wait_for_tpm_stat_cond(chip, sts_mask, check_cancel, + &canceled)) { + ret = canceled ? -ECANCELED : 0; + goto out; + } + timeout = stop - jiffies; if ((long)timeout <= 0) - return -ETIME; - rc = wait_event_interruptible_timeout(*queue, - wait_for_tpm_stat_cond(chip, sts_mask, check_cancel, - &canceled), - timeout); - if (rc > 0) { - if (canceled) - return -ECANCELED; - ret = 0; - } - if (rc == -ERESTARTSYS && freezing(current)) { - clear_thread_flag(TIF_SIGPENDING); - goto again; + goto out; + + if (signal_pending(current)) { + if (freezing(current)) { + clear_thread_flag(TIF_SIGPENDING); + goto again; + } + goto out; } + + wait_woken(&wait, TASK_INTERRUPTIBLE, timeout); + goto again; +out: + remove_wait_queue(queue, &wait); } if (ret) diff --git a/drivers/char/tpm/tpmrm-dev.c b/drivers/char/tpm/tpmrm-dev.c index f48d4d9e179cd..19e8f2779265f 100644 --- a/drivers/char/tpm/tpmrm-dev.c +++ b/drivers/char/tpm/tpmrm-dev.c @@ -29,7 +29,7 @@ static int tpmrm_open(struct inode *inode, struct file *file) tpm_common_open(file, chip, &priv->priv, &priv->space); - return 0; + return nonseekable_open(inode, file); } static int tpmrm_release(struct inode *inode, struct file *file) diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index 9a33217c68d9d..198b973141680 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -1771,32 +1771,40 @@ static void config_intr(struct virtio_device *vdev) schedule_work(&portdev->config_work); } -static void config_work_handler(struct work_struct *work) +static void update_size_from_config(struct ports_device *portdev) { - struct ports_device *portdev; + struct virtio_device *vdev; + struct port *port; + u16 rows, cols; - portdev = container_of(work, struct ports_device, config_work); - if (!use_multiport(portdev)) { - struct virtio_device *vdev; - struct port *port; - u16 rows, cols; + vdev = portdev->vdev; - vdev = portdev->vdev; - virtio_cread(vdev, struct virtio_console_config, cols, &cols); - virtio_cread(vdev, struct virtio_console_config, rows, &rows); + /* + * We'll use this way of resizing only for legacy support. + * For multiport devices, use control messages to indicate + * console size changes so that it can be done per-port. + * + * Don't test F_SIZE at all if we're rproc: not a valid feature. + */ + if (is_rproc_serial(vdev) || + use_multiport(portdev) || + !virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE)) + return; - port = find_port_by_id(portdev, 0); - set_console_size(port, rows, cols); + virtio_cread(vdev, struct virtio_console_config, cols, &cols); + virtio_cread(vdev, struct virtio_console_config, rows, &rows); - /* - * We'll use this way of resizing only for legacy - * support. For newer userspace - * (VIRTIO_CONSOLE_F_MULTPORT+), use control messages - * to indicate console size changes so that it can be - * done per-port. - */ - resize_console(port); - } + port = find_port_by_id(portdev, 0); + set_console_size(port, rows, cols); + resize_console(port); +} + +static void config_work_handler(struct work_struct *work) +{ + struct ports_device *portdev; + + portdev = container_of(work, struct ports_device, config_work); + update_size_from_config(portdev); } static int init_vqs(struct ports_device *portdev) @@ -2052,6 +2060,8 @@ static int virtcons_probe(struct virtio_device *vdev) __send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID, VIRTIO_CONSOLE_DEVICE_READY, 1); + update_size_from_config(portdev); + return 0; free_chrdev: diff --git a/drivers/clk/at91/pmc.c b/drivers/clk/at91/pmc.c index b618a5e00b006..03a6c31d6aa8b 100644 --- a/drivers/clk/at91/pmc.c +++ b/drivers/clk/at91/pmc.c @@ -180,9 +180,9 @@ static int __init pmc_register_ops(void) of_node_put(np); return -ENODEV; } - of_node_put(np); at91_pmc_backup_suspend = of_iomap(np, 0); + of_node_put(np); if (!at91_pmc_backup_suspend) { pr_warn("%s(): unable to map securam\n", __func__); return -ENOMEM; diff --git a/drivers/clk/at91/sam9x7.c b/drivers/clk/at91/sam9x7.c index 89868a0aeaba9..6b330c3e6bca8 100644 --- a/drivers/clk/at91/sam9x7.c +++ b/drivers/clk/at91/sam9x7.c @@ -569,6 +569,15 @@ static const struct { .pp_chg_id = INT_MIN, }, + { + .n = "gmac_gclk", + .id = 24, + .pp = { "audiopll_divpmcck", "plla_div2pmcck", }, + .pp_mux_table = { 6, 8, }, + .pp_count = 2, + .pp_chg_id = INT_MIN, + }, + { .n = "lcd_gclk", .id = 25, @@ -702,15 +711,6 @@ static const struct { .pp_count = 1, .pp_chg_id = INT_MIN, }, - - { - .n = "gmac_gclk", - .id = 67, - .pp = { "audiopll_divpmcck", "plla_div2pmcck", }, - .pp_mux_table = { 6, 8, }, - .pp_count = 2, - .pp_chg_id = INT_MIN, - }, }; static void __init sam9x7_pmc_setup(struct device_node *np) diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c index 6b286ea6f1218..b6a12f3bc123c 100644 --- a/drivers/clk/clk-scmi.c +++ b/drivers/clk/clk-scmi.c @@ -10,9 +10,9 @@ #include #include #include +#include #include #include -#include #define NOT_ATOMIC false #define ATOMIC true @@ -83,7 +83,7 @@ static int scmi_clk_determine_rate(struct clk_hw *hw, ftmp = req->rate - fmin; ftmp += clk->info->range.step_size - 1; /* to round up */ - do_div(ftmp, clk->info->range.step_size); + ftmp = div64_ul(ftmp, clk->info->range.step_size); req->rate = ftmp * clk->info->range.step_size + fmin; diff --git a/drivers/clk/clk-scpi.c b/drivers/clk/clk-scpi.c index 7806569cd0d5c..24cee7c9fda6c 100644 --- a/drivers/clk/clk-scpi.c +++ b/drivers/clk/clk-scpi.c @@ -258,7 +258,7 @@ static void scpi_clocks_remove(struct platform_device *pdev) } for_each_available_child_of_node(np, child) - of_clk_del_provider(np); + of_clk_del_provider(child); } static int scpi_clocks_probe(struct platform_device *pdev) diff --git a/drivers/clk/microchip/clk-mpfs-ccc.c b/drivers/clk/microchip/clk-mpfs-ccc.c index 0a76a1aaa50f7..40c17593e5941 100644 --- a/drivers/clk/microchip/clk-mpfs-ccc.c +++ b/drivers/clk/microchip/clk-mpfs-ccc.c @@ -32,6 +32,7 @@ #define MPFS_CCC_FIXED_DIV 4 #define MPFS_CCC_OUTPUTS_PER_PLL 4 #define MPFS_CCC_REFS_PER_PLL 2 +#define MPFS_CCC_NUM_CLKS 16 struct mpfs_ccc_data { void __iomem **pll_base; @@ -178,7 +179,7 @@ static int mpfs_ccc_register_outputs(struct device *dev, struct mpfs_ccc_out_hw_ return dev_err_probe(dev, ret, "failed to register clock id: %d\n", out_hw->id); - data->hw_data.hws[out_hw->id - 2] = &out_hw->divider.hw; + data->hw_data.hws[out_hw->id] = &out_hw->divider.hw; } return 0; @@ -231,17 +232,9 @@ static int mpfs_ccc_probe(struct platform_device *pdev) { struct mpfs_ccc_data *clk_data; void __iomem *pll_base[ARRAY_SIZE(mpfs_ccc_pll_clks)]; - unsigned int num_clks; int ret; - /* - * If DLLs get added here, mpfs_ccc_register_outputs() currently packs - * sparse clock IDs in the hws array - */ - num_clks = ARRAY_SIZE(mpfs_ccc_pll_clks) + ARRAY_SIZE(mpfs_ccc_pll0out_clks) + - ARRAY_SIZE(mpfs_ccc_pll1out_clks); - - clk_data = devm_kzalloc(&pdev->dev, struct_size(clk_data, hw_data.hws, num_clks), + clk_data = devm_kzalloc(&pdev->dev, struct_size(clk_data, hw_data.hws, MPFS_CCC_NUM_CLKS), GFP_KERNEL); if (!clk_data) return -ENOMEM; @@ -255,7 +248,7 @@ static int mpfs_ccc_probe(struct platform_device *pdev) return PTR_ERR(pll_base[1]); clk_data->pll_base = pll_base; - clk_data->hw_data.num = num_clks; + clk_data->hw_data.num = MPFS_CCC_NUM_CLKS; clk_data->dev = &pdev->dev; ret = mpfs_ccc_register_plls(clk_data->dev, mpfs_ccc_pll_clks, diff --git a/drivers/clk/qcom/a53-pll.c b/drivers/clk/qcom/a53-pll.c index 724a642311e50..0549b214fcfc8 100644 --- a/drivers/clk/qcom/a53-pll.c +++ b/drivers/clk/qcom/a53-pll.c @@ -20,7 +20,7 @@ static const struct pll_freq_tbl a53pll_freq[] = { { 998400000, 52, 0x0, 0x1, 0 }, { 1094400000, 57, 0x0, 0x1, 0 }, - { 1152000000, 62, 0x0, 0x1, 0 }, + { 1152000000, 60, 0x0, 0x1, 0 }, { 1209600000, 63, 0x0, 0x1, 0 }, { 1248000000, 65, 0x0, 0x1, 0 }, { 1363200000, 71, 0x0, 0x1, 0 }, diff --git a/drivers/clk/qcom/camcc-x1e80100.c b/drivers/clk/qcom/camcc-x1e80100.c index 81f579ff69933..c12994af42cfc 100644 --- a/drivers/clk/qcom/camcc-x1e80100.c +++ b/drivers/clk/qcom/camcc-x1e80100.c @@ -1052,6 +1052,31 @@ static struct clk_rcg2 cam_cc_mclk7_clk_src = { }, }; +static const struct freq_tbl ftbl_cam_cc_qdss_debug_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_CAM_CC_PLL8_OUT_EVEN, 8, 0, 0), + F(75000000, P_CAM_CC_PLL0_OUT_EVEN, 8, 0, 0), + F(150000000, P_CAM_CC_PLL0_OUT_EVEN, 4, 0, 0), + F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_qdss_debug_clk_src = { + .cmd_rcgr = 0x13938, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_qdss_debug_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + static const struct freq_tbl ftbl_cam_cc_sfe_0_clk_src[] = { F(345600000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0), F(432000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0), @@ -2182,6 +2207,42 @@ static struct clk_branch cam_cc_mclk7_clk = { }, }; +static struct clk_branch cam_cc_qdss_debug_clk = { + .halt_reg = 0x13a64, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13a64, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_qdss_debug_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_qdss_debug_xo_clk = { + .halt_reg = 0x13a68, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13a68, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_xo_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch cam_cc_sfe_0_clk = { .halt_reg = 0x133c0, .halt_check = BRANCH_HALT, @@ -2398,6 +2459,9 @@ static struct clk_regmap *cam_cc_x1e80100_clocks[] = { [CAM_CC_PLL6_OUT_EVEN] = &cam_cc_pll6_out_even.clkr, [CAM_CC_PLL8] = &cam_cc_pll8.clkr, [CAM_CC_PLL8_OUT_EVEN] = &cam_cc_pll8_out_even.clkr, + [CAM_CC_QDSS_DEBUG_CLK] = &cam_cc_qdss_debug_clk.clkr, + [CAM_CC_QDSS_DEBUG_CLK_SRC] = &cam_cc_qdss_debug_clk_src.clkr, + [CAM_CC_QDSS_DEBUG_XO_CLK] = &cam_cc_qdss_debug_xo_clk.clkr, [CAM_CC_SFE_0_CLK] = &cam_cc_sfe_0_clk.clkr, [CAM_CC_SFE_0_CLK_SRC] = &cam_cc_sfe_0_clk_src.clkr, [CAM_CC_SFE_0_FAST_AHB_CLK] = &cam_cc_sfe_0_fast_ahb_clk.clkr, diff --git a/drivers/clk/qcom/ipq-cmn-pll.c b/drivers/clk/qcom/ipq-cmn-pll.c index 5763e4df59a1a..889c176089c2f 100644 --- a/drivers/clk/qcom/ipq-cmn-pll.c +++ b/drivers/clk/qcom/ipq-cmn-pll.c @@ -199,7 +199,7 @@ static unsigned long clk_cmn_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { struct clk_cmn_pll *cmn_pll = to_clk_cmn_pll(hw); - u32 val, factor; + u32 val, factor, ref_div; /* * The value of CMN_PLL_DIVIDER_CTRL_FACTOR is automatically adjusted @@ -207,8 +207,15 @@ static unsigned long clk_cmn_pll_recalc_rate(struct clk_hw *hw, */ regmap_read(cmn_pll->regmap, CMN_PLL_DIVIDER_CTRL, &val); factor = FIELD_GET(CMN_PLL_DIVIDER_CTRL_FACTOR, val); + if (WARN_ON(factor == 0)) + factor = 1; - return parent_rate * 2 * factor; + regmap_read(cmn_pll->regmap, CMN_PLL_REFCLK_CONFIG, &val); + ref_div = FIELD_GET(CMN_PLL_REFCLK_DIV, val); + if (WARN_ON(ref_div == 0)) + ref_div = 1; + + return div_u64((u64)parent_rate * 2 * factor, ref_div); } static int clk_cmn_pll_determine_rate(struct clk_hw *hw, diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c index abfd8634d2bef..c7bb5b97e7c45 100644 --- a/drivers/clk/renesas/rzg2l-cpg.c +++ b/drivers/clk/renesas/rzg2l-cpg.c @@ -1340,10 +1340,10 @@ struct mod_clock { #define to_mod_clock(_hw) container_of(_hw, struct mod_clock, hw) #define for_each_mod_clock(mod_clock, hw, priv) \ - for (unsigned int i = 0; (priv) && i < (priv)->num_mod_clks; i++) \ - if ((priv)->clks[(priv)->num_core_clks + i] == ERR_PTR(-ENOENT)) \ + for (unsigned int __i = 0; (priv) && __i < (priv)->num_mod_clks; __i++) \ + if ((priv)->clks[(priv)->num_core_clks + __i] == ERR_PTR(-ENOENT)) \ continue; \ - else if (((hw) = __clk_get_hw((priv)->clks[(priv)->num_core_clks + i])) && \ + else if (((hw) = __clk_get_hw((priv)->clks[(priv)->num_core_clks + __i])) && \ ((mod_clock) = to_mod_clock(hw))) /* Need to be called with a lock held to avoid concurrent access to mstop->usecnt. */ diff --git a/drivers/clk/samsung/Kconfig b/drivers/clk/samsung/Kconfig index 70a8b82a0136b..198d8b6212893 100644 --- a/drivers/clk/samsung/Kconfig +++ b/drivers/clk/samsung/Kconfig @@ -97,7 +97,7 @@ config EXYNOS_CLKOUT config EXYNOS_ACPM_CLK tristate "Clock driver controlled via ACPM interface" - depends on EXYNOS_ACPM_PROTOCOL || (COMPILE_TEST && !EXYNOS_ACPM_PROTOCOL) + depends on EXYNOS_ACPM_PROTOCOL help This driver provides support for clocks that are controlled by firmware that implements the ACPM interface. diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c index bb8b75bdbdb30..1a53b14739fed 100644 --- a/drivers/clk/spacemit/ccu-k3.c +++ b/drivers/clk/spacemit/ccu-k3.c @@ -947,16 +947,16 @@ static const struct clk_parent_data edp1_pclk_parents[] = { }; CCU_MUX_GATE_DEFINE(edp1_pxclk, edp1_pclk_parents, APMU_LCD_EDP_CTRL, 18, 1, BIT(17), 0); -CCU_GATE_DEFINE(pciea_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_A, BIT(2), 0); -CCU_GATE_DEFINE(pciea_slv_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_A, BIT(1), 0); -CCU_GATE_DEFINE(pcieb_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_B, BIT(2), 0); -CCU_GATE_DEFINE(pcieb_slv_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_B, BIT(1), 0); -CCU_GATE_DEFINE(pciec_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_C, BIT(2), 0); -CCU_GATE_DEFINE(pciec_slv_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_C, BIT(1), 0); -CCU_GATE_DEFINE(pcied_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_D, BIT(2), 0); -CCU_GATE_DEFINE(pcied_slv_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_D, BIT(1), 0); -CCU_GATE_DEFINE(pciee_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_E, BIT(2), 0); -CCU_GATE_DEFINE(pciee_slv_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_E, BIT(1), 0); +CCU_GATE_DEFINE(pciea_mstr_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_A, BIT(2), 0); +CCU_GATE_DEFINE(pciea_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_A, BIT(1), 0); +CCU_GATE_DEFINE(pcieb_mstr_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_B, BIT(2), 0); +CCU_GATE_DEFINE(pcieb_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_B, BIT(1), 0); +CCU_GATE_DEFINE(pciec_mstr_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_C, BIT(2), 0); +CCU_GATE_DEFINE(pciec_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_C, BIT(1), 0); +CCU_GATE_DEFINE(pcied_mstr_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_D, BIT(2), 0); +CCU_GATE_DEFINE(pcied_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_D, BIT(1), 0); +CCU_GATE_DEFINE(pciee_mstr_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_E, BIT(2), 0); +CCU_GATE_DEFINE(pciee_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_E, BIT(1), 0); static const struct clk_parent_data emac_1588_parents[] = { CCU_PARENT_NAME(vctcxo_24m), diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index d1a33a231a44c..d9c76dd443f8a 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -793,4 +793,35 @@ config RTK_SYSTIMER this option only when building for a Realtek platform or for compilation testing. +choice + prompt "NXP clocksource for scheduler clock" + depends on SOC_VF610 || ARCH_S32 + # Default to Global Timer for Vybrid (32-bit) + default VF_USE_ARM_GLOBAL_TIMER if SOC_VF610 + # Default to None for S32 (64-bit) + default VF_TIMER_NONE if ARCH_S32 + + config VF_USE_ARM_GLOBAL_TIMER + bool "Use NXP Vybrid Global Timer" + depends on ARCH_MULTI_V7 && SOC_VF610 + select ARM_GLOBAL_TIMER + select CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK + help + Use the NXP Vybrid Global Timer as clocksource. + + config VF_USE_PIT_TIMER + bool "Use NXP PIT timer" + select NXP_PIT_TIMER + help + Use NXP Periodic Interrupt Timer (PIT) as clocksource. + + config VF_TIMER_NONE + bool "None (Use standard Arch Timer)" + depends on ARCH_S32 + help + Do not use any specific NXP timer driver. Use the standard + ARM Architected Timer instead. + +endchoice + endmenu diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c index f827d3f98f60e..d7e012992170b 100644 --- a/drivers/clocksource/timer-sun5i.c +++ b/drivers/clocksource/timer-sun5i.c @@ -286,6 +286,9 @@ static int sun5i_timer_probe(struct platform_device *pdev) } rstc = devm_reset_control_get_optional_exclusive(dev, NULL); + if (IS_ERR(rstc)) + return dev_err_probe(dev, PTR_ERR(rstc), + "failed to get reset\n"); if (rstc) reset_control_deassert(rstc); diff --git a/drivers/clocksource/timer-tegra186.c b/drivers/clocksource/timer-tegra186.c index 355558893e5f3..bfe16d2d5104e 100644 --- a/drivers/clocksource/timer-tegra186.c +++ b/drivers/clocksource/timer-tegra186.c @@ -149,7 +149,8 @@ static void tegra186_wdt_enable(struct tegra186_wdt *wdt) u32 value; /* unmask hardware IRQ, this may have been lost across powergate */ - value = TKEIE_WDT_MASK(wdt->index, 1); + value = readl(tegra->regs + TKEIE(wdt->tmr->hwirq)); + value |= TKEIE_WDT_MASK(wdt->index, 1); writel(value, tegra->regs + TKEIE(wdt->tmr->hwirq)); /* clear interrupt */ diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 44eb1b7e7fc1b..d41067cd1f1b2 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1972,6 +1972,7 @@ void cpufreq_suspend(void) if (!cpufreq_driver) return; + cpus_read_lock(); if (!has_target() && !cpufreq_driver->suspend) goto suspend; @@ -1991,6 +1992,7 @@ void cpufreq_suspend(void) suspend: cpufreq_suspended = true; + cpus_read_unlock(); } /** diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index df01d33993d82..0b32ae28ec857 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -103,10 +103,6 @@ static unsigned int cs_dbs_update(struct cpufreq_policy *policy) if (load > dbs_data->up_threshold) { dbs_info->down_skip = 0; - /* if we are already at full speed then break out early */ - if (requested_freq == policy->max) - goto out; - requested_freq += freq_step; if (requested_freq > policy->max) requested_freq = policy->max; @@ -124,13 +120,7 @@ static unsigned int cs_dbs_update(struct cpufreq_policy *policy) /* Check for frequency decrease */ if (load < cs_tuners->down_threshold) { - /* - * if we cannot reduce the frequency anymore, break out early - */ - if (requested_freq == policy->min) - goto out; - - if (requested_freq > freq_step) + if (requested_freq > policy->min + freq_step) requested_freq -= freq_step; else requested_freq = policy->min; diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 86f35e4519142..8a85bd32defe7 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -90,7 +90,14 @@ EXPORT_SYMBOL_GPL(sampling_rate_store); * (that may be a single policy or a bunch of them if governor tunables are * system-wide). * - * Call under the @dbs_data mutex. + * Call under the @dbs_data->attr_set.update_lock. The per-policy + * update_mutex is acquired and released internally for each policy. + * + * Note: prev_cpu_nice is reset here unconditionally alongside prev_cpu_idle. + * When io_is_busy changes, both baselines must be advanced to the same + * timestamp so that the next dbs_update() computes idle_time and nice_delta + * over the same interval, preventing an artificially inflated idle_time when + * ignore_nice_load is enabled. */ void gov_update_cpu_data(struct dbs_data *dbs_data) { @@ -99,14 +106,15 @@ void gov_update_cpu_data(struct dbs_data *dbs_data) list_for_each_entry(policy_dbs, &dbs_data->attr_set.policy_list, list) { unsigned int j; + mutex_lock(&policy_dbs->update_mutex); for_each_cpu(j, policy_dbs->policy->cpus) { struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time, dbs_data->io_is_busy); - if (dbs_data->ignore_nice_load) - j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); } + mutex_unlock(&policy_dbs->update_mutex); } } EXPORT_SYMBOL_GPL(gov_update_cpu_data); @@ -118,6 +126,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy) unsigned int ignore_nice = dbs_data->ignore_nice_load; unsigned int max_load = 0, idle_periods = UINT_MAX; unsigned int sampling_rate, io_busy, j; + u64 cur_nice; /* * Sometimes governors may use an additional multiplier to increase @@ -164,12 +173,18 @@ unsigned int dbs_update(struct cpufreq_policy *policy) j_cdbs->prev_cpu_idle = cur_idle_time; - if (ignore_nice) { - u64 cur_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); - + /* + * Always sample cur_nice and advance prev_cpu_nice, regardless + * of ignore_nice. This keeps prev_cpu_nice current so that + * enabling ignore_nice_load via sysfs never produces a + * stale-baseline spike (the delta will be at most one sampling + * interval of accumulated nice time, not since boot). + */ + cur_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + if (ignore_nice) idle_time += div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC); - j_cdbs->prev_cpu_nice = cur_nice; - } + + j_cdbs->prev_cpu_nice = cur_nice; if (unlikely(!time_elapsed)) { /* @@ -516,7 +531,7 @@ int cpufreq_dbs_governor_start(struct cpufreq_policy *policy) struct dbs_governor *gov = dbs_governor_of(policy); struct policy_dbs_info *policy_dbs = policy->governor_data; struct dbs_data *dbs_data = policy_dbs->dbs_data; - unsigned int sampling_rate, ignore_nice, j; + unsigned int sampling_rate, j; unsigned int io_busy; if (!policy->cur) @@ -526,9 +541,9 @@ int cpufreq_dbs_governor_start(struct cpufreq_policy *policy) policy_dbs->rate_mult = 1; sampling_rate = dbs_data->sampling_rate; - ignore_nice = dbs_data->ignore_nice_load; - io_busy = dbs_data->io_is_busy; + mutex_lock(&policy_dbs->update_mutex); + io_busy = dbs_data->io_is_busy; for_each_cpu(j, policy->cpus) { struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); @@ -537,10 +552,9 @@ int cpufreq_dbs_governor_start(struct cpufreq_policy *policy) * Make the first invocation of dbs_update() compute the load. */ j_cdbs->prev_load = 0; - - if (ignore_nice) - j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); } + mutex_unlock(&policy_dbs->update_mutex); gov->start(policy); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 1f093e3464301..3a44d2da08d98 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1058,12 +1058,14 @@ static void hybrid_clear_cpu_capacity(unsigned int cpunum) static void hybrid_get_capacity_perf(struct cpudata *cpu) { + u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached); + if (READ_ONCE(global.no_turbo)) { - cpu->capacity_perf = cpu->pstate.max_pstate_physical; + cpu->capacity_perf = HWP_GUARANTEED_PERF(hwp_cap); return; } - cpu->capacity_perf = HWP_HIGHEST_PERF(READ_ONCE(cpu->hwp_cap_cached)); + cpu->capacity_perf = HWP_HIGHEST_PERF(hwp_cap); } static void hybrid_set_capacity_of_cpus(void) @@ -2984,10 +2986,12 @@ static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy) * from getting to lower performance levels, so force the minimum * performance on CPU offline to prevent that from happening. */ - if (hwp_active) + if (hwp_active) { intel_pstate_hwp_offline(cpu); - else + } else { intel_pstate_set_min_pstate(cpu); + policy->cur = cpu->pstate.min_freq; + } intel_pstate_exit_perf_limits(policy); diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index ac2e90a65f0c4..a355ec4f3dd4c 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -352,6 +352,8 @@ static int __init pcc_cpufreq_do_osc(acpi_handle *handle) } kfree(output.pointer); + output.pointer = NULL; + output.length = ACPI_ALLOCATE_BUFFER; capabilities[0] = 0x0; capabilities[1] = 0x1; diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index ea9a20d27b8fd..ef19faedbfec3 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -578,7 +578,6 @@ static void qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); qcom_cpufreq_hw_lmh_exit(data); kfree(policy->freq_table); - kfree(data); } static void qcom_cpufreq_ready(struct cpufreq_policy *policy) diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index 370664c47e659..e355b42043cf2 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -195,14 +195,6 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv) s->exit_latency_ns = 0; else s->exit_latency = div_u64(s->exit_latency_ns, NSEC_PER_USEC); - - /* - * Warn if the exit latency of a CPU idle state exceeds its - * target residency which is assumed to never happen in cpuidle - * in multiple places. - */ - if (s->exit_latency_ns > s->target_residency_ns) - pr_warn("Idle state %d target residency too low\n", i); } } diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index d23b58b81ca34..8f84cce69fbc9 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -301,7 +301,6 @@ config CRYPTO_DEV_PPC4XX select CRYPTO_CCM select CRYPTO_CTR select CRYPTO_GCM - select CRYPTO_RNG select CRYPTO_SKCIPHER help This option allows you to have support for AMCC crypto acceleration. @@ -722,7 +721,6 @@ config CRYPTO_DEV_XILINX_TRNG tristate "Support for Xilinx True Random Generator" depends on ZYNQMP_FIRMWARE || COMPILE_TEST select CRYPTO_DF80090A - select CRYPTO_RNG select HW_RANDOM help Xilinx Versal SoC driver provides kernel-side support for True Random Number @@ -846,7 +844,6 @@ config CRYPTO_DEV_CCREE If unsure say Y. source "drivers/crypto/hisilicon/Kconfig" -source "drivers/crypto/loongson/Kconfig" source "drivers/crypto/amlogic/Kconfig" diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 283bbc650b5b2..ad773158ae560 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -43,7 +43,6 @@ obj-y += inside-secure/ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-y += xilinx/ obj-y += hisilicon/ -obj-y += loongson/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ obj-y += intel/ obj-y += starfive/ diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 7270e5fbc5738..1048f8e95ba80 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -25,14 +25,6 @@ config CRYPTO_DEV_SUN4I_SS To compile this driver as a module, choose M here: the module will be called sun4i-ss. -config CRYPTO_DEV_SUN4I_SS_PRNG - bool "Support for Allwinner Security System PRNG" - depends on CRYPTO_DEV_SUN4I_SS - select CRYPTO_RNG - help - Select this option if you want to provide kernel-side support for - the Pseudo-Random Number Generator found in the Security System. - config CRYPTO_DEV_SUN4I_SS_DEBUG bool "Enable sun4i-ss stats" depends on CRYPTO_DEV_SUN4I_SS diff --git a/drivers/crypto/allwinner/sun4i-ss/Makefile b/drivers/crypto/allwinner/sun4i-ss/Makefile index c0a2797d31682..06a9ae81f9f80 100644 --- a/drivers/crypto/allwinner/sun4i-ss/Makefile +++ b/drivers/crypto/allwinner/sun4i-ss/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o -sun4i-ss-$(CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG) += sun4i-ss-prng.o diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index 58a76e2ba64e2..35ef0930e77f1 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -213,23 +213,6 @@ static struct sun4i_ss_alg_template ss_algs[] = { } } }, -#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG -{ - .type = CRYPTO_ALG_TYPE_RNG, - .alg.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "sun4i_ss_rng", - .cra_priority = 300, - .cra_ctxsize = 0, - .cra_module = THIS_MODULE, - }, - .generate = sun4i_ss_prng_generate, - .seed = sun4i_ss_prng_seed, - .seedsize = SS_SEED_LEN / BITS_PER_BYTE, - } -}, -#endif }; static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v) @@ -247,12 +230,6 @@ static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v) ss_algs[i].stat_req, ss_algs[i].stat_opti, ss_algs[i].stat_fb, ss_algs[i].stat_bytes); break; - case CRYPTO_ALG_TYPE_RNG: - seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", - ss_algs[i].alg.rng.base.cra_driver_name, - ss_algs[i].alg.rng.base.cra_name, - ss_algs[i].stat_req, ss_algs[i].stat_bytes); - break; case CRYPTO_ALG_TYPE_AHASH: seq_printf(seq, "%s %s reqs=%lu\n", ss_algs[i].alg.hash.halg.base.cra_driver_name, @@ -471,13 +448,6 @@ static int sun4i_ss_probe(struct platform_device *pdev) goto error_alg; } break; - case CRYPTO_ALG_TYPE_RNG: - err = crypto_register_rng(&ss_algs[i].alg.rng); - if (err) { - dev_err(ss->dev, "Fail to register %s\n", - ss_algs[i].alg.rng.base.cra_name); - } - break; } } @@ -497,9 +467,6 @@ static int sun4i_ss_probe(struct platform_device *pdev) case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); break; - case CRYPTO_ALG_TYPE_RNG: - crypto_unregister_rng(&ss_algs[i].alg.rng); - break; } } error_pm: @@ -520,9 +487,6 @@ static void sun4i_ss_remove(struct platform_device *pdev) case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); break; - case CRYPTO_ALG_TYPE_RNG: - crypto_unregister_rng(&ss_algs[i].alg.rng); - break; } } diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c deleted file mode 100644 index 491fcb7b81b40..0000000000000 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -#include "sun4i-ss.h" - -int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct sun4i_ss_alg_template *algt; - struct rng_alg *alg = crypto_rng_alg(tfm); - - algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng); - memcpy(algt->ss->seed, seed, slen); - - return 0; -} - -int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen) -{ - struct sun4i_ss_alg_template *algt; - struct rng_alg *alg = crypto_rng_alg(tfm); - int i, err; - u32 v; - u32 *data = (u32 *)dst; - const u32 mode = SS_OP_PRNG | SS_PRNG_CONTINUE | SS_ENABLED; - size_t len; - struct sun4i_ss_ctx *ss; - unsigned int todo = (dlen / 4) * 4; - - algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng); - ss = algt->ss; - - err = pm_runtime_resume_and_get(ss->dev); - if (err < 0) - return err; - - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { - algt->stat_req++; - algt->stat_bytes += todo; - } - - spin_lock_bh(&ss->slock); - - writel(mode, ss->base + SS_CTL); - - while (todo > 0) { - /* write the seed */ - for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) - writel(ss->seed[i], ss->base + SS_KEY0 + i * 4); - - /* Read the random data */ - len = min_t(size_t, SS_DATA_LEN / BITS_PER_BYTE, todo); - readsl(ss->base + SS_TXFIFO, data, len / 4); - data += len / 4; - todo -= len; - - /* Update the seed */ - for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) { - v = readl(ss->base + SS_KEY0 + i * 4); - ss->seed[i] = v; - } - } - - writel(0, ss->base + SS_CTL); - spin_unlock_bh(&ss->slock); - - pm_runtime_put(ss->dev); - - return 0; -} diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 6c5d4aa6453c7..f7d1c79ac677d 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -31,8 +31,6 @@ #include #include #include -#include -#include #define SS_CTL 0x00 #define SS_KEY0 0x04 @@ -62,10 +60,6 @@ /* SS_CTL configuration values */ -/* PRNG generator mode - bit 15 */ -#define SS_PRNG_ONESHOT (0 << 15) -#define SS_PRNG_CONTINUE (1 << 15) - /* IV mode for hash */ #define SS_IV_ARBITRARY (1 << 14) @@ -94,14 +88,10 @@ #define SS_OP_3DES (2 << 4) #define SS_OP_SHA1 (3 << 4) #define SS_OP_MD5 (4 << 4) -#define SS_OP_PRNG (5 << 4) /* Data end bit - bit 2 */ #define SS_DATA_END (1 << 2) -/* PRNG start bit - bit 1 */ -#define SS_PRNG_START (1 << 1) - /* SS Enable bit - bit 0 */ #define SS_DISABLED (0 << 0) #define SS_ENABLED (1 << 0) @@ -128,9 +118,6 @@ #define SS_RXFIFO_EMP_INT_ENABLE (1 << 2) #define SS_TXFIFO_AVA_INT_ENABLE (1 << 0) -#define SS_SEED_LEN 192 -#define SS_DATA_LEN 160 - /* * struct ss_variant - Describe SS hardware variant * @sha1_in_be: The SHA1 digest is given by SS in BE, and so need to be inverted. @@ -151,9 +138,6 @@ struct sun4i_ss_ctx { char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ spinlock_t slock; /* control the use of the device */ -#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG - u32 seed[SS_SEED_LEN / BITS_PER_LONG]; -#endif struct dentry *dbgfs_dir; struct dentry *dbgfs_stats; }; @@ -164,7 +148,6 @@ struct sun4i_ss_alg_template { union { struct skcipher_alg crypto; struct ahash_alg hash; - struct rng_alg rng; } alg; struct sun4i_ss_ctx *ss; unsigned long stat_req; @@ -231,6 +214,3 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); -int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen); -int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index b7b6c97d2147b..68c5ff7a85b4f 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -31,11 +31,9 @@ #include #include #include -#include #include #include #include -#include #include #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" @@ -985,10 +983,6 @@ static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, rc = crypto_register_aead(&alg->alg.u.aead); break; - case CRYPTO_ALG_TYPE_RNG: - rc = crypto_register_rng(&alg->alg.u.rng); - break; - default: rc = crypto_register_skcipher(&alg->alg.u.cipher); break; @@ -1014,10 +1008,6 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) crypto_unregister_aead(&alg->alg.u.aead); break; - case CRYPTO_ALG_TYPE_RNG: - crypto_unregister_rng(&alg->alg.u.rng); - break; - default: crypto_unregister_skcipher(&alg->alg.u.cipher); } @@ -1076,69 +1066,6 @@ static irqreturn_t crypto4xx_ce_interrupt_handler_revb(int irq, void *data) PPC4XX_TMO_ERR_INT); } -static int ppc4xx_prng_data_read(struct crypto4xx_device *dev, - u8 *data, unsigned int max) -{ - unsigned int i, curr = 0; - u32 val[2]; - - do { - /* trigger PRN generation */ - writel(PPC4XX_PRNG_CTRL_AUTO_EN, - dev->ce_base + CRYPTO4XX_PRNG_CTRL); - - for (i = 0; i < 1024; i++) { - /* usually 19 iterations are enough */ - if ((readl(dev->ce_base + CRYPTO4XX_PRNG_STAT) & - CRYPTO4XX_PRNG_STAT_BUSY)) - continue; - - val[0] = readl_be(dev->ce_base + CRYPTO4XX_PRNG_RES_0); - val[1] = readl_be(dev->ce_base + CRYPTO4XX_PRNG_RES_1); - break; - } - if (i == 1024) - return -ETIMEDOUT; - - if ((max - curr) >= 8) { - memcpy(data, &val, 8); - data += 8; - curr += 8; - } else { - /* copy only remaining bytes */ - memcpy(data, &val, max - curr); - break; - } - } while (curr < max); - - return curr; -} - -static int crypto4xx_prng_generate(struct crypto_rng *tfm, - const u8 *src, unsigned int slen, - u8 *dstn, unsigned int dlen) -{ - struct rng_alg *alg = crypto_rng_alg(tfm); - struct crypto4xx_alg *amcc_alg; - struct crypto4xx_device *dev; - int ret; - - amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.rng); - dev = amcc_alg->dev; - - mutex_lock(&dev->core_dev->rng_lock); - ret = ppc4xx_prng_data_read(dev, dstn, dlen); - mutex_unlock(&dev->core_dev->rng_lock); - return ret; -} - - -static int crypto4xx_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - return 0; -} - /* * Supported Crypto Algorithms */ @@ -1268,18 +1195,6 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_module = THIS_MODULE, }, } }, - { .type = CRYPTO_ALG_TYPE_RNG, .u.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "crypto4xx_rng", - .cra_priority = 300, - .cra_ctxsize = 0, - .cra_module = THIS_MODULE, - }, - .generate = crypto4xx_prng_generate, - .seed = crypto4xx_prng_seed, - .seedsize = 0, - } }, }; /* @@ -1353,9 +1268,6 @@ static int crypto4xx_probe(struct platform_device *ofdev) core_dev->dev->core_dev = core_dev; core_dev->dev->is_revb = is_revb; core_dev->device = dev; - rc = devm_mutex_init(&ofdev->dev, &core_dev->rng_lock); - if (rc) - return rc; spin_lock_init(&core_dev->lock); INIT_LIST_HEAD(&core_dev->dev->alg_list); ratelimit_default_init(&core_dev->dev->aead_ratelimit); diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index ee36630c670f2..3a028aec3f0c1 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -14,10 +14,8 @@ #define __CRYPTO4XX_CORE_H__ #include -#include #include #include -#include #include #include "crypto4xx_reg_def.h" #include "crypto4xx_sa.h" @@ -111,7 +109,6 @@ struct crypto4xx_core_device { u32 irq; struct tasklet_struct tasklet; spinlock_t lock; - struct mutex rng_lock; }; struct crypto4xx_ctx { @@ -135,7 +132,6 @@ struct crypto4xx_alg_common { union { struct skcipher_alg cipher; struct aead_alg aead; - struct rng_alg rng; } u; }; diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h index 1038061224da6..73d626308a848 100644 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -90,20 +90,9 @@ #define CRYPTO4XX_BYTE_ORDER_CFG 0x000600d8 #define CRYPTO4XX_ENDIAN_CFG 0x000600d8 -#define CRYPTO4XX_PRNG_STAT 0x00070000 -#define CRYPTO4XX_PRNG_STAT_BUSY 0x1 #define CRYPTO4XX_PRNG_CTRL 0x00070004 #define CRYPTO4XX_PRNG_SEED_L 0x00070008 #define CRYPTO4XX_PRNG_SEED_H 0x0007000c - -#define CRYPTO4XX_PRNG_RES_0 0x00070020 -#define CRYPTO4XX_PRNG_RES_1 0x00070024 -#define CRYPTO4XX_PRNG_RES_2 0x00070028 -#define CRYPTO4XX_PRNG_RES_3 0x0007002C - -#define CRYPTO4XX_PRNG_LFSR_L 0x00070030 -#define CRYPTO4XX_PRNG_LFSR_H 0x00070034 - /* * Initialize CRYPTO ENGINE registers, and memory bases. */ diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c index 1c18a5b8470e0..6cb33949915fe 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-core.c +++ b/drivers/crypto/amlogic/amlogic-gxl-core.c @@ -291,8 +291,8 @@ static int meson_crypto_probe(struct platform_device *pdev) return 0; error_alg: meson_unregister_algs(mc); -error_flow: meson_free_chanlist(mc, MAXFLOW - 1); +error_flow: clk_disable_unprepare(mc->busclk); return err; } diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index dbb39ed0cea11..aa1db7ac2bb2a 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -19,6 +19,12 @@ #include #include "atmel-i2c.h" +/* + * According to review by Bill Cox [1], the ATSHA204 has very low entropy. + * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html + */ +static const unsigned short atsha204_quality = 1; + static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, void *areq, int status) { @@ -48,8 +54,8 @@ static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data, if (rng->priv) { work_data = (struct atmel_i2c_work_data *)rng->priv; - max = min(sizeof(work_data->cmd.data), max); - memcpy(data, &work_data->cmd.data, max); + max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max); + memcpy(data, &work_data->cmd.data[RSP_DATA_IDX], max); rng->priv = 0; } else { work_data = kmalloc_obj(*work_data, GFP_ATOMIC); @@ -87,8 +93,8 @@ static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max, if (ret) return ret; - max = min(sizeof(cmd.data), max); - memcpy(data, cmd.data, max); + max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max); + memcpy(data, &cmd.data[RSP_DATA_IDX], max); return max; } @@ -158,6 +164,7 @@ static const struct attribute_group atmel_sha204a_groups = { static int atmel_sha204a_probe(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv; + const unsigned short *quality; int ret; ret = atmel_i2c_probe(client); @@ -171,19 +178,19 @@ static int atmel_sha204a_probe(struct i2c_client *client) i2c_priv->hwrng.name = dev_name(&client->dev); i2c_priv->hwrng.read = atmel_sha204a_rng_read; - /* - * According to review by Bill Cox [1], this HWRNG has very low entropy. - * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html - */ - i2c_priv->hwrng.quality = 1; + quality = i2c_get_match_data(client); + if (quality) + i2c_priv->hwrng.quality = *quality; ret = devm_hwrng_register(&client->dev, &i2c_priv->hwrng); - if (ret) - dev_warn(&client->dev, "failed to register RNG (%d)\n", ret); + if (ret) { + dev_err(&client->dev, "failed to register RNG (%d)\n", ret); + return ret; + } ret = sysfs_create_group(&client->dev.kobj, &atmel_sha204a_groups); if (ret) { - dev_err(&client->dev, "failed to register sysfs entry\n"); + dev_err(&client->dev, "failed to create sysfs group (%d)\n", ret); return ret; } @@ -203,14 +210,14 @@ static void atmel_sha204a_remove(struct i2c_client *client) } static const struct of_device_id atmel_sha204a_dt_ids[] __maybe_unused = { - { .compatible = "atmel,atsha204", }, + { .compatible = "atmel,atsha204", .data = &atsha204_quality }, { .compatible = "atmel,atsha204a", }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids); static const struct i2c_device_id atmel_sha204a_id[] = { - { "atsha204" }, + { "atsha204", (kernel_ulong_t)&atsha204_quality }, { "atsha204a" }, { /* sentinel */ } }; diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 32a6e6e15ee25..ddbd60cf3b3c1 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -603,7 +603,7 @@ static int aead_setkey(struct crypto_aead *aead, dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* @@ -639,7 +639,7 @@ static int aead_setkey(struct crypto_aead *aead, dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); - print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); @@ -680,7 +680,7 @@ static int gcm_setkey(struct crypto_aead *aead, if (err) return err; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -701,7 +701,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (err) return err; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -727,7 +727,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (err) return err; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -754,7 +754,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, u32 *desc; const bool is_rfc3686 = alg->caam.rfc3686; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* Here keylen is actual key length */ diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 65f6adb6c673f..aa779caacfe59 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -212,7 +212,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* @@ -248,7 +248,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); - print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); @@ -371,7 +371,7 @@ static int gcm_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -475,7 +475,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -581,7 +581,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -631,7 +631,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, const bool is_rfc3686 = alg->caam.rfc3686; int ret = 0; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); ctx->cdata.keylen = keylen; diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index bf10c3dda745a..6b47bcc16a506 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -301,7 +301,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); ctx->adata.keylen = keys.authkeylen; @@ -315,7 +315,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); - print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); @@ -732,7 +732,7 @@ static int gcm_setkey(struct crypto_aead *aead, ret = aes_check_keylen(keylen); if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -828,7 +828,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -927,7 +927,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -955,7 +955,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, u32 *desc; const bool is_rfc3686 = alg->caam.rfc3686; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); ctx->cdata.keylen = keylen; diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index ddb2a35aec2d5..3cd71810380f6 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -505,7 +505,7 @@ static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key, DMA_TO_DEVICE); ctx->adata.keylen = keylen; - print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ", + print_hex_dump_devel("axcbc ctx.key@" __stringify(__LINE__)" : ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1); return axcbc_set_sh_desc(ahash); @@ -525,7 +525,7 @@ static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key, ctx->adata.key_virt = key; ctx->adata.keylen = keylen; - print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ", + print_hex_dump_devel("acmac ctx.key@" __stringify(__LINE__)" : ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); return acmac_set_sh_desc(ahash); diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index 88cc4fe2a585d..de2fcc387477d 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -58,7 +58,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out, dev_dbg(jrdev, "split keylen %d split keylen padded %d\n", adata->keylen, adata->keylen_pad); - print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); if (local_max > max_keylen) @@ -113,7 +113,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out, wait_for_completion(&result.completion); ret = result.err; - print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key_out, adata->keylen_pad, 1); } diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index e183b60277ff1..de305cbeccbe0 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -108,8 +108,8 @@ static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list, sg_cleanup: for (j = 0; j < i; j++) { if (list[j].dma_addr) { - dma_unmap_single(&pdev->dev, list[i].dma_addr, - list[i].size, DMA_BIDIRECTIONAL); + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); } list[j].dma_addr = 0; diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 5c7f7e02a7d8a..b14ce51065d5d 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -316,15 +316,15 @@ void psp_dev_destroy(struct sp_device *sp) if (!psp) return; - sev_dev_destroy(psp); + dbc_dev_destroy(psp); - tee_dev_destroy(psp); + platform_access_dev_destroy(psp); sfs_dev_destroy(psp); - dbc_dev_destroy(psp); + tee_dev_destroy(psp); - platform_access_dev_destroy(psp); + sev_dev_destroy(psp); sp_free_psp_irq(sp, psp); diff --git a/drivers/crypto/ccp/sev-dev-tsm.c b/drivers/crypto/ccp/sev-dev-tsm.c index b07ae529b5910..46f2539d2d5ab 100644 --- a/drivers/crypto/ccp/sev-dev-tsm.c +++ b/drivers/crypto/ccp/sev-dev-tsm.c @@ -58,13 +58,13 @@ static int stream_enable(struct pci_ide *ide) struct pci_dev *rp = pcie_find_root_port(ide->pdev); int ret; - ret = pci_ide_stream_enable(rp, ide); - if (ret) + ret = pci_ide_stream_enable(ide->pdev, ide); + if (ret && ret != -ENXIO) return ret; - ret = pci_ide_stream_enable(ide->pdev, ide); - if (ret) - pci_ide_stream_disable(rp, ide); + ret = pci_ide_stream_enable(rp, ide); + if (ret && ret != -ENXIO) + pci_ide_stream_disable(ide->pdev, ide); return ret; } @@ -248,12 +248,19 @@ static void dsm_remove(struct pci_tsm *tsm) static int dsm_create(struct tio_dsm *dsm) { struct pci_dev *pdev = dsm->tsm.base_tsm.pdev; - u8 segment_id = pdev->bus ? pci_domain_nr(pdev->bus) : 0; - struct pci_dev *rootport = pcie_find_root_port(pdev); - u16 device_id = pci_dev_id(pdev); + struct pci_dev *rootport; + u8 segment_id; + u16 device_id; u16 root_port_id; u32 lnkcap = 0; + if (!pdev->bus) + return -ENODEV; + + segment_id = pci_domain_nr(pdev->bus); + rootport = pcie_find_root_port(pdev); + device_id = pci_dev_id(pdev); + if (pci_read_config_dword(rootport, pci_pcie_cap(rootport) + PCI_EXP_LNKCAP, &lnkcap)) return -ENODEV; diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index d1e9e0ac63b60..068b901034cbc 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1328,10 +1328,11 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) size_t size; /* - * Ensure the list of HV_FIXED pages that will be passed to firmware - * do not exceed the page-sized argument buffer. + * Ensure the list of HV_FIXED pages passed to the firmware including + * the one about to be written to do not exceed the page-sized argument + * buffer. */ - if ((range_list->num_elements * sizeof(struct sev_data_range) + + if (((range_list->num_elements + 1) * sizeof(struct sev_data_range) + sizeof(struct sev_data_range_list)) > PAGE_SIZE) return -E2BIG; @@ -1355,7 +1356,7 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { struct sev_data_range_list *snp_range_list __free(kfree) = NULL; struct psp_device *psp = psp_master; - struct sev_data_snp_init_ex data; + struct sev_data_snp_init_ex data = {}; struct sev_device *sev; void *arg = &data; int cmd, rc = 0; @@ -1419,8 +1420,6 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) */ snp_add_hv_fixed_pages(sev, snp_range_list); - memset(&data, 0, sizeof(data)); - if (max_snp_asid) { data.ciphertext_hiding_en = 1; data.max_snp_asid = max_snp_asid; @@ -1487,6 +1486,8 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) &snp_panic_notifier); if (data.tio_en) { + struct page *page; + /* * This executes with the sev_cmd_mutex held so down the stack * snp_reclaim_pages(locked=false) might be needed (which is extremely @@ -1494,12 +1495,14 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) * Instead of exporting __snp_alloc_firmware_pages(), allocate a page * for this one call here. */ - void *tio_status = page_address(__snp_alloc_firmware_pages( - GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0, true)); + page = __snp_alloc_firmware_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, + 0, true); + if (page) { + void *tio_status = page_address(page); - if (tio_status) { sev_tsm_init_locked(sev, tio_status); - __snp_free_firmware_pages(virt_to_page(tio_status), 0, true); + + __snp_free_firmware_pages(page, 0, true); } } @@ -1716,29 +1719,11 @@ static int sev_get_platform_state(int *state, int *error) static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) { - struct sev_platform_init_args init_args = {0}; int rc; - rc = _sev_platform_init_locked(&init_args); - if (rc) { - argp->error = SEV_RET_INVALID_PLATFORM_STATE; - return rc; - } - - *shutdown_required = true; - - return 0; -} - -static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) -{ - int error, rc; - - rc = __sev_snp_init_locked(&error, 0); - if (rc) { - argp->error = SEV_RET_INVALID_PLATFORM_STATE; + rc = __sev_platform_init_locked(&argp->error); + if (rc) return rc; - } *shutdown_required = true; @@ -2301,7 +2286,8 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) /* Userspace wants to query the certificate length. */ if (!input.pdh_cert_address || !input.pdh_cert_len || - !input.cert_chain_address) + !input.cert_chain_address || + !input.cert_chain_len) goto cmd; /* Allocate a physically contiguous buffer to store the PDH blob. */ @@ -2440,24 +2426,13 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp) { - struct sev_device *sev = psp_master->sev_data; struct sev_data_snp_commit buf; - bool shutdown_required = false; - int ret, error; - - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - return ret; - } + int ret; buf.len = sizeof(buf); ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - return ret; } @@ -2465,8 +2440,6 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_config config; - bool shutdown_required = false; - int ret, error; if (!argp->data) return -EINVAL; @@ -2474,30 +2447,21 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable if (!writable) return -EPERM; + if (!sev->snp_initialized) + return -ENODEV; + if (copy_from_user(&config, (void __user *)argp->data, sizeof(config))) return -EFAULT; - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - return ret; - } - - ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); - - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - - return ret; + return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); } static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_vlek_load input; - bool shutdown_required = false; - int ret, error; void *blob; + int ret; if (!argp->data) return -EINVAL; @@ -2505,6 +2469,9 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) if (!writable) return -EPERM; + if (!sev->snp_initialized) + return -ENODEV; + if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input))) return -EFAULT; @@ -2518,18 +2485,7 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) input.vlek_wrapped_address = __psp_pa(blob); - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - goto cleanup; - } - ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error); - - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - -cleanup: kfree(blob); return ret; diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 1e6d772f4bb65..8aa23c9397759 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -80,6 +80,5 @@ config CRYPTO_DEV_HISI_TRNG tristate "Support for HISI TRNG Driver" depends on ARM64 && ACPI select HW_RANDOM - select CRYPTO_RNG help Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 3ca47e2a9719f..26e2ccb70c2ae 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4917,8 +4917,6 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) u32 delay = 0; int ret; - hisi_qm_dev_err_uninit(pf_qm); - /* * Check whether there is an ECC mbit error, If it occurs, need to * wait for soft reset to fix it. @@ -4935,6 +4933,8 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) return; } + hisi_qm_dev_err_uninit(pf_qm); + /* PF obtains the information of VF by querying the register. */ if (qm->fun_type == QM_HW_PF) qm_cmd_uninit(qm); diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index 5ca0b90859a81..6584ed051e099 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -1,234 +1,27 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2019 HiSilicon Limited. */ -#include #include -#include #include #include #include #include #include -#include #include -#include #include #include #define HISI_TRNG_REG 0x00F0 #define HISI_TRNG_BYTES 4 #define HISI_TRNG_QUALITY 512 -#define HISI_TRNG_VERSION 0x01B8 -#define HISI_TRNG_VER_V1 GENMASK(31, 0) #define SLEEP_US 10 #define TIMEOUT_US 10000 -#define SW_DRBG_NUM_SHIFT 2 -#define SW_DRBG_KEY_BASE 0x082C -#define SW_DRBG_SEED(n) (SW_DRBG_KEY_BASE - ((n) << SW_DRBG_NUM_SHIFT)) -#define SW_DRBG_SEED_REGS_NUM 12 -#define SW_DRBG_SEED_SIZE 48 -#define SW_DRBG_BLOCKS 0x0830 -#define SW_DRBG_INIT 0x0834 -#define SW_DRBG_GEN 0x083c -#define SW_DRBG_STATUS 0x0840 -#define SW_DRBG_BLOCKS_NUM 4095 -#define SW_DRBG_DATA_BASE 0x0850 -#define SW_DRBG_DATA_NUM 4 -#define SW_DRBG_DATA(n) (SW_DRBG_DATA_BASE - ((n) << SW_DRBG_NUM_SHIFT)) -#define SW_DRBG_BYTES 16 -#define SW_DRBG_ENABLE_SHIFT 12 -#define SEED_SHIFT_24 24 -#define SEED_SHIFT_16 16 -#define SEED_SHIFT_8 8 -#define SW_MAX_RANDOM_BYTES 65520 - -struct hisi_trng_list { - struct mutex lock; - struct list_head list; - bool is_init; -}; struct hisi_trng { void __iomem *base; - struct hisi_trng_list *trng_list; - struct list_head list; struct hwrng rng; - u32 ver; - u32 ctx_num; - /* The bytes of the random number generated since the last seeding. */ - u32 random_bytes; - struct mutex lock; -}; - -struct hisi_trng_ctx { - struct hisi_trng *trng; }; -static atomic_t trng_active_devs; -static struct hisi_trng_list trng_devices; -static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait); - -static int hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) -{ - u32 val, seed_reg, i; - int ret; - - writel(0x0, trng->base + SW_DRBG_BLOCKS); - - for (i = 0; i < SW_DRBG_SEED_SIZE; - i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { - val = seed[i] << SEED_SHIFT_24; - val |= seed[i + 1UL] << SEED_SHIFT_16; - val |= seed[i + 2UL] << SEED_SHIFT_8; - val |= seed[i + 3UL]; - - seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; - writel(val, trng->base + SW_DRBG_SEED(seed_reg)); - } - - writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), - trng->base + SW_DRBG_BLOCKS); - writel(0x1, trng->base + SW_DRBG_INIT); - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(0), SLEEP_US, TIMEOUT_US); - if (ret) { - pr_err("failed to init trng(%d)\n", ret); - return -EIO; - } - - trng->random_bytes = 0; - - return 0; -} - -static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; - int ret; - - if (slen < SW_DRBG_SEED_SIZE) { - pr_err("slen(%u) is not matched with trng(%d)\n", slen, - SW_DRBG_SEED_SIZE); - return -EINVAL; - } - - mutex_lock(&trng->lock); - ret = hisi_trng_set_seed(trng, seed); - mutex_unlock(&trng->lock); - - return ret; -} - -static int hisi_trng_reseed(struct hisi_trng *trng) -{ - u8 seed[SW_DRBG_SEED_SIZE]; - int size; - - if (!trng->random_bytes) - return 0; - - size = hisi_trng_read(&trng->rng, seed, SW_DRBG_SEED_SIZE, false); - if (size != SW_DRBG_SEED_SIZE) - return -EIO; - - return hisi_trng_set_seed(trng, seed); -} - -static int hisi_trng_get_bytes(struct hisi_trng *trng, u8 *dstn, unsigned int dlen) -{ - u32 data[SW_DRBG_DATA_NUM]; - u32 currsize = 0; - u32 val = 0; - int ret; - u32 i; - - ret = hisi_trng_reseed(trng); - if (ret) - return ret; - - do { - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(1), SLEEP_US, TIMEOUT_US); - if (ret) { - pr_err("failed to generate random number(%d)!\n", ret); - break; - } - - for (i = 0; i < SW_DRBG_DATA_NUM; i++) - data[i] = readl(trng->base + SW_DRBG_DATA(i)); - - if (dlen - currsize >= SW_DRBG_BYTES) { - memcpy(dstn + currsize, data, SW_DRBG_BYTES); - currsize += SW_DRBG_BYTES; - } else { - memcpy(dstn + currsize, data, dlen - currsize); - currsize = dlen; - } - - trng->random_bytes += SW_DRBG_BYTES; - writel(0x1, trng->base + SW_DRBG_GEN); - } while (currsize < dlen); - - return ret; -} - -static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) -{ - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; - unsigned int currsize = 0; - unsigned int block_size; - int ret; - - if (!dstn || !dlen) { - pr_err("output is error, dlen %u!\n", dlen); - return -EINVAL; - } - - do { - block_size = min_t(unsigned int, dlen - currsize, SW_MAX_RANDOM_BYTES); - mutex_lock(&trng->lock); - ret = hisi_trng_get_bytes(trng, dstn + currsize, block_size); - mutex_unlock(&trng->lock); - if (ret) - return ret; - currsize += block_size; - } while (currsize < dlen); - - return 0; -} - -static int hisi_trng_init(struct crypto_tfm *tfm) -{ - struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); - struct hisi_trng *trng; - u32 ctx_num = ~0; - - mutex_lock(&trng_devices.lock); - list_for_each_entry(trng, &trng_devices.list, list) { - if (trng->ctx_num < ctx_num) { - ctx_num = trng->ctx_num; - ctx->trng = trng; - } - } - ctx->trng->ctx_num++; - mutex_unlock(&trng_devices.lock); - - return 0; -} - -static void hisi_trng_exit(struct crypto_tfm *tfm) -{ - struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); - - mutex_lock(&trng_devices.lock); - ctx->trng->ctx_num--; - mutex_unlock(&trng_devices.lock); -} - static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) { struct hisi_trng *trng; @@ -260,42 +53,6 @@ static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) return currsize; } -static struct rng_alg hisi_trng_alg = { - .generate = hisi_trng_generate, - .seed = hisi_trng_seed, - .seedsize = SW_DRBG_SEED_SIZE, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "hisi_stdrng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct hisi_trng_ctx), - .cra_module = THIS_MODULE, - .cra_init = hisi_trng_init, - .cra_exit = hisi_trng_exit, - }, -}; - -static void hisi_trng_add_to_list(struct hisi_trng *trng) -{ - mutex_lock(&trng_devices.lock); - list_add_tail(&trng->list, &trng_devices.list); - mutex_unlock(&trng_devices.lock); -} - -static int hisi_trng_del_from_list(struct hisi_trng *trng) -{ - int ret = -EBUSY; - - mutex_lock(&trng_devices.lock); - if (!trng->ctx_num) { - list_del(&trng->list); - ret = 0; - } - mutex_unlock(&trng_devices.lock); - - return ret; -} - static int hisi_trng_probe(struct platform_device *pdev) { struct hisi_trng *trng; @@ -305,68 +62,20 @@ static int hisi_trng_probe(struct platform_device *pdev) if (!trng) return -ENOMEM; - platform_set_drvdata(pdev, trng); - trng->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(trng->base)) return PTR_ERR(trng->base); - trng->ctx_num = 0; - trng->random_bytes = SW_MAX_RANDOM_BYTES; - mutex_init(&trng->lock); - trng->ver = readl(trng->base + HISI_TRNG_VERSION); - if (!trng_devices.is_init) { - INIT_LIST_HEAD(&trng_devices.list); - mutex_init(&trng_devices.lock); - trng_devices.is_init = true; - } - - hisi_trng_add_to_list(trng); - if (trng->ver != HISI_TRNG_VER_V1 && - atomic_inc_return(&trng_active_devs) == 1) { - ret = crypto_register_rng(&hisi_trng_alg); - if (ret) { - dev_err(&pdev->dev, - "failed to register crypto(%d)\n", ret); - atomic_dec_return(&trng_active_devs); - goto err_remove_from_list; - } - } - trng->rng.name = pdev->name; trng->rng.read = hisi_trng_read; trng->rng.quality = HISI_TRNG_QUALITY; + ret = devm_hwrng_register(&pdev->dev, &trng->rng); - if (ret) { + if (ret) dev_err(&pdev->dev, "failed to register hwrng: %d!\n", ret); - goto err_crypto_unregister; - } - - return ret; - -err_crypto_unregister: - if (trng->ver != HISI_TRNG_VER_V1 && - atomic_dec_return(&trng_active_devs) == 0) - crypto_unregister_rng(&hisi_trng_alg); - -err_remove_from_list: - hisi_trng_del_from_list(trng); return ret; } -static void hisi_trng_remove(struct platform_device *pdev) -{ - struct hisi_trng *trng = platform_get_drvdata(pdev); - - /* Wait until the task is finished */ - while (hisi_trng_del_from_list(trng)) - ; - - if (trng->ver != HISI_TRNG_VER_V1 && - atomic_dec_return(&trng_active_devs) == 0) - crypto_unregister_rng(&hisi_trng_alg); -} - static const struct acpi_device_id hisi_trng_acpi_match[] = { { "HISI02B3", 0 }, { } @@ -375,7 +84,6 @@ MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); static struct platform_driver hisi_trng_driver = { .probe = hisi_trng_probe, - .remove = hisi_trng_remove, .driver = { .name = "hisi-trng-v2", .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c index 7dccfdeb7b11c..276839e1a515f 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-main.c +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -433,6 +433,8 @@ static int eip93_crypto_probe(struct platform_device *pdev) ret = devm_request_threaded_irq(eip93->dev, eip93->irq, eip93_irq_handler, NULL, IRQF_ONESHOT, dev_name(eip93->dev), eip93); + if (ret) + return ret; eip93->ring = devm_kcalloc(eip93->dev, 1, sizeof(*eip93->ring), GFP_KERNEL); if (!eip93->ring) diff --git a/drivers/crypto/inside-secure/eip93/eip93-regs.h b/drivers/crypto/inside-secure/eip93/eip93-regs.h index 96285ca6fbbe8..96d28c6651bdb 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-regs.h +++ b/drivers/crypto/inside-secure/eip93/eip93-regs.h @@ -103,7 +103,7 @@ #define EIP93_PE_TARGET_COMMAND_NO_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x2) #define EIP93_PE_TARGET_COMMAND_WITH_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x1) #define EIP93_PE_DIRECT_HOST_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x0) -#define EIP93_PE_CONFIG_RST_RING BIT(2) +#define EIP93_PE_CONFIG_RST_RING BIT(1) #define EIP93_PE_CONFIG_RST_PE BIT(0) #define EIP93_REG_PE_STATUS 0x104 #define EIP93_REG_PE_BUF_THRESH 0x10c diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index fb4936e7afa2d..2bd8641a07b36 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1893,7 +1893,7 @@ static int safexcel_pci_probe(struct pci_dev *pdev, ent->vendor, ent->device, ent->subvendor, ent->subdevice, ent->driver_data); - priv = kzalloc_obj(*priv); + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index cfa00daeb4fba..3acbbb586dc0c 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -146,8 +146,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } - pci_set_master(pdev); - if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state.\n"); ret = -ENOMEM; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index c9be5dcddb277..f33182c101e2e 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -148,8 +148,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } - pci_set_master(pdev); - if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state.\n"); ret = -ENOMEM; diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c index c52462a48c340..ab62b91ecb510 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -189,8 +189,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } - pci_set_master(pdev); - /* * The PCI config space is saved at this point and will be restored * after a Function Level Reset (FLR) as the FLR does not completely diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index bceb5dd8b148c..c745397e63a52 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -167,7 +167,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_free_reg; } } - pci_set_master(pdev); if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c index c622793e94a87..b31c58fdfb58e 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c @@ -163,7 +163,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_free_reg; } } - pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index 23ccb72b6ea22..941400c64d4be 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -167,7 +167,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_free_reg; } } - pci_set_master(pdev); if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c index 4840d44bbd5b2..631eb2b411506 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c @@ -163,7 +163,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_free_reg; } } - pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 03a4e96902080..58cf98fadb6ca 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -28,15 +28,10 @@ #define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_420XX_DEVICE_NAME "420xx" #define ADF_6XXX_DEVICE_NAME "6xxx" -#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 #define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941 -#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 #define PCI_DEVICE_ID_INTEL_QAT_401XXIOV 0x4943 -#define PCI_DEVICE_ID_INTEL_QAT_402XX 0x4944 #define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 -#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 #define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 -#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 #define PCI_DEVICE_ID_INTEL_QAT_6XXX_IOV 0x4949 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 @@ -480,6 +475,8 @@ struct adf_accel_dev { struct { /* protects VF2PF interrupts access */ spinlock_t vf2pf_ints_lock; + /* prevents VF2PF handling from racing with VF state teardown */ + bool vf2pf_disabled; /* vf_info is non-zero when SR-IOV is init'ed */ struct adf_accel_vf_info *vf_info; } pf; diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index ed01fb9ad74ef..3fc7d13e882c8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -17,6 +17,58 @@ struct adf_fatal_error_data { static struct workqueue_struct *device_reset_wq; static struct workqueue_struct *device_sriov_wq; +static pci_ers_result_t reset_prepare(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + dev_err(&pdev->dev, "Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (!adf_dev_started(accel_dev)) + return PCI_ERS_RESULT_CAN_RECOVER; + + set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + if (accel_dev->hw_device->exit_arb) { + dev_dbg(&pdev->dev, "Disabling arbitration\n"); + accel_dev->hw_device->exit_arb(accel_dev); + } + adf_dev_restarting_notify(accel_dev); + adf_dev_down(accel_dev); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t reset_done(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + int res; + + if (!accel_dev) { + dev_err(&pdev->dev, "Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (!adf_devmgr_in_reset(accel_dev)) + goto reset_complete; + + pci_restore_state(pdev); + res = adf_dev_up(accel_dev, false); + if (res && res != -EALREADY) + return PCI_ERS_RESULT_DISCONNECT; + + adf_reenable_sriov(accel_dev); + adf_pf2vf_notify_restarted(accel_dev); + adf_dev_restarted_notify(accel_dev); + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + +reset_complete: + dev_info(&pdev->dev, "Device reset completed successfully\n"); + + return PCI_ERS_RESULT_RECOVERED; +} + static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { @@ -33,18 +85,10 @@ static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, return PCI_ERS_RESULT_DISCONNECT; } - set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); - if (accel_dev->hw_device->exit_arb) { - dev_dbg(&pdev->dev, "Disabling arbitration\n"); - accel_dev->hw_device->exit_arb(accel_dev); - } adf_error_notifier(accel_dev); adf_pf2vf_notify_fatal_error(accel_dev); - adf_dev_restarting_notify(accel_dev); - pci_clear_master(pdev); - adf_dev_down(accel_dev); - return PCI_ERS_RESULT_NEED_RESET; + return reset_prepare(pdev); } /* reset dev data */ @@ -106,6 +150,13 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) } } +void adf_set_bme(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + pci_set_master(pdev); +} + static void adf_device_sriov_worker(struct work_struct *work) { struct adf_sriov_dev_data *sriov_data = @@ -156,13 +207,14 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, struct adf_reset_dev_data *reset_data; if (!adf_dev_started(accel_dev) || - test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + test_and_set_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) return 0; - set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); reset_data = kzalloc_obj(*reset_data); - if (!reset_data) + if (!reset_data) { + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); return -ENOMEM; + } reset_data->accel_dev = accel_dev; init_completion(&reset_data->compl); reset_data->mode = mode; @@ -190,26 +242,7 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) { - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - int res = 0; - - if (!accel_dev) { - pr_err("QAT: Can't find acceleration device\n"); - return PCI_ERS_RESULT_DISCONNECT; - } - - if (!pdev->is_busmaster) - pci_set_master(pdev); - pci_restore_state(pdev); - res = adf_dev_up(accel_dev, false); - if (res && res != -EALREADY) - return PCI_ERS_RESULT_DISCONNECT; - - adf_reenable_sriov(accel_dev); - adf_pf2vf_notify_restarted(accel_dev); - adf_dev_restarted_notify(accel_dev); - clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); - return PCI_ERS_RESULT_RECOVERED; + return reset_done(pdev); } static void adf_resume(struct pci_dev *pdev) @@ -218,10 +251,22 @@ static void adf_resume(struct pci_dev *pdev) dev_info(&pdev->dev, "Device is up and running\n"); } +static void adf_reset_prepare(struct pci_dev *pdev) +{ + reset_prepare(pdev); +} + +static void adf_reset_done(struct pci_dev *pdev) +{ + reset_done(pdev); +} + const struct pci_error_handlers adf_err_handler = { .error_detected = adf_error_detected, .slot_reset = adf_slot_reset, .resume = adf_resume, + .reset_prepare = adf_reset_prepare, + .reset_done = adf_reset_done, }; EXPORT_SYMBOL_GPL(adf_err_handler); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c index c202209f17d5e..ea5d72d5090c0 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c @@ -103,16 +103,6 @@ static void adf_cfg_section_del_all(struct list_head *head); static void adf_cfg_section_del_all_except(struct list_head *head, const char *section_name); -void adf_cfg_del_all(struct adf_accel_dev *accel_dev) -{ - struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; - - down_write(&dev_cfg_data->lock); - adf_cfg_section_del_all(&dev_cfg_data->sec_list); - up_write(&dev_cfg_data->lock); - clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); -} - void adf_cfg_del_all_except(struct adf_accel_dev *accel_dev, const char *section_name) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.h b/drivers/crypto/intel/qat/qat_common/adf_cfg.h index 2afa6f0d15c51..108032b392425 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.h @@ -34,7 +34,6 @@ void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev); void adf_cfg_dev_dbgfs_add(struct adf_accel_dev *accel_dev); void adf_cfg_dev_dbgfs_rm(struct adf_accel_dev *accel_dev); int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name); -void adf_cfg_del_all(struct adf_accel_dev *accel_dev); void adf_cfg_del_all_except(struct adf_accel_dev *accel_dev, const char *section_name); int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index 81e9e9d7eccd2..d63f4dcccbb5d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -4,18 +4,11 @@ #define ADF_CFG_COMMON_H_ #include -#include #define ADF_CFG_MAX_STR_LEN 64 #define ADF_CFG_MAX_KEY_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN #define ADF_CFG_MAX_VAL_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN #define ADF_CFG_MAX_SECTION_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN -#define ADF_CFG_BASE_DEC 10 -#define ADF_CFG_BASE_HEX 16 -#define ADF_CFG_ALL_DEVICES 0xFE -#define ADF_CFG_NO_DEVICE 0xFF -#define ADF_CFG_AFFINITY_WHATEVER 0xFF -#define MAX_DEVICE_NAME_SIZE 32 #define ADF_MAX_DEVICES (32 * 32) #define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES) @@ -51,29 +44,4 @@ enum adf_device_type { DEV_420XX, DEV_6XXX, }; - -struct adf_dev_status_info { - enum adf_device_type type; - __u32 accel_id; - __u32 instance_id; - __u8 num_ae; - __u8 num_accel; - __u8 num_logical_accel; - __u8 banks_per_accel; - __u8 state; - __u8 bus; - __u8 dev; - __u8 fun; - char name[MAX_DEVICE_NAME_SIZE]; -}; - -#define ADF_CTL_IOC_MAGIC 'a' -#define IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS _IOW(ADF_CTL_IOC_MAGIC, 0, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_STOP_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 1, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_START_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 2, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_STATUS_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 3, __u32) -#define IOCTL_GET_NUM_DEVICES _IOW(ADF_CTL_IOC_MAGIC, 4, __s32) #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h deleted file mode 100644 index 421f4fb8b4dd2..0000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_CFG_USER_H_ -#define ADF_CFG_USER_H_ - -#include "adf_cfg_common.h" -#include "adf_cfg_strings.h" - -struct adf_user_cfg_key_val { - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - union { - struct adf_user_cfg_key_val *next; - __u64 padding3; - }; - enum adf_cfg_val_type type; -} __packed; - -struct adf_user_cfg_section { - char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; - union { - struct adf_user_cfg_key_val *params; - __u64 padding1; - }; - union { - struct adf_user_cfg_section *next; - __u64 padding3; - }; -} __packed; - -struct adf_user_cfg_ctl_data { - union { - struct adf_user_cfg_section *config_section; - __u64 padding; - }; - __u8 device_id; -} __packed; -#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index 7b8b295ac4590..7d7d64fce6140 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -68,10 +68,7 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf); struct list_head *adf_devmgr_get_head(void); -struct adf_accel_dev *adf_devmgr_get_dev_by_id(u32 id); struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev); -int adf_devmgr_verify_id(u32 id); -void adf_devmgr_get_num_dev(u32 *num); int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev); int adf_dev_started(struct adf_accel_dev *accel_dev); int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev); @@ -87,12 +84,12 @@ extern const struct pci_error_handlers adf_err_handler; void adf_reset_sbr(struct adf_accel_dev *accel_dev); void adf_reset_flr(struct adf_accel_dev *accel_dev); void adf_dev_restore(struct adf_accel_dev *accel_dev); +void adf_set_bme(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); int adf_init_arb(struct adf_accel_dev *accel_dev); void adf_exit_arb(struct adf_accel_dev *accel_dev); void adf_update_ring_arb(struct adf_etr_ring_data *ring); -int adf_disable_arb_thd(struct adf_accel_dev *accel_dev, u32 ae, u32 thr); int adf_dev_get(struct adf_accel_dev *accel_dev); void adf_dev_put(struct adf_accel_dev *accel_dev); @@ -121,6 +118,7 @@ void qat_comp_alg_callback(void *resp); int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); void adf_isr_resource_free(struct adf_accel_dev *accel_dev); +void adf_isr_sync_ae_cluster(struct adf_accel_dev *accel_dev); int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); @@ -194,6 +192,7 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); void adf_reenable_sriov(struct adf_accel_dev *accel_dev); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); +void adf_enable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 num_vfs); void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev); bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev); bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr); diff --git a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c index c2e6f0cb7480b..f01f2946de6e8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c @@ -2,410 +2,13 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" #include "adf_common_drv.h" -#include "adf_cfg.h" -#include "adf_cfg_common.h" -#include "adf_cfg_user.h" - -#define ADF_CFG_MAX_SECTION 512 -#define ADF_CFG_MAX_KEY_VAL 256 - -#define DEVICE_NAME "qat_adf_ctl" - -static DEFINE_MUTEX(adf_ctl_lock); -static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg); - -static const struct file_operations adf_ctl_ops = { - .owner = THIS_MODULE, - .unlocked_ioctl = adf_ctl_ioctl, - .compat_ioctl = compat_ptr_ioctl, -}; - -static const struct class adf_ctl_class = { - .name = DEVICE_NAME, -}; - -struct adf_ctl_drv_info { - unsigned int major; - struct cdev drv_cdev; -}; - -static struct adf_ctl_drv_info adf_ctl_drv; - -static void adf_chr_drv_destroy(void) -{ - device_destroy(&adf_ctl_class, MKDEV(adf_ctl_drv.major, 0)); - cdev_del(&adf_ctl_drv.drv_cdev); - class_unregister(&adf_ctl_class); - unregister_chrdev_region(MKDEV(adf_ctl_drv.major, 0), 1); -} - -static int adf_chr_drv_create(void) -{ - dev_t dev_id; - struct device *drv_device; - int ret; - - if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) { - pr_err("QAT: unable to allocate chrdev region\n"); - return -EFAULT; - } - - ret = class_register(&adf_ctl_class); - if (ret) - goto err_chrdev_unreg; - - adf_ctl_drv.major = MAJOR(dev_id); - cdev_init(&adf_ctl_drv.drv_cdev, &adf_ctl_ops); - if (cdev_add(&adf_ctl_drv.drv_cdev, dev_id, 1)) { - pr_err("QAT: cdev add failed\n"); - goto err_class_destr; - } - - drv_device = device_create(&adf_ctl_class, NULL, - MKDEV(adf_ctl_drv.major, 0), - NULL, DEVICE_NAME); - if (IS_ERR(drv_device)) { - pr_err("QAT: failed to create device\n"); - goto err_cdev_del; - } - return 0; -err_cdev_del: - cdev_del(&adf_ctl_drv.drv_cdev); -err_class_destr: - class_unregister(&adf_ctl_class); -err_chrdev_unreg: - unregister_chrdev_region(dev_id, 1); - return -EFAULT; -} - -static struct adf_user_cfg_ctl_data *adf_ctl_alloc_resources(unsigned long arg) -{ - struct adf_user_cfg_ctl_data *cfg_data; - - cfg_data = memdup_user((void __user *)arg, sizeof(*cfg_data)); - if (IS_ERR(cfg_data)) - pr_err("QAT: failed to copy from user cfg_data.\n"); - return cfg_data; -} - -static int adf_add_key_value_data(struct adf_accel_dev *accel_dev, - const char *section, - const struct adf_user_cfg_key_val *key_val) -{ - if (key_val->type == ADF_HEX) { - long *ptr = (long *)key_val->val; - long val = *ptr; - - if (adf_cfg_add_key_value_param(accel_dev, section, - key_val->key, (void *)val, - key_val->type)) { - dev_err(&GET_DEV(accel_dev), - "failed to add hex keyvalue.\n"); - return -EFAULT; - } - } else { - if (adf_cfg_add_key_value_param(accel_dev, section, - key_val->key, key_val->val, - key_val->type)) { - dev_err(&GET_DEV(accel_dev), - "failed to add keyvalue.\n"); - return -EFAULT; - } - } - return 0; -} - -static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, - struct adf_user_cfg_ctl_data *ctl_data) -{ - struct adf_user_cfg_key_val key_val; - struct adf_user_cfg_key_val *params_head; - struct adf_user_cfg_section section, *section_head; - int i, j; - - section_head = ctl_data->config_section; - - for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) { - if (copy_from_user(§ion, (void __user *)section_head, - sizeof(*section_head))) { - dev_err(&GET_DEV(accel_dev), - "failed to copy section info\n"); - goto out_err; - } - - if (adf_cfg_section_add(accel_dev, section.name)) { - dev_err(&GET_DEV(accel_dev), - "failed to add section.\n"); - goto out_err; - } - - params_head = section.params; - - for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) { - if (copy_from_user(&key_val, (void __user *)params_head, - sizeof(key_val))) { - dev_err(&GET_DEV(accel_dev), - "Failed to copy keyvalue.\n"); - goto out_err; - } - if (adf_add_key_value_data(accel_dev, section.name, - &key_val)) { - goto out_err; - } - params_head = key_val.next; - } - section_head = section.next; - } - return 0; -out_err: - adf_cfg_del_all(accel_dev); - return -EFAULT; -} - -static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - struct adf_user_cfg_ctl_data *ctl_data; - struct adf_accel_dev *accel_dev; - int ret = 0; - - ctl_data = adf_ctl_alloc_resources(arg); - if (IS_ERR(ctl_data)) - return PTR_ERR(ctl_data); - - accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); - if (!accel_dev) { - ret = -EFAULT; - goto out; - } - - if (adf_dev_started(accel_dev)) { - ret = -EFAULT; - goto out; - } - - if (adf_copy_key_value_data(accel_dev, ctl_data)) { - ret = -EFAULT; - goto out; - } - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_is_device_in_use(int id) -{ - struct adf_accel_dev *dev; - - list_for_each_entry(dev, adf_devmgr_get_head(), list) { - if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) { - dev_info(&GET_DEV(dev), - "device qat_dev%d is busy\n", - dev->accel_id); - return -EBUSY; - } - } - } - return 0; -} - -static void adf_ctl_stop_devices(u32 id) -{ - struct adf_accel_dev *accel_dev; - - list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { - if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (!adf_dev_started(accel_dev)) - continue; - - /* First stop all VFs */ - if (!accel_dev->is_vf) - continue; - - adf_dev_down(accel_dev); - } - } - - list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { - if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (!adf_dev_started(accel_dev)) - continue; - - adf_dev_down(accel_dev); - } - } -} - -static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - int ret; - struct adf_user_cfg_ctl_data *ctl_data; - - ctl_data = adf_ctl_alloc_resources(arg); - if (IS_ERR(ctl_data)) - return PTR_ERR(ctl_data); - - if (adf_devmgr_verify_id(ctl_data->device_id)) { - pr_err("QAT: Device %d not found\n", ctl_data->device_id); - ret = -ENODEV; - goto out; - } - - ret = adf_ctl_is_device_in_use(ctl_data->device_id); - if (ret) - goto out; - - if (ctl_data->device_id == ADF_CFG_ALL_DEVICES) - pr_info("QAT: Stopping all acceleration devices.\n"); - else - pr_info("QAT: Stopping acceleration device qat_dev%d.\n", - ctl_data->device_id); - - adf_ctl_stop_devices(ctl_data->device_id); - -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - int ret; - struct adf_user_cfg_ctl_data *ctl_data; - struct adf_accel_dev *accel_dev; - - ctl_data = adf_ctl_alloc_resources(arg); - if (IS_ERR(ctl_data)) - return PTR_ERR(ctl_data); - - ret = -ENODEV; - accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); - if (!accel_dev) - goto out; - - dev_info(&GET_DEV(accel_dev), - "Starting acceleration device qat_dev%d.\n", - ctl_data->device_id); - - ret = adf_dev_up(accel_dev, false); - - if (ret) { - dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", - ctl_data->device_id); - adf_dev_down(accel_dev); - } -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - u32 num_devices = 0; - - adf_devmgr_get_num_dev(&num_devices); - if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices))) - return -EFAULT; - - return 0; -} - -static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - struct adf_hw_device_data *hw_data; - struct adf_dev_status_info dev_info; - struct adf_accel_dev *accel_dev; - - if (copy_from_user(&dev_info, (void __user *)arg, - sizeof(struct adf_dev_status_info))) { - pr_err("QAT: failed to copy from user.\n"); - return -EFAULT; - } - - accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id); - if (!accel_dev) - return -ENODEV; - - hw_data = accel_dev->hw_device; - dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; - dev_info.num_ae = hw_data->get_num_aes(hw_data); - dev_info.num_accel = hw_data->get_num_accels(hw_data); - dev_info.num_logical_accel = hw_data->num_logical_accel; - dev_info.banks_per_accel = hw_data->num_banks - / hw_data->num_logical_accel; - strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); - dev_info.instance_id = hw_data->instance_id; - dev_info.type = hw_data->dev_class->type; - dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number; - dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn); - dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn); - - if (copy_to_user((void __user *)arg, &dev_info, - sizeof(struct adf_dev_status_info))) { - dev_err(&GET_DEV(accel_dev), "failed to copy status.\n"); - return -EFAULT; - } - return 0; -} - -static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) -{ - int ret; - - if (mutex_lock_interruptible(&adf_ctl_lock)) - return -EFAULT; - - switch (cmd) { - case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS: - ret = adf_ctl_ioctl_dev_config(fp, cmd, arg); - break; - - case IOCTL_STOP_ACCEL_DEV: - ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg); - break; - - case IOCTL_START_ACCEL_DEV: - ret = adf_ctl_ioctl_dev_start(fp, cmd, arg); - break; - - case IOCTL_GET_NUM_DEVICES: - ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg); - break; - - case IOCTL_STATUS_ACCEL_DEV: - ret = adf_ctl_ioctl_get_status(fp, cmd, arg); - break; - default: - pr_err_ratelimited("QAT: Invalid ioctl %d\n", cmd); - ret = -EFAULT; - break; - } - mutex_unlock(&adf_ctl_lock); - return ret; -} static int __init adf_register_ctl_device_driver(void) { - if (adf_chr_drv_create()) - goto err_chr_dev; - if (adf_init_misc_wq()) goto err_misc_wq; @@ -437,15 +40,11 @@ static int __init adf_register_ctl_device_driver(void) err_aer: adf_exit_misc_wq(); err_misc_wq: - adf_chr_drv_destroy(); -err_chr_dev: - mutex_destroy(&adf_ctl_lock); return -EFAULT; } static void __exit adf_unregister_ctl_device_driver(void) { - adf_chr_drv_destroy(); adf_exit_misc_wq(); adf_exit_aer(); adf_exit_vf_wq(); @@ -453,7 +52,6 @@ static void __exit adf_unregister_ctl_device_driver(void) qat_crypto_unregister(); qat_compression_unregister(); adf_clean_vf_map(false); - mutex_destroy(&adf_ctl_lock); } module_init(adf_register_ctl_device_driver); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c index e050de16ab5d0..161863841a527 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c @@ -45,19 +45,6 @@ static struct vf_id_map *adf_find_vf(u32 bdf) return NULL; } -static int adf_get_vf_real_id(u32 fake) -{ - struct list_head *itr; - - list_for_each(itr, &vfs_table) { - struct vf_id_map *ptr = - list_entry(itr, struct vf_id_map, list); - if (ptr->fake_id == fake) - return ptr->id; - } - return -1; -} - /** * adf_clean_vf_map() - Cleans VF id mappings * @vf: flag indicating whether mappings is cleaned @@ -304,63 +291,6 @@ struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev) } EXPORT_SYMBOL_GPL(adf_devmgr_pci_to_accel_dev); -struct adf_accel_dev *adf_devmgr_get_dev_by_id(u32 id) -{ - struct list_head *itr; - int real_id; - - mutex_lock(&table_lock); - real_id = adf_get_vf_real_id(id); - if (real_id < 0) - goto unlock; - - id = real_id; - - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = - list_entry(itr, struct adf_accel_dev, list); - if (ptr->accel_id == id) { - mutex_unlock(&table_lock); - return ptr; - } - } -unlock: - mutex_unlock(&table_lock); - return NULL; -} - -int adf_devmgr_verify_id(u32 id) -{ - if (id == ADF_CFG_ALL_DEVICES) - return 0; - - if (adf_devmgr_get_dev_by_id(id)) - return 0; - - return -ENODEV; -} - -static int adf_get_num_dettached_vfs(void) -{ - struct list_head *itr; - int vfs = 0; - - mutex_lock(&table_lock); - list_for_each(itr, &vfs_table) { - struct vf_id_map *ptr = - list_entry(itr, struct vf_id_map, list); - if (ptr->bdf != ~0 && !ptr->attached) - vfs++; - } - mutex_unlock(&table_lock); - return vfs; -} - -void adf_devmgr_get_num_dev(u32 *num) -{ - *num = num_devices - adf_get_num_dettached_vfs(); -} - /** * adf_dev_in_use() - Check whether accel_dev is currently in use * @accel_dev: Pointer to acceleration device. diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c index a3b474bdef6c8..023c5f1e78b07 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c @@ -64,10 +64,8 @@ int adf_heartbeat_inject_error(struct adf_accel_dev *accel_dev) if (ret) return ret; - /* Configure worker threads to stop processing any packet */ - ret = adf_disable_arb_thd(accel_dev, rand_ae, rand_thr); - if (ret) - return ret; + /* Disable arbiter to stop processing any packet */ + hw_device->exit_arb(accel_dev); /* Change HB counters memory to simulate a hang */ adf_set_hb_counters_fail(accel_dev, rand_ae, rand_thr); diff --git a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c index f93d9cca70cee..dd9a31c20bc9c 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c @@ -99,28 +99,3 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); } EXPORT_SYMBOL_GPL(adf_exit_arb); - -int adf_disable_arb_thd(struct adf_accel_dev *accel_dev, u32 ae, u32 thr) -{ - void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - const u32 *thd_2_arb_cfg; - struct arb_info info; - u32 ae_thr_map; - - if (ADF_AE_STRAND0_THREAD == thr || ADF_AE_STRAND1_THREAD == thr) - thr = ADF_AE_ADMIN_THREAD; - - hw_data->get_arb_info(&info); - thd_2_arb_cfg = hw_data->get_arb_mapping(accel_dev); - if (!thd_2_arb_cfg) - return -EFAULT; - - /* Disable scheduling for this particular AE and thread */ - ae_thr_map = *(thd_2_arb_cfg + ae); - ae_thr_map &= ~(GENMASK(3, 0) << (thr * BIT(2))); - - WRITE_CSR_ARB_WT2SAM(csr, info.arb_offset, info.wt2sam_offset, ae, - ae_thr_map); - return 0; -} diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c index f8088388cf12d..1c7f9e49914d4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_init.c +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -74,6 +74,8 @@ static int adf_dev_init(struct adf_accel_dev *accel_dev) return -EFAULT; } + adf_set_bme(accel_dev); + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && !accel_dev->is_vf) { dev_err(&GET_DEV(accel_dev), "Device not configured\n"); @@ -153,15 +155,18 @@ static int adf_dev_init(struct adf_accel_dev *accel_dev) * This is to facilitate any ordering dependencies between services * prior to starting any of the accelerators. */ + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { dev_err(&GET_DEV(accel_dev), "Failed to initialise service %s\n", service->name); + mutex_unlock(&service_lock); return -EFAULT; } set_bit(accel_dev->accel_id, service->init_status); } + mutex_unlock(&service_lock); return 0; } @@ -231,15 +236,18 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) if (ret && ret != -EOPNOTSUPP) return ret; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_START)) { dev_err(&GET_DEV(accel_dev), "Failed to start service %s\n", service->name); + mutex_unlock(&service_lock); return -EFAULT; } set_bit(accel_dev->accel_id, service->start_status); } + mutex_unlock(&service_lock); clear_bit(ADF_STATUS_STARTING, &accel_dev->status); set_bit(ADF_STATUS_STARTED, &accel_dev->status); @@ -313,6 +321,7 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev) qat_comp_algs_unregister(hw_data->accel_capabilities_ext_mask); clear_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status); + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (!test_bit(accel_dev->accel_id, service->start_status)) continue; @@ -324,6 +333,7 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev) clear_bit(accel_dev->accel_id, service->start_status); } } + mutex_unlock(&service_lock); if (hw_data->stop_timer) hw_data->stop_timer(accel_dev); @@ -373,6 +383,7 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) &accel_dev->status); } + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (!test_bit(accel_dev->accel_id, service->init_status)) continue; @@ -383,6 +394,7 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) else clear_bit(accel_dev->accel_id, service->init_status); } + mutex_unlock(&service_lock); adf_rl_exit(accel_dev); @@ -417,12 +429,14 @@ int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) { struct service_hndl *service; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", service->name); } + mutex_unlock(&service_lock); return 0; } @@ -430,12 +444,14 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) { struct service_hndl *service; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", service->name); } + mutex_unlock(&service_lock); return 0; } @@ -443,12 +459,14 @@ void adf_error_notifier(struct adf_accel_dev *accel_dev) { struct service_hndl *service; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_FATAL_ERROR)) dev_err(&GET_DEV(accel_dev), "Failed to send error event to %s.\n", service->name); } + mutex_unlock(&service_lock); } int adf_dev_down(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index 4639d7fd93e6e..159e91a50106b 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -62,6 +62,23 @@ void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + if (!READ_ONCE(accel_dev->pf.vf2pf_disabled)) + GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); + spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); +} + +void adf_enable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 num_vfs) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + unsigned long flags; + u32 vf_mask; + + vf_mask = BIT_ULL(num_vfs) - 1; + if (!vf_mask) + return; + + spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + WRITE_ONCE(accel_dev->pf.vf2pf_disabled, false); GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } @@ -72,6 +89,7 @@ void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev) unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + WRITE_ONCE(accel_dev->pf.vf2pf_disabled, true); GET_PFVF_OPS(accel_dev)->disable_all_vf2pf_interrupts(pmisc_addr); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } @@ -174,6 +192,27 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) return IRQ_NONE; } +void adf_isr_sync_ae_cluster(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 num_entries = pci_dev_info->msix_entries.num_entries; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; + u32 irq_idx; + int irq; + + if (!test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status) || !irqs) + return; + + irq_idx = num_entries > 1 ? hw_data->num_banks : 0; + if (irq_idx >= num_entries || !irqs[irq_idx].enabled) + return; + + irq = pci_irq_vector(pci_dev_info->pci_dev, hw_data->num_banks); + if (irq > 0) + synchronize_irq(irq); +} + static void adf_free_irqs(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; diff --git a/drivers/crypto/intel/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c index 8bf0fe1fcb4db..f2011300a9294 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c @@ -26,6 +26,9 @@ static void adf_iov_send_resp(struct work_struct *work) u32 vf_nr = vf_info->vf_nr; bool ret; + if (READ_ONCE(accel_dev->pf.vf2pf_disabled)) + goto out; + mutex_lock(&vf_info->pfvf_mig_lock); ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr); if (ret) @@ -33,13 +36,18 @@ static void adf_iov_send_resp(struct work_struct *work) adf_enable_vf2pf_interrupts(accel_dev, 1 << vf_nr); mutex_unlock(&vf_info->pfvf_mig_lock); +out: kfree(pf2vf_resp); } void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info) { + struct adf_accel_dev *accel_dev = vf_info->accel_dev; struct adf_pf2vf_resp *pf2vf_resp; + if (READ_ONCE(accel_dev->pf.vf2pf_disabled)) + return; + pf2vf_resp = kzalloc_obj(*pf2vf_resp, GFP_ATOMIC); if (!pf2vf_resp) return; @@ -49,6 +57,12 @@ void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info) queue_work(pf2vf_resp_wq, &pf2vf_resp->pf2vf_resp_work); } +static void adf_flush_pf2vf_resp_wq(void) +{ + if (pf2vf_resp_wq) + flush_workqueue(pf2vf_resp_wq); +} + static int adf_enable_sriov(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); @@ -75,7 +89,11 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); + adf_enable_all_vf2pf_interrupts(accel_dev, totalvfs); + + /* Do not enable SR-IOV if already enabled */ + if (pci_num_vf(pdev)) + return 0; /* * Due to the hardware design, when SR-IOV and the ring arbiter @@ -246,10 +264,18 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) adf_pf2vf_notify_restarting(accel_dev); adf_pf2vf_wait_for_restarting_complete(accel_dev); - pci_disable_sriov(accel_to_pci_dev(accel_dev)); + /* + * When the device is restarting, preserve VF PCI devices across + * the reset by skipping pci_disable_sriov(). VFs are notified to + * quiesce regardless so the PF can safely shut down. + */ + if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + pci_disable_sriov(accel_to_pci_dev(accel_dev)); - /* Disable VF to PF interrupts */ + /* Block VF2PF work and disable VF to PF interrupts */ adf_disable_all_vf2pf_interrupts(accel_dev); + adf_isr_sync_ae_cluster(accel_dev); + adf_flush_pf2vf_resp_wq(); /* Clear Valid bits in AE Thread to PCIe Function Mapping */ if (hw_data->configure_iov_threads) diff --git a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c index e09b9edfce423..75c15c8e41dba 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c @@ -1085,7 +1085,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->p; len = rsa_key->p_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto err; ctx->p = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL); if (!ctx->p) @@ -1096,7 +1096,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->q; len = rsa_key->q_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_p; ctx->q = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL); if (!ctx->q) @@ -1107,7 +1107,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->dp; len = rsa_key->dp_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_q; ctx->dp = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dp, GFP_KERNEL); @@ -1119,7 +1119,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->dq; len = rsa_key->dq_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_dp; ctx->dq = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dq, GFP_KERNEL); @@ -1131,7 +1131,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->qinv; len = rsa_key->qinv_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_dq; ctx->qinv = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_qinv, GFP_KERNEL); diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index b59e0cc49e521..99a6bf9e2b005 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -167,7 +167,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_free_reg; } } - pci_set_master(pdev); if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c index 7cd528ee31e77..e3eab03ad65bd 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c @@ -163,7 +163,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_free_reg; } } - pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); diff --git a/drivers/crypto/loongson/Kconfig b/drivers/crypto/loongson/Kconfig deleted file mode 100644 index 15475da8fc11d..0000000000000 --- a/drivers/crypto/loongson/Kconfig +++ /dev/null @@ -1,5 +0,0 @@ -config CRYPTO_DEV_LOONGSON_RNG - tristate "Support for Loongson RNG Driver" - depends on MFD_LOONGSON_SE - help - Support for Loongson RNG Driver. diff --git a/drivers/crypto/loongson/Makefile b/drivers/crypto/loongson/Makefile deleted file mode 100644 index 1ce5ec32b5534..0000000000000 --- a/drivers/crypto/loongson/Makefile +++ /dev/null @@ -1 +0,0 @@ -obj-$(CONFIG_CRYPTO_DEV_LOONGSON_RNG) += loongson-rng.o diff --git a/drivers/crypto/loongson/loongson-rng.c b/drivers/crypto/loongson/loongson-rng.c deleted file mode 100644 index 3a4940260f9e5..0000000000000 --- a/drivers/crypto/loongson/loongson-rng.c +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019 HiSilicon Limited. */ -/* Copyright (c) 2025 Loongson Technology Corporation Limited. */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SE_SEED_SIZE 32 - -struct loongson_rng_list { - struct mutex lock; - struct list_head list; - int registered; -}; - -struct loongson_rng { - u32 used; - struct loongson_se_engine *engine; - struct list_head list; - struct mutex lock; -}; - -struct loongson_rng_ctx { - struct loongson_rng *rng; -}; - -struct loongson_rng_cmd { - u32 cmd_id; - union { - u32 len; - u32 ret; - } u; - u32 seed_off; - u32 out_off; - u32 pad[4]; -}; - -static struct loongson_rng_list rng_devices = { - .lock = __MUTEX_INITIALIZER(rng_devices.lock), - .list = LIST_HEAD_INIT(rng_devices.list), -}; - -static int loongson_rng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) -{ - struct loongson_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct loongson_rng *rng = ctx->rng; - struct loongson_rng_cmd *cmd = rng->engine->command; - int err, len; - - mutex_lock(&rng->lock); - cmd->seed_off = 0; - do { - len = min(dlen, rng->engine->buffer_size); - cmd = rng->engine->command; - cmd->u.len = len; - err = loongson_se_send_engine_cmd(rng->engine); - if (err) - break; - - cmd = rng->engine->command_ret; - if (cmd->u.ret) { - err = -EIO; - break; - } - - memcpy(dstn, rng->engine->data_buffer, len); - dlen -= len; - dstn += len; - } while (dlen > 0); - mutex_unlock(&rng->lock); - - return err; -} - -static int loongson_rng_init(struct crypto_tfm *tfm) -{ - struct loongson_rng_ctx *ctx = crypto_tfm_ctx(tfm); - struct loongson_rng *rng; - u32 min_used = U32_MAX; - - mutex_lock(&rng_devices.lock); - list_for_each_entry(rng, &rng_devices.list, list) { - if (rng->used < min_used) { - ctx->rng = rng; - min_used = rng->used; - } - } - ctx->rng->used++; - mutex_unlock(&rng_devices.lock); - - return 0; -} - -static void loongson_rng_exit(struct crypto_tfm *tfm) -{ - struct loongson_rng_ctx *ctx = crypto_tfm_ctx(tfm); - - mutex_lock(&rng_devices.lock); - ctx->rng->used--; - mutex_unlock(&rng_devices.lock); -} - -static int loongson_rng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct loongson_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct loongson_rng *rng = ctx->rng; - struct loongson_rng_cmd *cmd; - int err; - - if (slen < SE_SEED_SIZE) - return -EINVAL; - - slen = min(slen, rng->engine->buffer_size); - - mutex_lock(&rng->lock); - cmd = rng->engine->command; - cmd->u.len = slen; - cmd->seed_off = rng->engine->buffer_off; - memcpy(rng->engine->data_buffer, seed, slen); - err = loongson_se_send_engine_cmd(rng->engine); - if (err) - goto out; - - cmd = rng->engine->command_ret; - if (cmd->u.ret) - err = -EIO; -out: - mutex_unlock(&rng->lock); - - return err; -} - -static struct rng_alg loongson_rng_alg = { - .generate = loongson_rng_generate, - .seed = loongson_rng_seed, - .seedsize = SE_SEED_SIZE, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "loongson_stdrng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct loongson_rng_ctx), - .cra_module = THIS_MODULE, - .cra_init = loongson_rng_init, - .cra_exit = loongson_rng_exit, - }, -}; - -static int loongson_rng_probe(struct platform_device *pdev) -{ - struct loongson_rng_cmd *cmd; - struct loongson_rng *rng; - int ret = 0; - - rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); - if (!rng) - return -ENOMEM; - - rng->engine = loongson_se_init_engine(pdev->dev.parent, SE_ENGINE_RNG); - if (!rng->engine) - return -ENODEV; - cmd = rng->engine->command; - cmd->cmd_id = SE_CMD_RNG; - cmd->out_off = rng->engine->buffer_off; - mutex_init(&rng->lock); - - mutex_lock(&rng_devices.lock); - - if (!rng_devices.registered) { - ret = crypto_register_rng(&loongson_rng_alg); - if (ret) { - dev_err(&pdev->dev, "failed to register crypto(%d)\n", ret); - goto out; - } - rng_devices.registered = 1; - } - - list_add_tail(&rng->list, &rng_devices.list); -out: - mutex_unlock(&rng_devices.lock); - - return ret; -} - -static struct platform_driver loongson_rng_driver = { - .probe = loongson_rng_probe, - .driver = { - .name = "loongson-rng", - }, -}; -module_platform_driver(loongson_rng_driver); - -MODULE_ALIAS("platform:loongson-rng"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Yinggang Gu "); -MODULE_AUTHOR("Qunqin Zhao "); -MODULE_DESCRIPTION("Loongson Random Number Generator driver"); diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c index c80baf1ad90b2..89030e2711ce4 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c @@ -157,8 +157,8 @@ static inline int setup_sgio_components(struct pci_dev *pdev, sg_cleanup: for (j = 0; j < i; j++) { if (list[j].dma_addr) { - dma_unmap_single(&pdev->dev, list[i].dma_addr, - list[i].size, DMA_BIDIRECTIONAL); + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); } list[j].dma_addr = 0; diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 78135fb13f5c2..1b7509e2ce443 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -714,15 +714,13 @@ int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm) /** * nx_crypto_ctx_exit - destroy a crypto api context * - * @tfm: the crypto transform pointer for the context + * @nx_ctx: the crypto api context * * As crypto API contexts are destroyed, this exit hook is called to free the * memory associated with it. */ -void nx_crypto_ctx_exit(struct crypto_tfm *tfm) +void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); - kfree_sensitive(nx_ctx->kmem); nx_ctx->csbcpb = NULL; nx_ctx->csbcpb_aead = NULL; diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index 36974f08490a9..6dfabfbf8192c 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -153,7 +153,7 @@ int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_sha_init(struct crypto_shash *tfm); -void nx_crypto_ctx_exit(struct crypto_tfm *tfm); +void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx); void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm); void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm); diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index bc61d0fe35140..4b53b13f96d9f 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -12,7 +12,6 @@ * All rights reserved. */ -#include #include #include #include @@ -255,6 +254,46 @@ static int init_device(struct device *dev) return 0; } +static void dma_map_request(struct device *dev, struct talitos_request *request, + struct talitos_desc *desc, bool is_sec1) +{ + struct talitos_edesc *edesc = + container_of(desc, struct talitos_edesc, desc); + dma_addr_t dma_desc, prev_dma_desc; + struct talitos_edesc *prev_edesc = NULL; + + if (is_sec1) { + while (edesc) { + edesc->desc.hdr1 = edesc->desc.hdr; + + dma_desc = dma_map_single(dev, &edesc->desc.hdr1, + TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + if (!prev_edesc) { + request->dma_desc = dma_desc; + goto next; + } + + /* Chain in any previous descriptors. */ + + prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); + + dma_sync_single_for_device(dev, prev_dma_desc, + TALITOS_DESC_SIZE, + DMA_TO_DEVICE); + +next: + prev_edesc = edesc; + prev_dma_desc = dma_desc; + edesc = edesc->next_desc; + } + } else { + request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } +} + /** * talitos_submit - submits a descriptor to the device for processing * @dev: the SEC device to be used @@ -291,16 +330,7 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, request = &priv->chan[ch].fifo[head]; /* map descriptor and save caller data */ - if (is_sec1) { - desc->hdr1 = desc->hdr; - request->dma_desc = dma_map_single(dev, &desc->hdr1, - TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); - } else { - request->dma_desc = dma_map_single(dev, desc, - TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); - } + dma_map_request(dev, request, desc, is_sec1); request->callback = callback; request->context = context; @@ -322,19 +352,51 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, return -EINPROGRESS; } -static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) +static __be32 get_request_hdr(struct device *dev, + struct talitos_request *request, bool is_sec1) { struct talitos_edesc *edesc; + dma_addr_t dma_desc; - if (!is_sec1) - return request->desc->hdr; + if (!is_sec1) { + dma_sync_single_for_cpu(dev, request->dma_desc, + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); - if (!request->desc->next_desc) - return request->desc->hdr1; + return request->desc->hdr; + } edesc = container_of(request->desc, struct talitos_edesc, desc); + dma_desc = request->dma_desc; + while (edesc->next_desc) { + dma_desc = be32_to_cpu(edesc->desc.next_desc); + edesc = edesc->next_desc; + } + + dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + return edesc->desc.hdr1; +} - return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; +static void dma_unmap_request(struct device *dev, + struct talitos_request *request, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (is_sec1) { + dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + edesc = container_of(request->desc, struct talitos_edesc, desc); + while (edesc->next_desc) { + dma_unmap_single(dev, + be32_to_cpu(edesc->desc.next_desc), + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + edesc = edesc->next_desc; + } + } else { + dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } } /* @@ -358,7 +420,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - hdr = get_request_hdr(request, is_sec1); + hdr = get_request_hdr(dev, request, is_sec1); if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; @@ -368,9 +430,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) else status = error; - dma_unmap_single(dev, request->dma_desc, - TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); + dma_unmap_request(dev, request, is_sec1); /* copy entries so we can call callback outside lock */ saved_req.desc = request->desc; @@ -459,14 +519,35 @@ DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) +static __be32 search_desc_hdr_in_request(struct talitos_request *request, + dma_addr_t cur_desc, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (request->dma_desc == cur_desc) { + return request->desc->hdr; + } else if (is_sec1) { + edesc = container_of(request->desc, struct talitos_edesc, desc); + while (edesc->next_desc) { + if (edesc->desc.next_desc == cpu_to_be32(cur_desc)) + return edesc->next_desc->desc.hdr1; + edesc = edesc->next_desc; + } + } + return 0; +} + /* * locate current (offending) descriptor */ static __be32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + struct talitos_request *request; int tail, iter; dma_addr_t cur_desc; + __be32 hdr = 0; cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); @@ -477,27 +558,21 @@ static __be32 current_desc_hdr(struct device *dev, int ch) } tail = priv->chan[ch].tail; - iter = tail; - while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && - priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { - iter = (iter + 1) & (priv->fifo_len - 1); - if (iter == tail) { - dev_err(dev, "couldn't locate current descriptor\n"); - return 0; - } - } + do { + request = &priv->chan[ch].fifo[iter]; - if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { - struct talitos_edesc *edesc; + hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1); + if (hdr) + break; - edesc = container_of(priv->chan[ch].fifo[iter].desc, - struct talitos_edesc, desc); - return ((struct talitos_desc *) - (edesc->buf + edesc->dma_len))->hdr; - } + iter = (iter + 1) & (priv->fifo_len - 1); + } while (iter != tail); - return priv->chan[ch].fifo[iter].desc->hdr; + if (!hdr) + dev_err(dev, "couldn't locate current descriptor\n"); + + return hdr; } /* @@ -869,28 +944,20 @@ struct talitos_ahash_req_ctx { u8 buf[2][HASH_MAX_BLOCK_SIZE]; int buf_idx; unsigned int swinit; - unsigned int first_desc; - unsigned int last_desc; + unsigned int first_request; unsigned int last_request; unsigned int to_hash_later; unsigned int nbuf; struct scatterlist bufsl[2]; struct scatterlist *psrc; - - struct scatterlist request_bufsl[2]; - struct ahash_request *areq; - struct scatterlist *request_sl; - unsigned int remaining_ahash_request_bytes; - unsigned int current_ahash_request_bytes; - struct work_struct sec1_ahash_process_remaining; }; struct talitos_export_state { u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; u8 buf[HASH_MAX_BLOCK_SIZE]; unsigned int swinit; - unsigned int first_desc; - unsigned int last_desc; + unsigned int first_request; + unsigned int last_request; unsigned int to_hash_later; unsigned int nbuf; }; @@ -1396,10 +1463,6 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, dma_len = 0; } alloc_len += icv_stashing ? authsize : 0; - - /* if its a ahash, add space for a second desc next to the first one */ - if (is_sec1 && !dst) - alloc_len += sizeof(struct talitos_desc); alloc_len += ivsize; edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); @@ -1415,6 +1478,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; + edesc->next_desc = NULL; if (dma_len) edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], edesc->dma_len, @@ -1715,39 +1779,37 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); struct talitos_desc *desc = &edesc->desc; - struct talitos_desc *desc2 = (struct talitos_desc *) - (edesc->buf + edesc->dma_len); unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE); - if (desc->next_desc && - desc->ptr[5].ptr != desc2->ptr[5].ptr) - unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); - if (req_ctx->last_desc) + + if (edesc->last && req_ctx->last_request) memcpy(areq->result, req_ctx->hw_context, crypto_ahash_digestsize(tfm)); - if (req_ctx->psrc) - talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); + if (edesc->src) + talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0); /* When using hashctx-in, must unmap it. */ if (from_talitos_ptr_len(&desc->ptr[1], is_sec1)) unmap_single_talitos_ptr(dev, &desc->ptr[1], DMA_TO_DEVICE); - else if (desc->next_desc) - unmap_single_talitos_ptr(dev, &desc2->ptr[1], - DMA_TO_DEVICE); - - if (is_sec1 && req_ctx->nbuf) - unmap_single_talitos_ptr(dev, &desc->ptr[3], - DMA_TO_DEVICE); if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); +} + +static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc) +{ + struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct talitos_edesc *next; - if (desc->next_desc) - dma_unmap_single(dev, be32_to_cpu(desc->next_desc), - TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + while (edesc) { + next = edesc->next_desc; + common_nonsnoop_hash_unmap(ctx->dev, edesc, areq); + kfree(edesc); + edesc = next; + } } static void ahash_done(struct device *dev, @@ -1758,30 +1820,46 @@ static void ahash_done(struct device *dev, struct talitos_edesc *edesc = container_of(desc, struct talitos_edesc, desc); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev)); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct talitos_edesc *next; - if (!req_ctx->last_desc && req_ctx->to_hash_later) { - /* Position any partial block for next update/final/finup */ - req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; - req_ctx->nbuf = req_ctx->to_hash_later; - } - common_nonsnoop_hash_unmap(dev, edesc, areq); - - kfree(edesc); + if (is_sec1) { + if (!req_ctx->last_request && req_ctx->to_hash_later) { + /* Position any partial block for next update/final/finup */ + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; + req_ctx->nbuf = req_ctx->to_hash_later; + } - if (err) { + free_edesc_list_from(areq, edesc); ahash_request_complete(areq, err); - return; - } + } else { + next = edesc->next_desc; - req_ctx->remaining_ahash_request_bytes -= - req_ctx->current_ahash_request_bytes; + common_nonsnoop_hash_unmap(dev, edesc, areq); + kfree(edesc); - if (!req_ctx->remaining_ahash_request_bytes) { - ahash_request_complete(areq, 0); - return; - } + if (err) + goto out; - schedule_work(&req_ctx->sec1_ahash_process_remaining); + if (next) { + err = talitos_submit(dev, ctx->ch, &next->desc, + ahash_done, areq); + if (err != -EINPROGRESS) + goto out; + return; + } +out: + if (!req_ctx->last_request && req_ctx->to_hash_later) { + /* Position any partial block for next update/final/finup */ + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; + req_ctx->nbuf = req_ctx->to_hash_later; + } + if (err && next) + free_edesc_list_from(areq, next); + ahash_request_complete(areq, err); + } } /* @@ -1805,18 +1883,15 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, (char *)padded_hash, DMA_TO_DEVICE); } -static int common_nonsnoop_hash(struct talitos_edesc *edesc, - struct ahash_request *areq, unsigned int length, - void (*callback) (struct device *dev, - struct talitos_desc *desc, - void *context, int error)) +static void common_nonsnoop_hash(struct talitos_edesc *edesc, + struct ahash_request *areq, + unsigned int length) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; - int ret; bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1825,7 +1900,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* first DWORD empty */ /* hash context in */ - if (!req_ctx->first_desc || req_ctx->swinit) { + if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) { map_single_talitos_ptr_nosync(dev, &desc->ptr[1], req_ctx->hw_context_size, req_ctx->hw_context, @@ -1833,40 +1908,31 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, req_ctx->swinit = 0; } /* Indicate next op is not the first. */ - req_ctx->first_desc = 0; + req_ctx->first_request = 0; /* HMAC key */ if (ctx->keylen) to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); - if (is_sec1 && req_ctx->nbuf) - length -= req_ctx->nbuf; - sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) - sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); + sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length); else if (length) - sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, - DMA_TO_DEVICE); + sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE); + /* * data in */ - if (is_sec1 && req_ctx->nbuf) { - map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf, - req_ctx->buf[req_ctx->buf_idx], - DMA_TO_DEVICE); - } else { - sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc->ptr[3], sg_count, 0, 0); - if (sg_count > 1) - sync_needed = true; - } + sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3], + sg_count, 0, 0); + if (sg_count > 1) + sync_needed = true; /* fifth DWORD empty */ /* hash/HMAC out -or- hash context out */ - if (req_ctx->last_desc) + if (edesc->last && req_ctx->last_request) map_single_talitos_ptr(dev, &desc->ptr[5], crypto_ahash_digestsize(tfm), req_ctx->hw_context, DMA_FROM_DEVICE); @@ -1881,73 +1947,94 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); - if (is_sec1 && req_ctx->nbuf && length) { - struct talitos_desc *desc2 = (struct talitos_desc *) - (edesc->buf + edesc->dma_len); - dma_addr_t next_desc; - - memset(desc2, 0, sizeof(*desc2)); - desc2->hdr = desc->hdr; - desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; - desc2->hdr1 = desc2->hdr; - desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; - desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; - desc->hdr &= ~DESC_HDR_DONE_NOTIFY; - - if (desc->ptr[1].ptr) - copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1], - is_sec1); - else - map_single_talitos_ptr_nosync(dev, &desc2->ptr[1], - req_ctx->hw_context_size, - req_ctx->hw_context, - DMA_TO_DEVICE); - copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); - sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc2->ptr[3], sg_count, 0, 0); - if (sg_count > 1) - sync_needed = true; - copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); - if (req_ctx->last_desc) - map_single_talitos_ptr_nosync(dev, &desc->ptr[5], - req_ctx->hw_context_size, - req_ctx->hw_context, - DMA_FROM_DEVICE); - - next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); - desc->next_desc = cpu_to_be32(next_desc); - } - if (sync_needed) dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - - ret = talitos_submit(dev, ctx->ch, desc, callback, areq); - if (ret != -EINPROGRESS) { - common_nonsnoop_hash_unmap(dev, edesc, areq); - kfree(edesc); - } - return ret; } static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, + struct scatterlist *src, unsigned int nbytes) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + + return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0, + nbytes, 0, 0, 0, areq->base.flags, false); +} + +static struct talitos_edesc * +ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes, + unsigned int blocksize, bool is_sec1) +{ + struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - struct talitos_private *priv = dev_get_drvdata(ctx->dev); - bool is_sec1 = has_ftr_sec1(priv); + struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc; + size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN : + TALITOS2_MAX_DATA_LEN; + struct scatterlist tmp[2]; + size_t to_hash_this_desc; + struct scatterlist *src; + size_t offset = 0; + + do { + src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset); + + to_hash_this_desc = + min(nbytes, ALIGN_DOWN(desc_max, blocksize)); + + /* Allocate extended descriptor */ + edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc); + if (IS_ERR(edesc)) { + if (first) + free_edesc_list_from(areq, first); + return edesc; + } - if (is_sec1) - nbytes -= req_ctx->nbuf; + edesc->src = + scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset); + edesc->desc.hdr = ctx->desc_hdr_template; + edesc->first = offset == 0; + edesc->last = nbytes - to_hash_this_desc == 0; - return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, - nbytes, 0, 0, 0, areq->base.flags, false); + /* On last one, request SEC to pad; otherwise continue */ + if (req_ctx->last_request && edesc->last) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; + else + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; + + /* request SEC to INIT hash. */ + if (req_ctx->first_request && edesc->first && !req_ctx->swinit) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; + + /* + * When the tfm context has a keylen, it's an HMAC. + * A first or last (ie. not middle) descriptor must request HMAC. + */ + if (ctx->keylen && ((req_ctx->first_request && edesc->first) || + (req_ctx->last_request && edesc->last))) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; + + /* clear the DN bit */ + if (is_sec1 && !edesc->last) + edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY; + + common_nonsnoop_hash(edesc, areq, to_hash_this_desc); + + offset += to_hash_this_desc; + nbytes -= to_hash_this_desc; + + if (!prev_edesc) + first = edesc; + else + prev_edesc->next_desc = edesc; + prev_edesc = edesc; + } while (nbytes); + + return first; } -static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes) +static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); @@ -1955,23 +2042,23 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes struct talitos_edesc *edesc; unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev)); unsigned int nbytes_to_hash; unsigned int to_hash_later; unsigned int nsg; int nents; struct device *dev = ctx->dev; - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; + int ret; - if (!req_ctx->last_desc && (nbytes + req_ctx->nbuf <= blocksize)) { + if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ - nents = sg_nents_for_len(req_ctx->request_sl, nbytes); + nents = sg_nents_for_len(areq->src, nbytes); if (nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); return nents; } - sg_copy_to_buffer(req_ctx->request_sl, nents, + sg_copy_to_buffer(areq->src, nents, ctx_buf + req_ctx->nbuf, nbytes); req_ctx->nbuf += nbytes; return 0; @@ -1981,7 +2068,7 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes nbytes_to_hash = nbytes + req_ctx->nbuf; to_hash_later = nbytes_to_hash & (blocksize - 1); - if (req_ctx->last_desc) + if (req_ctx->last_request) to_hash_later = 0; else if (to_hash_later) /* There is a partial block. Hash the full block(s) now */ @@ -1993,123 +2080,39 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes } /* Chain in any previously buffered data */ - if (!is_sec1 && req_ctx->nbuf) { + if (req_ctx->nbuf) { nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; sg_init_table(req_ctx->bufsl, nsg); sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); if (nsg > 1) - sg_chain(req_ctx->bufsl, 2, req_ctx->request_sl); + sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; - } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { - int offset; - - if (nbytes_to_hash > blocksize) - offset = blocksize - req_ctx->nbuf; - else - offset = nbytes_to_hash - req_ctx->nbuf; - nents = sg_nents_for_len(req_ctx->request_sl, offset); - if (nents < 0) { - dev_err(dev, "Invalid number of src SG.\n"); - return nents; - } - sg_copy_to_buffer(req_ctx->request_sl, nents, - ctx_buf + req_ctx->nbuf, offset); - req_ctx->nbuf += offset; - req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, req_ctx->request_sl, - offset); } else - req_ctx->psrc = req_ctx->request_sl; + req_ctx->psrc = areq->src; if (to_hash_later) { - nents = sg_nents_for_len(req_ctx->request_sl, nbytes); + nents = sg_nents_for_len(areq->src, nbytes); if (nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); return nents; } - sg_pcopy_to_buffer(req_ctx->request_sl, nents, + sg_pcopy_to_buffer(areq->src, nents, req_ctx->buf[(req_ctx->buf_idx + 1) & 1], to_hash_later, nbytes - to_hash_later); } req_ctx->to_hash_later = to_hash_later; - /* Allocate extended descriptor */ - edesc = ahash_edesc_alloc(req_ctx->areq, nbytes_to_hash); + edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize, + is_sec1); if (IS_ERR(edesc)) return PTR_ERR(edesc); - edesc->desc.hdr = ctx->desc_hdr_template; - - /* On last one, request SEC to pad; otherwise continue */ - if (req_ctx->last_desc) - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; - else - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; - - /* request SEC to INIT hash. */ - if (req_ctx->first_desc && !req_ctx->swinit) - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; - - /* When the tfm context has a keylen, it's an HMAC. - * A first or last (ie. not middle) descriptor must request HMAC. - */ - if (ctx->keylen && (req_ctx->first_desc || req_ctx->last_desc)) - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - - return common_nonsnoop_hash(edesc, req_ctx->areq, nbytes_to_hash, ahash_done); -} - -static void sec1_ahash_process_remaining(struct work_struct *work) -{ - struct talitos_ahash_req_ctx *req_ctx = - container_of(work, struct talitos_ahash_req_ctx, - sec1_ahash_process_remaining); - int err = 0; - - req_ctx->request_sl = scatterwalk_ffwd(req_ctx->request_bufsl, - req_ctx->request_sl, TALITOS1_MAX_DATA_LEN); + ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq); + if (ret != -EINPROGRESS) + free_edesc_list_from(areq, edesc); - if (req_ctx->remaining_ahash_request_bytes > TALITOS1_MAX_DATA_LEN) - req_ctx->current_ahash_request_bytes = TALITOS1_MAX_DATA_LEN; - else { - req_ctx->current_ahash_request_bytes = - req_ctx->remaining_ahash_request_bytes; - - if (req_ctx->last_request) - req_ctx->last_desc = 1; - } - - err = ahash_process_req_one(req_ctx->areq, - req_ctx->current_ahash_request_bytes); - - if (err != -EINPROGRESS) - ahash_request_complete(req_ctx->areq, err); -} - -static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); - struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); - struct device *dev = ctx->dev; - struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - req_ctx->areq = areq; - req_ctx->request_sl = areq->src; - req_ctx->remaining_ahash_request_bytes = nbytes; - - if (is_sec1) { - if (nbytes > TALITOS1_MAX_DATA_LEN) - nbytes = TALITOS1_MAX_DATA_LEN; - else if (req_ctx->last_request) - req_ctx->last_desc = 1; - } - - req_ctx->current_ahash_request_bytes = nbytes; - - return ahash_process_req_one(req_ctx->areq, - req_ctx->current_ahash_request_bytes); + return ret; } static int ahash_init(struct ahash_request *areq) @@ -2124,15 +2127,13 @@ static int ahash_init(struct ahash_request *areq) /* Initialize the context */ req_ctx->buf_idx = 0; req_ctx->nbuf = 0; - req_ctx->first_desc = 1; /* first_desc indicates h/w must init its context */ + req_ctx->first_request = 1; req_ctx->swinit = 0; /* assume h/w init of context */ size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; req_ctx->hw_context_size = size; req_ctx->last_request = 0; - req_ctx->last_desc = 0; - INIT_WORK(&req_ctx->sec1_ahash_process_remaining, sec1_ahash_process_remaining); dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, DMA_TO_DEVICE); @@ -2224,8 +2225,8 @@ static int ahash_export(struct ahash_request *areq, void *out) req_ctx->hw_context_size); memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); export->swinit = req_ctx->swinit; - export->first_desc = req_ctx->first_desc; - export->last_desc = req_ctx->last_desc; + export->first_request = req_ctx->first_request; + export->last_request = req_ctx->last_request; export->to_hash_later = req_ctx->to_hash_later; export->nbuf = req_ctx->nbuf; @@ -2250,8 +2251,8 @@ static int ahash_import(struct ahash_request *areq, const void *in) memcpy(req_ctx->hw_context, export->hw_context, size); memcpy(req_ctx->buf[0], export->buf, export->nbuf); req_ctx->swinit = export->swinit; - req_ctx->first_desc = export->first_desc; - req_ctx->last_desc = export->last_desc; + req_ctx->first_request = export->first_request; + req_ctx->last_request = export->last_request; req_ctx->to_hash_later = export->to_hash_later; req_ctx->nbuf = export->nbuf; diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 1a93ee355929d..11f0eb6a41db6 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -44,11 +44,17 @@ struct talitos_desc { /* * talitos_edesc - s/w-extended descriptor + * @bufsl: scatterlist buffer + * @src: pointer to input scatterlist + * @first: first descriptor of a chain + * @last: last descriptor of a chain + * * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist * @iv_dma: dma address of iv for checking continuity and link table * @dma_len: length of dma mapped link_tbl space * @dma_link_tbl: bus physical address of link_tbl/buf + * @next_desc: next descriptor * @desc: h/w descriptor * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) @@ -58,11 +64,17 @@ struct talitos_desc { * of link_tbl data */ struct talitos_edesc { + struct scatterlist bufsl[2]; + struct scatterlist *src; + int first; + int last; + int src_nents; int dst_nents; dma_addr_t iv_dma; int dma_len; dma_addr_t dma_link_tbl; + struct talitos_edesc *next_desc; struct talitos_desc desc; union { DECLARE_FLEX_ARRAY(struct talitos_ptr, link_tbl); diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 30c78afe3dea6..9094c03e991f6 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -1201,6 +1201,7 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se *se = ctx->se; + unsigned int bufsize; int ret; ret = tegra_ccm_crypt_init(req, se, rctx); @@ -1210,19 +1211,19 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) rctx->key_id = ctx->key_id; /* Allocate buffers required */ - rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, + bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; + rctx->inbuf.size = bufsize; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->inbuf.addr, GFP_KERNEL); + ret = -ENOMEM; if (!rctx->inbuf.buf) goto out_finalize; - rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, + rctx->outbuf.size = bufsize; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->outbuf.addr, GFP_KERNEL); - if (!rctx->outbuf.buf) { - ret = -ENOMEM; + if (!rctx->outbuf.buf) goto out_free_inbuf; - } if (!ctx->key_id) { ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, @@ -1254,11 +1255,11 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) } out: - dma_free_coherent(ctx->se->dev, rctx->inbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->outbuf.buf, rctx->outbuf.addr); out_free_inbuf: - dma_free_coherent(ctx->se->dev, rctx->outbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->inbuf.buf, rctx->inbuf.addr); if (tegra_key_is_reserved(rctx->key_id)) @@ -1278,6 +1279,7 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + unsigned int bufsize; int ret; rctx->src_sg = req->src; @@ -1296,16 +1298,17 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) rctx->key_id = ctx->key_id; /* Allocate buffers required */ - rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, + bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen; + rctx->inbuf.size = bufsize; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->inbuf.addr, GFP_KERNEL); if (!rctx->inbuf.buf) { ret = -ENOMEM; goto out_finalize; } - rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, + rctx->outbuf.size = bufsize; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->outbuf.addr, GFP_KERNEL); if (!rctx->outbuf.buf) { ret = -ENOMEM; @@ -1342,11 +1345,11 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) ret = tegra_gcm_do_verify(ctx->se, rctx); out: - dma_free_coherent(ctx->se->dev, rctx->outbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->outbuf.buf, rctx->outbuf.addr); out_free_inbuf: - dma_free_coherent(ctx->se->dev, rctx->inbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->inbuf.buf, rctx->inbuf.addr); if (tegra_key_is_reserved(rctx->key_id)) diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index eb71113ed1469..e8d8c3a23d7ad 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -180,7 +180,7 @@ int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, MAX_SCHEDULE_TIMEOUT, NULL); if (ret) { dev_err(se->dev, "host1x job timed out\n"); - return ret; + goto job_put; } host1x_job_put(job); diff --git a/drivers/crypto/xilinx/xilinx-trng.c b/drivers/crypto/xilinx/xilinx-trng.c index 5276ac2d82bbe..e3ced111c4095 100644 --- a/drivers/crypto/xilinx/xilinx-trng.c +++ b/drivers/crypto/xilinx/xilinx-trng.c @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -14,14 +13,12 @@ #include #include #include -#include #include #include #include #include #include #include -#include /* TRNG Registers Offsets */ #define TRNG_STATUS_OFFSET 0x4U @@ -61,16 +58,9 @@ struct xilinx_rng { struct device *dev; unsigned char *scratchpadbuf; struct aes_enckey *aeskey; - struct mutex lock; /* Protect access to TRNG device */ struct hwrng trng; }; -struct xilinx_rng_ctx { - struct xilinx_rng *rng; -}; - -static struct xilinx_rng *xilinx_rng_dev; - static void xtrng_readwrite32(void __iomem *addr, u32 mask, u8 value) { u32 val; @@ -246,47 +236,6 @@ static int xtrng_random_bytes_generate(struct xilinx_rng *rng, u8 *rand_buf_ptr, return nbytes; } -static int xtrng_trng_generate(struct crypto_rng *tfm, const u8 *src, u32 slen, - u8 *dst, u32 dlen) -{ - struct xilinx_rng_ctx *ctx = crypto_rng_ctx(tfm); - int ret; - - mutex_lock(&ctx->rng->lock); - ret = xtrng_random_bytes_generate(ctx->rng, dst, dlen, true); - mutex_unlock(&ctx->rng->lock); - - return ret < 0 ? ret : 0; -} - -static int xtrng_trng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) -{ - return 0; -} - -static int xtrng_trng_init(struct crypto_tfm *rtfm) -{ - struct xilinx_rng_ctx *ctx = crypto_tfm_ctx(rtfm); - - ctx->rng = xilinx_rng_dev; - - return 0; -} - -static struct rng_alg xtrng_trng_alg = { - .generate = xtrng_trng_generate, - .seed = xtrng_trng_seed, - .seedsize = 0, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "xilinx-trng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct xilinx_rng_ctx), - .cra_module = THIS_MODULE, - .cra_init = xtrng_trng_init, - }, -}; - static int xtrng_hwrng_trng_read(struct hwrng *hwrng, void *data, size_t max, bool wait) { u8 buf[TRNG_SEC_STRENGTH_BYTES]; @@ -294,12 +243,6 @@ static int xtrng_hwrng_trng_read(struct hwrng *hwrng, void *data, size_t max, bo int ret = -EINVAL, i = 0; rng = container_of(hwrng, struct xilinx_rng, trng); - /* Return in case wait not set and lock not available. */ - if (!mutex_trylock(&rng->lock) && !wait) - return 0; - else if (!mutex_is_locked(&rng->lock) && wait) - mutex_lock(&rng->lock); - while (i < max) { ret = xtrng_random_bytes_generate(rng, buf, TRNG_SEC_STRENGTH_BYTES, wait); if (ret < 0) @@ -308,8 +251,6 @@ static int xtrng_hwrng_trng_read(struct hwrng *hwrng, void *data, size_t max, bo memcpy(data + i, buf, min_t(int, ret, (max - i))); i += min_t(int, ret, (max - i)); } - mutex_unlock(&rng->lock); - return ret; } @@ -355,40 +296,24 @@ static int xtrng_probe(struct platform_device *pdev) sb_size = crypto_drbg_ctr_df_datalen(TRNG_SEED_LEN_BYTES, AES_BLOCK_SIZE); rng->scratchpadbuf = devm_kzalloc(&pdev->dev, sb_size, GFP_KERNEL); - if (!rng->scratchpadbuf) { - ret = -ENOMEM; - goto end; - } + if (!rng->scratchpadbuf) + return -ENOMEM; xtrng_trng_reset(rng->rng_base); ret = xtrng_reseed_internal(rng); if (ret) { dev_err(&pdev->dev, "TRNG Seed fail\n"); - goto end; - } - - xilinx_rng_dev = rng; - mutex_init(&rng->lock); - ret = crypto_register_rng(&xtrng_trng_alg); - if (ret) { - dev_err(&pdev->dev, "Crypto Random device registration failed: %d\n", ret); - goto end; + return ret; } ret = xtrng_hwrng_register(&rng->trng); if (ret) { dev_err(&pdev->dev, "HWRNG device registration failed: %d\n", ret); - goto crypto_rng_free; + return ret; } platform_set_drvdata(pdev, rng); return 0; - -crypto_rng_free: - crypto_unregister_rng(&xtrng_trng_alg); - -end: - return ret; } static void xtrng_remove(struct platform_device *pdev) @@ -398,13 +323,11 @@ static void xtrng_remove(struct platform_device *pdev) rng = platform_get_drvdata(pdev); xtrng_hwrng_unregister(&rng->trng); - crypto_unregister_rng(&xtrng_trng_alg); xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, zero, TRNG_NUM_INIT_REGS); xtrng_write_multiple_registers(rng->rng_base + TRNG_PER_STRNG_OFFSET, zero, TRNG_NUM_INIT_REGS); xtrng_hold_reset(rng->rng_base); - xilinx_rng_dev = NULL; } static const struct of_device_id xtrng_of_match[] = { diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 82ca3a4767080..07555ae638594 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -52,6 +52,7 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, u64 dpa); int devm_cxl_add_dax_region(struct cxl_region *cxlr); int devm_cxl_add_pmem_region(struct cxl_region *cxlr); +void kill_regions(struct cxl_root_decoder *cxlrd); #else static inline u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, @@ -81,6 +82,7 @@ static inline int cxl_region_init(void) static inline void cxl_region_exit(void) { } +static inline void kill_regions(struct cxl_root_decoder *cxlrd) { }; #define CXL_REGION_ATTR(x) NULL #define CXL_REGION_TYPE(x) NULL #define SET_CXL_REGION_ATTR(x) diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c index 3435db9ea6b11..85185af46b72d 100644 --- a/drivers/cxl/core/features.c +++ b/drivers/cxl/core/features.c @@ -423,6 +423,7 @@ static void *cxlctl_get_supported_features(struct cxl_features_state *cxlfs, rpc_out->size = struct_size(feat_out, ents, requested); feat_out = &rpc_out->get_sup_feats_out; + feat_out->num_entries = cpu_to_le16(requested); for (i = start, pos = &feat_out->ents[0]; i < cxlfs->entries->num_features; i++, pos++) { @@ -444,7 +445,6 @@ static void *cxlctl_get_supported_features(struct cxl_features_state *cxlfs, } } - feat_out->num_entries = cpu_to_le16(requested); feat_out->supported_feats = cpu_to_le16(cxlfs->entries->num_features); rpc_out->retval = CXL_MBOX_CMD_RC_SUCCESS; *out_len = out_size; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 80e65690eb773..91c99eeea92ce 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -25,9 +25,11 @@ static DEFINE_IDA(cxl_memdev_ida); static void cxl_memdev_release(struct device *dev) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct device *parent = dev->parent; ida_free(&cxl_memdev_ida, cxlmd->id); kfree(cxlmd); + put_device(parent); } static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, @@ -707,7 +709,7 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, dev = &cxlmd->dev; device_initialize(dev); lockdep_set_class(&dev->mutex, &cxl_memdev_key); - dev->parent = cxlds->dev; + dev->parent = get_device(cxlds->dev); dev->bus = &cxl_bus_type; dev->devt = MKDEV(cxl_mem_major, cxlmd->id); dev->type = &cxl_memdev_type; diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index d1f487b3d809a..e4338fd7e01b4 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -89,7 +89,7 @@ static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id) d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id), &temp); if (rc) - return rc; + return pcibios_err_to_errno(rc); valid = FIELD_GET(PCI_DVSEC_CXL_MEM_INFO_VALID, temp); if (valid) @@ -123,7 +123,7 @@ static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id) rc = pci_read_config_dword( pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id), &temp); if (rc) - return rc; + return pcibios_err_to_errno(rc); active = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE, temp); if (active) @@ -156,7 +156,7 @@ int cxl_await_media_ready(struct cxl_dev_state *cxlds) rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CAP, &cap); if (rc) - return rc; + return pcibios_err_to_errno(rc); hdm_count = FIELD_GET(PCI_DVSEC_CXL_HDM_COUNT, cap); for (i = 0; i < hdm_count; i++) { @@ -187,8 +187,8 @@ static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val) int rc; rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl); - if (rc < 0) - return rc; + if (rc) + return pcibios_err_to_errno(rc); if ((ctrl & PCI_DVSEC_CXL_MEM_ENABLE) == val) return 1; @@ -196,8 +196,8 @@ static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val) ctrl |= val; rc = pci_write_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, ctrl); - if (rc < 0) - return rc; + if (rc) + return pcibios_err_to_errno(rc); return 0; } @@ -275,7 +275,7 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds, rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CAP, &cap); if (rc) - return rc; + return pcibios_err_to_errno(rc); if (!(cap & PCI_DVSEC_CXL_MEM_CAPABLE)) { dev_dbg(dev, "Not MEM Capable\n"); @@ -299,7 +299,7 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds, */ rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl); if (rc) - return rc; + return pcibios_err_to_errno(rc); info->mem_enabled = FIELD_GET(PCI_DVSEC_CXL_MEM_ENABLE, ctrl); if (!info->mem_enabled) @@ -316,14 +316,14 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds, rc = pci_read_config_dword( pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_HIGH(i), &temp); if (rc) - return rc; + return pcibios_err_to_errno(rc); size = (u64)temp << 32; rc = pci_read_config_dword( pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(i), &temp); if (rc) - return rc; + return pcibios_err_to_errno(rc); size |= temp & PCI_DVSEC_CXL_MEM_SIZE_LOW; if (!size) { @@ -333,14 +333,14 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds, rc = pci_read_config_dword( pdev, d + PCI_DVSEC_CXL_RANGE_BASE_HIGH(i), &temp); if (rc) - return rc; + return pcibios_err_to_errno(rc); base = (u64)temp << 32; rc = pci_read_config_dword( pdev, d + PCI_DVSEC_CXL_RANGE_BASE_LOW(i), &temp); if (rc) - return rc; + return pcibios_err_to_errno(rc); base |= temp & PCI_DVSEC_CXL_MEM_BASE_LOW; diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index c5aacd7054f1d..1215ee4f40351 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -458,6 +458,8 @@ static void cxl_root_decoder_release(struct device *dev) if (atomic_read(&cxlrd->region_id) >= 0) memregion_free(atomic_read(&cxlrd->region_id)); + mutex_destroy(&cxlrd->regions_lock); + xa_destroy(&cxlrd->regions); __cxl_decoder_release(&cxlrd->cxlsd.cxld); kfree(cxlrd); } @@ -2016,7 +2018,8 @@ struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, return ERR_PTR(rc); } - mutex_init(&cxlrd->range_lock); + mutex_init(&cxlrd->regions_lock); + xa_init(&cxlrd->regions); cxld = &cxlsd->cxld; cxld->dev.type = &cxl_decoder_root_type; @@ -2192,6 +2195,8 @@ static void cxld_unregister(void *dev) if (is_endpoint_decoder(dev)) cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1, DETACH_INVALIDATE); + if (is_root_decoder(dev)) + kill_regions(to_cxl_root_decoder(dev)); device_unregister(dev); } diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c index 006c6ffc2f567..99fb00949c2fa 100644 --- a/drivers/cxl/core/ras.c +++ b/drivers/cxl/core/ras.c @@ -8,6 +8,10 @@ #include #include "trace.h" +/* Check that UCE header definition is maintained to keep ABI intact */ +static_assert(CXL_HEADERLOG_TRACE_SIZE_U32 == 128, + "rasdaemon ABI requires exactly 128 u32s"); + static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, struct cxl_ras_capability_regs ras_cap) { @@ -19,6 +23,7 @@ static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, struct cxl_ras_capability_regs ras_cap) { + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; u32 fe; @@ -28,8 +33,8 @@ static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, else fe = status; - trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, - ras_cap.header_log); + memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); + trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, hl); } static void cxl_cper_trace_corr_prot_err(struct cxl_memdev *cxlmd, @@ -44,6 +49,7 @@ static void cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, struct cxl_ras_capability_regs ras_cap) { + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; u32 fe; @@ -53,8 +59,15 @@ cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, else fe = status; - trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, - ras_cap.header_log); + /* + * ras_cap.header_log[] holds CXL_HEADERLOG_SIZE_U32 (16) hardware + * dwords. Copy them into the front of a zero-filled + * CXL_HEADERLOG_TRACE_SIZE_U32 (128) u32 staging buffer so the trace + * event memcpy sees a full 512-byte source and the userspace ABI + * (rasdaemon) is preserved. + */ + memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); + trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl); } static int match_memdev_by_parent(struct device *dev, const void *uport) @@ -204,12 +217,12 @@ static void header_log_copy(void __iomem *ras_base, u32 *log) { void __iomem *addr; u32 *log_addr; - int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32); + int i; addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; log_addr = log; - for (i = 0; i < log_u32_size; i++) { + for (i = 0; i < CXL_HEADERLOG_SIZE_U32; i++) { *log_addr = readl(addr); log_addr++; addr += sizeof(u32); @@ -222,7 +235,7 @@ static void header_log_copy(void __iomem *ras_base, u32 *log) */ bool cxl_handle_ras(struct device *dev, void __iomem *ras_base) { - u32 hl[CXL_HEADERLOG_SIZE_U32]; + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; void __iomem *addr; u32 status; u32 fe; diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index e50dc716d4e82..fda9e97a6ddcd 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1848,8 +1848,21 @@ static int cxl_region_attach_auto(struct cxl_region *cxlr, * this means that userspace can view devices in the wrong position * before the region activates, and must be careful to understand when * it might be racing region autodiscovery. + * + * The endpoint decoder will be recorded into the first free slot of + * the target array. */ - pos = p->nr_targets; + for (pos = 0; pos < p->interleave_ways; pos++) { + if (!p->targets[pos]) + break; + } + + if (pos == p->interleave_ways) { + dev_err(&cxlr->dev, "%s: unable to find a free target slot\n", + dev_name(&cxled->cxld.dev)); + return -ENXIO; + } + p->targets[pos] = cxled; cxled->pos = pos; cxled->state = CXL_DECODER_STATE_AUTO_STAGED; @@ -2011,8 +2024,9 @@ static int cxl_region_sort_targets(struct cxl_region *cxlr) cxled->pos = cxl_calc_interleave_pos(cxled, &cxlr->hpa_range); /* * Record that sorting failed, but still continue to calc - * cxled->pos so that follow-on code paths can reliably - * do p->targets[cxled->pos] to self-reference their entry. + * cxled->pos so that cxl_calc_interleave_pos() emits its + * dev_dbg() for every member. which is useful for auto + * discovery debug. */ if (cxled->pos < 0) rc = -ENXIO; @@ -2202,18 +2216,30 @@ static int cxl_region_attach(struct cxl_region *cxlr, return 0; } -static int cxl_region_by_target(struct device *dev, const void *data) +static int cxl_region_remove_target(struct device *dev, void *data) { - const struct cxl_endpoint_decoder *cxled = data; + struct cxl_endpoint_decoder *cxled = data; struct cxl_region_params *p; struct cxl_region *cxlr; + int i; if (!is_cxl_region(dev)) return 0; cxlr = to_cxl_region(dev); p = &cxlr->params; - return p->targets[cxled->pos] == cxled; + for (i = 0; i < p->interleave_ways; i++) { + if (p->targets[i] == cxled) { + p->nr_targets--; + cxled->state = CXL_DECODER_STATE_AUTO; + cxled->pos = -1; + p->targets[i] = NULL; + + return 1; + } + } + + return 0; } /* @@ -2222,25 +2248,10 @@ static int cxl_region_by_target(struct device *dev, const void *data) */ static void cxl_cancel_auto_attach(struct cxl_endpoint_decoder *cxled) { - struct cxl_region_params *p; - struct cxl_region *cxlr; - int pos = cxled->pos; - if (cxled->state != CXL_DECODER_STATE_AUTO_STAGED) return; - struct device *dev __free(put_device) = - bus_find_device(&cxl_bus_type, NULL, cxled, cxl_region_by_target); - if (!dev) - return; - - cxlr = to_cxl_region(dev); - p = &cxlr->params; - - p->nr_targets--; - cxled->state = CXL_DECODER_STATE_AUTO; - cxled->pos = -1; - p->targets[pos] = NULL; + bus_for_each_dev(&cxl_bus_type, NULL, cxled, cxl_region_remove_target); } static struct cxl_region * @@ -2537,12 +2548,13 @@ static struct cxl_region *to_cxl_region(struct device *dev) return container_of(dev, struct cxl_region, dev); } -static void unregister_region(void *_cxlr) +static void unregister_region(struct cxl_region *cxlr) { - struct cxl_region *cxlr = _cxlr; + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); struct cxl_region_params *p = &cxlr->params; int i; + xa_erase(&cxlrd->regions, cxlr->id); device_del(&cxlr->dev); /* @@ -2673,6 +2685,19 @@ static int cxl_region_calculate_adistance(struct notifier_block *nb, return NOTIFY_STOP; } +/* unwind all remaining regions */ +void kill_regions(struct cxl_root_decoder *cxlrd) +{ + unsigned long index; + struct cxl_region *cxlr; + + guard(mutex)(&cxlrd->regions_lock); + /* no more region creation */ + cxlrd->dead = true; + xa_for_each(&cxlrd->regions, index, cxlr) + unregister_region(cxlr); +} + /** * devm_cxl_add_region - Adds a region to a decoder * @cxlrd: root decoder @@ -2711,14 +2736,15 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; - rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); - if (rc) + rc = xa_insert(&cxlrd->regions, cxlr->id, cxlr, GFP_KERNEL); + if (rc) { + unregister_region(cxlr); return ERR_PTR(rc); + } dev_dbg(port->uport_dev, "%s: created %s\n", dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); return cxlr; - err: put_device(dev); return ERR_PTR(rc); @@ -2747,6 +2773,9 @@ static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, { int rc; + if (cxlrd->dead) + return ERR_PTR(-ENXIO); + switch (mode) { case CXL_PARTMODE_RAM: case CXL_PARTMODE_PMEM: @@ -2779,6 +2808,10 @@ static ssize_t create_region_store(struct device *dev, const char *buf, if (rc != 1) return -EINVAL; + ACQUIRE(mutex_intr, regions_lock)(&cxlrd->regions_lock); + if ((rc = ACQUIRE_ERR(mutex_intr, ®ions_lock))) + return rc; + cxlr = __create_region(cxlrd, mode, id, CXL_DECODER_HOSTONLYMEM); if (IS_ERR(cxlr)) return PTR_ERR(cxlr); @@ -2818,33 +2851,27 @@ static ssize_t region_show(struct device *dev, struct device_attribute *attr, } DEVICE_ATTR_RO(region); -static struct cxl_region * -cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) -{ - struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; - struct device *region_dev; - - region_dev = device_find_child_by_name(&cxld->dev, name); - if (!region_dev) - return ERR_PTR(-ENODEV); - - return to_cxl_region(region_dev); -} - static ssize_t delete_region_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); - struct cxl_port *port = to_cxl_port(dev->parent); struct cxl_region *cxlr; + int rc, id; - cxlr = cxl_find_region_by_name(cxlrd, buf); - if (IS_ERR(cxlr)) - return PTR_ERR(cxlr); + ACQUIRE(mutex_intr, regions_lock)(&cxlrd->regions_lock); + if ((rc = ACQUIRE_ERR(mutex_intr, ®ions_lock))) + return rc; - devm_release_action(port->uport_dev, unregister_region, cxlr); - put_device(&cxlr->dev); + rc = sscanf(buf, "region%d\n", &id); + if (rc != 1) + return -EINVAL; + + cxlr = xa_load(&cxlrd->regions, id); + if (!cxlr || !sysfs_streq(buf, dev_name(&cxlr->dev))) + return -ENODEV; + + unregister_region(cxlr); return len; } @@ -3709,7 +3736,6 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, { struct cxl_endpoint_decoder *cxled = ctx->cxled; struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); - struct cxl_port *port = cxlrd_to_port(cxlrd); struct cxl_dev_state *cxlds = cxlmd->cxlds; int rc, part = READ_ONCE(cxled->part); struct cxl_region *cxlr; @@ -3730,7 +3756,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, rc = __construct_region(cxlr, ctx); if (rc) { - devm_release_action(port->uport_dev, unregister_region, cxlr); + unregister_region(cxlr); return ERR_PTR(rc); } @@ -3776,12 +3802,11 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled) * for the HPA range, one does the construction and the others * add to that. */ - mutex_lock(&cxlrd->range_lock); + guard(mutex)(&cxlrd->regions_lock); struct cxl_region *cxlr __free(put_cxl_region) = cxl_find_region_by_range(cxlrd, &ctx.hpa_range); if (!cxlr) cxlr = construct_region(cxlrd, &ctx); - mutex_unlock(&cxlrd->range_lock); rc = PTR_ERR_OR_ZERO(cxlr); if (rc) diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index a972e4ef19368..d37876096dd7c 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -56,7 +56,7 @@ TRACE_EVENT(cxl_port_aer_uncorrectable_error, __string(host, dev_name(dev->parent)) __field(u32, status) __field(u32, first_error) - __array(u32, header_log, CXL_HEADERLOG_SIZE_U32) + __array(u32, header_log, CXL_HEADERLOG_TRACE_SIZE_U32) ), TP_fast_assign( __assign_str(device); @@ -64,10 +64,14 @@ TRACE_EVENT(cxl_port_aer_uncorrectable_error, __entry->status = status; __entry->first_error = fe; /* - * Embed the 512B headerlog data for user app retrieval and - * parsing, but no need to print this in the trace buffer. + * Embed headerlog data for user app retrieval and parsing, + * but no need to print in the trace buffer. Only + * CXL_HEADERLOG_SIZE_U32 (16) dwords are hardware data; + * the remaining entries preserve the 512-byte ABI layout + * rasdaemon depends on and are zero-filled by the caller. */ - memcpy(__entry->header_log, hl, CXL_HEADERLOG_SIZE); + memcpy(__entry->header_log, hl, + CXL_HEADERLOG_TRACE_SIZE_U32 * sizeof(u32)); ), TP_printk("device=%s host=%s status: '%s' first_error: '%s'", __get_str(device), __get_str(host), @@ -85,7 +89,7 @@ TRACE_EVENT(cxl_aer_uncorrectable_error, __field(u64, serial) __field(u32, status) __field(u32, first_error) - __array(u32, header_log, CXL_HEADERLOG_SIZE_U32) + __array(u32, header_log, CXL_HEADERLOG_TRACE_SIZE_U32) ), TP_fast_assign( __assign_str(memdev); @@ -94,10 +98,14 @@ TRACE_EVENT(cxl_aer_uncorrectable_error, __entry->status = status; __entry->first_error = fe; /* - * Embed the 512B headerlog data for user app retrieval and - * parsing, but no need to print this in the trace buffer. + * Embed headerlog data for user app retrieval and parsing, + * but no need to print in the trace buffer. Only + * CXL_HEADERLOG_SIZE_U32 (16) dwords are hardware data; + * the remaining entries preserve the 512-byte ABI layout + * rasdaemon depends on and are zero-filled by the caller. */ - memcpy(__entry->header_log, hl, CXL_HEADERLOG_SIZE); + memcpy(__entry->header_log, hl, + CXL_HEADERLOG_TRACE_SIZE_U32 * sizeof(u32)); ), TP_printk("memdev=%s host=%s serial=%lld: status: '%s' first_error: '%s'", __get_str(memdev), __get_str(host), __entry->serial, diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 1297594beaec3..765419af2c6b7 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -158,8 +158,18 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw) #define CXL_RAS_CAP_CONTROL_FE_MASK GENMASK(5, 0) #define CXL_RAS_HEADER_LOG_OFFSET 0x18 #define CXL_RAS_CAPABILITY_LENGTH 0x58 -#define CXL_HEADERLOG_SIZE SZ_512 -#define CXL_HEADERLOG_SIZE_U32 SZ_512 / sizeof(u32) +#define CXL_HEADERLOG_SIZE SZ_64 +#define CXL_HEADERLOG_SIZE_U32 (CXL_HEADERLOG_SIZE / sizeof(u32)) + +/* + * The RAS UCE trace event header array was originally sized at SZ_512/sizeof(u32) + * = 128 u32s due to a bug. Userspace tools (rasdaemon) have grown a dependency + * on that 512-byte layout. Keep the trace array at 128 u32s to preserve the + * ABI; only CXL_HEADERLOG_SIZE_U32 (16) dwords are valid hardware data, the + * remainder are zero-filled. + */ +#define CXL_HEADERLOG_TRACE_SIZE SZ_512 +#define CXL_HEADERLOG_TRACE_SIZE_U32 (CXL_HEADERLOG_TRACE_SIZE / sizeof(u32)) /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ #define CXLDEV_CAP_ARRAY_OFFSET 0x0 @@ -359,7 +369,9 @@ struct cxl_rd_ops { * @cache_size: extended linear cache size if exists, otherwise zero. * @region_id: region id for next region provisioning event * @platform_data: platform specific configuration data - * @range_lock: sync region autodiscovery by address range + * @regions_lock: sync region discovery, construction, and deletion + * @regions: regions to remove at root decoder destruct time + * @dead: root decoder dead to region creation * @qos_class: QoS performance class cookie * @ops: CXL root decoder operations * @cxlsd: base cxl switch decoder @@ -369,7 +381,9 @@ struct cxl_root_decoder { resource_size_t cache_size; atomic_t region_id; void *platform_data; - struct mutex range_lock; + struct mutex regions_lock; + struct xarray regions; + bool dead; int qos_class; struct cxl_rd_ops ops; struct cxl_switch_decoder cxlsd; diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 2cc8749bc8711..a18e2b968e4da 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -227,6 +227,12 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) if (rc) continue; + /* range was never added during probe */ + if (!data->res[i]) { + success++; + continue; + } + rc = remove_memory(range.start, range_len(&range)); if (rc == 0) { remove_resource(data->res[i]); diff --git a/drivers/dibs/dibs_loopback.c b/drivers/dibs/dibs_loopback.c index ec3b48cb0e874..0f2e093111526 100644 --- a/drivers/dibs/dibs_loopback.c +++ b/drivers/dibs/dibs_loopback.c @@ -254,6 +254,11 @@ static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok, read_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; } + if ((u64)offset + size > rmb_node->len) { + read_unlock_bh(&ldev->dmb_ht_lock); + return -EINVAL; + } + memcpy((char *)rmb_node->cpu_addr + offset, data, size); sba_idx = rmb_node->sba_idx; read_unlock_bh(&ldev->dmb_ht_lock); diff --git a/drivers/dma-buf/dma-fence-unwrap.c b/drivers/dma-buf/dma-fence-unwrap.c index 07fe9bf45aea7..cc11c036f2b1d 100644 --- a/drivers/dma-buf/dma-fence-unwrap.c +++ b/drivers/dma-buf/dma-fence-unwrap.c @@ -97,6 +97,9 @@ int dma_fence_dedup_array(struct dma_fence **fences, int num_fences) { int i, j; + if (!num_fences) + return 0; + sort(fences, num_fences, sizeof(*fences), fence_cmp, NULL); /* diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index a2aa82f4eedd4..87797bea91cbd 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -363,6 +363,8 @@ void dma_fence_signal_timestamp_locked(struct dma_fence *fence, &fence->flags))) return; + trace_dma_fence_signaled(fence); + /* * When neither a release nor a wait operation is specified set the ops * pointer to NULL to allow the fence structure to become independent @@ -377,7 +379,6 @@ void dma_fence_signal_timestamp_locked(struct dma_fence *fence, fence->timestamp = timestamp; set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); - trace_dma_fence_signaled(fence); list_for_each_entry_safe(cur, tmp, &cb_list, node) { INIT_LIST_HEAD(&cur->node); @@ -1167,7 +1168,7 @@ const char __rcu *dma_fence_driver_name(struct dma_fence *fence) /* RCU protection is required for safe access to returned string */ ops = rcu_dereference(fence->ops); - if (!dma_fence_test_signaled_flag(fence)) + if (ops) return (const char __rcu *)ops->get_driver_name(fence); else return (const char __rcu *)"detached-driver"; @@ -1200,8 +1201,8 @@ const char __rcu *dma_fence_timeline_name(struct dma_fence *fence) /* RCU protection is required for safe access to returned string */ ops = rcu_dereference(fence->ops); - if (!dma_fence_test_signaled_flag(fence)) - return (const char __rcu *)ops->get_driver_name(fence); + if (ops) + return (const char __rcu *)ops->get_timeline_name(fence); else return (const char __rcu *)"signaled-timeline"; } diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index 94b8ecb892bb1..2f94be1df525a 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -40,6 +40,7 @@ struct udmabuf { struct folio **pinned_folios; struct sg_table *sg; + enum dma_data_direction sg_dir; struct miscdevice *device; pgoff_t *offsets; }; @@ -162,7 +163,7 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, sg_set_folio(sgl, ubuf->folios[i], PAGE_SIZE, ubuf->offsets[i]); - ret = dma_map_sgtable(dev, sg, direction, 0); + ret = dma_map_sgtable(dev, sg, direction, DMA_ATTR_SKIP_CPU_SYNC); if (ret < 0) goto err_map; return sg; @@ -177,7 +178,7 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, static void put_sg_table(struct device *dev, struct sg_table *sg, enum dma_data_direction direction) { - dma_unmap_sgtable(dev, sg, direction, 0); + dma_unmap_sgtable(dev, sg, direction, DMA_ATTR_SKIP_CPU_SYNC); sg_free_table(sg); kfree(sg); } @@ -235,7 +236,7 @@ static void release_udmabuf(struct dma_buf *buf) struct device *dev = ubuf->device->this_device; if (ubuf->sg) - put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL); + put_sg_table(dev, ubuf->sg, ubuf->sg_dir); deinit_udmabuf(ubuf); kfree(ubuf); @@ -253,6 +254,8 @@ static int begin_cpu_udmabuf(struct dma_buf *buf, if (IS_ERR(ubuf->sg)) { ret = PTR_ERR(ubuf->sg); ubuf->sg = NULL; + } else { + ubuf->sg_dir = direction; } } else { dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction); diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 45c2c8e4bc451..ce5cabc460651 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -147,6 +148,7 @@ struct axi_dmac_chan { struct virt_dma_chan vchan; struct axi_dmac_desc *next_desc; + void *pool; struct list_head active_descs; enum dma_transfer_direction direction; @@ -650,11 +652,17 @@ static void axi_dmac_issue_pending(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); } +static void axi_dmac_free_desc(struct axi_dmac_desc *desc) +{ + for (unsigned int i = 0; i < desc->num_sgs; i++) + dma_pool_free(desc->chan->pool, desc->sg[i].hw, desc->sg[i].hw_phys); + + kfree(desc); +} + static struct axi_dmac_desc * axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) { - struct axi_dmac *dmac = chan_to_axi_dmac(chan); - struct device *dev = dmac->dma_dev.dev; struct axi_dmac_hw_desc *hws; struct axi_dmac_desc *desc; dma_addr_t hw_phys; @@ -666,22 +674,22 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) desc->num_sgs = num_sgs; desc->chan = chan; - hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)), - &hw_phys, GFP_ATOMIC); - if (!hws) { - kfree(desc); - return NULL; - } - for (i = 0; i < num_sgs; i++) { - desc->sg[i].hw = &hws[i]; - desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws); + hws = dma_pool_zalloc(chan->pool, GFP_NOWAIT, &hw_phys); + if (!hws) { + desc->num_sgs = i; + axi_dmac_free_desc(desc); + return NULL; + } + + desc->sg[i].hw = hws; + desc->sg[i].hw_phys = hw_phys; - hws[i].id = AXI_DMAC_SG_UNUSED; - hws[i].flags = 0; + hws->id = AXI_DMAC_SG_UNUSED; /* Link hardware descriptors */ - hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws); + if (i) + desc->sg[i - 1].hw->next_sg_addr = hw_phys; } /* The last hardware descriptor will trigger an interrupt */ @@ -690,18 +698,6 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) return desc; } -static void axi_dmac_free_desc(struct axi_dmac_desc *desc) -{ - struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan); - struct device *dev = dmac->dma_dev.dev; - struct axi_dmac_hw_desc *hw = desc->sg[0].hw; - dma_addr_t hw_phys = desc->sg[0].hw_phys; - - dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)), - hw, hw_phys); - kfree(desc); -} - static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, enum dma_transfer_direction direction, dma_addr_t addr, unsigned int num_periods, unsigned int period_len, @@ -769,7 +765,7 @@ axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, for (i = 0; i < nb; i++) { if (!axi_dmac_check_addr(chan, vecs[i].addr) || !axi_dmac_check_len(chan, vecs[i].len)) { - kfree(desc); + axi_dmac_free_desc(desc); return NULL; } @@ -935,9 +931,26 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); } +static int axi_dmac_alloc_chan_resources(struct dma_chan *c) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct device *dev = c->device->dev; + + chan->pool = dma_pool_create(dev_name(dev), dev, + sizeof(struct axi_dmac_hw_desc), + __alignof__(struct axi_dmac_hw_desc), 0); + if (!chan->pool) + return -ENOMEM; + + return 0; +} + static void axi_dmac_free_chan_resources(struct dma_chan *c) { + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + vchan_free_chan_resources(to_virt_chan(c)); + dma_pool_destroy(chan->pool); } static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) @@ -1239,6 +1252,7 @@ static int axi_dmac_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask); + dma_dev->device_alloc_chan_resources = axi_dmac_alloc_chan_resources; dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources; dma_dev->device_tx_status = dma_cookie_status; dma_dev->device_issue_pending = axi_dmac_issue_pending; diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 405bd2fbb4a3b..9049171df8578 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -905,11 +905,12 @@ void dma_release_channel(struct dma_chan *chan) mutex_lock(&dma_list_mutex); WARN_ONCE(chan->client_count != 1, "chan reference count %d != 1\n", chan->client_count); - dma_chan_put(chan); /* drop PRIVATE cap enabled by __dma_request_channel() */ if (--chan->device->privatecnt == 0) dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask); + dma_chan_put(chan); + if (chan->slave) { sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME); sysfs_remove_link(&chan->slave->kobj, chan->name); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 902574b1ba867..6474cacf71953 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -109,7 +109,7 @@ struct dw_edma { struct dw_edma_chan *chan; - raw_spinlock_t lock; /* Only for legacy */ + raw_spinlock_t lock; /* Protect v0 shared registers */ struct dw_edma_chip *chip; diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 0b30ce1385036..6639efffca584 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -314,6 +314,9 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, int i, mask; bool non_ll = false; + if (!pdata) + return -ENODEV; + struct dw_edma_pcie_data *vsec_data __free(kfree) = kmalloc_obj(*vsec_data); if (!vsec_data) diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 69e8279adec81..cfdd6463252e6 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -364,6 +364,7 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; struct dw_edma *dw = chan->dw; + unsigned long flags; u32 tmp; dw_edma_v0_core_write_chunk(chunk); @@ -408,6 +409,8 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) } } /* Interrupt unmask - done, abort */ + raw_spin_lock_irqsave(&dw->lock, flags); + tmp = GET_RW_32(dw, chan->dir, int_mask); tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); @@ -416,6 +419,9 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); SET_RW_32(dw, chan->dir, linked_list_err_en, tmp); + + raw_spin_unlock_irqrestore(&dw->lock, flags); + /* Channel control */ SET_CH_32(dw, chan->dir, chan->id, ch_control1, (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index f95411363ea9a..1dce26d4da835 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -10,9 +10,6 @@ #endif /* PCI Config */ -#define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb -#define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 -#define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 #define PCI_DEVICE_ID_INTEL_IAA_PTL 0xb02d #define PCI_DEVICE_ID_INTEL_IAA_WCL 0xfd2d diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 3d527883776b4..36368835a845c 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -2364,7 +2364,9 @@ static int sdma_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "failed to register controller\n"); - spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus"); + struct device_node *sdma_parent_np __free(device_node) = of_get_parent(np); + + spba_bus = of_get_compatible_child(sdma_parent_np, "fsl,spba-bus"); ret = of_address_to_resource(spba_bus, 0, &spba_res); if (!ret) { sdma->spba_start_addr = spba_res.start; diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index c9a6f610ffd9f..a5055a6273af6 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -2260,6 +2260,7 @@ static int gpi_probe(struct platform_device *pdev) /* clear and Set capabilities */ dma_cap_zero(gpi_dev->dma_device.cap_mask); dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask); + dma_cap_set(DMA_PRIVATE, gpi_dev->dma_device.cap_mask); /* configure dmaengine apis */ gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index 625ff29024de0..6d80cb668957f 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -723,8 +723,8 @@ static u32 rz_dmac_chan_get_residue(struct rz_dmac_chan *channel, u32 crla, crtb, i; /* Get current processing virtual descriptor */ - current_desc = list_first_entry(&channel->ld_active, - struct rz_dmac_desc, node); + current_desc = list_first_entry_or_null(&channel->ld_active, + struct rz_dmac_desc, node); if (!current_desc) return 0; @@ -981,25 +981,6 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, channel->index = index; channel->mid_rid = -EINVAL; - /* Request the channel interrupt. */ - scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); - irq = platform_get_irq_byname(pdev, pdev_irqname); - if (irq < 0) - return irq; - - irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", - dev_name(dmac->dev), index); - if (!irqname) - return -ENOMEM; - - ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler, - rz_dmac_irq_handler_thread, 0, - irqname, channel); - if (ret) { - dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); - return ret; - } - /* Set io base address for each channel */ if (index < 8) { channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + @@ -1012,9 +993,9 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, } /* Allocate descriptors */ - lmdesc = dma_alloc_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - &channel->lmdesc.base_dma, GFP_KERNEL); + lmdesc = dmam_alloc_coherent(&pdev->dev, + sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, + &channel->lmdesc.base_dma, GFP_KERNEL); if (!lmdesc) { dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); return -ENOMEM; @@ -1030,7 +1011,24 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, INIT_LIST_HEAD(&channel->ld_free); INIT_LIST_HEAD(&channel->ld_active); - return 0; + /* Request the channel interrupt. */ + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); + irq = platform_get_irq_byname(pdev, pdev_irqname); + if (irq < 0) + return irq; + + irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", + dev_name(dmac->dev), index); + if (!irqname) + return -ENOMEM; + + ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler, + rz_dmac_irq_handler_thread, 0, + irqname, channel); + if (ret) + dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); + + return ret; } static void rz_dmac_put_device(void *_dev) @@ -1099,7 +1097,6 @@ static int rz_dmac_probe(struct platform_device *pdev) const char *irqname = "error"; struct dma_device *engine; struct rz_dmac *dmac; - int channel_num; int ret; int irq; u8 i; @@ -1132,18 +1129,6 @@ static int rz_dmac_probe(struct platform_device *pdev) return PTR_ERR(dmac->ext_base); } - /* Register interrupt handler for error */ - irq = platform_get_irq_byname_optional(pdev, irqname); - if (irq > 0) { - ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, - irqname, NULL); - if (ret) { - dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", - irq, ret); - return ret; - } - } - /* Initialize the channels. */ INIT_LIST_HEAD(&dmac->engine.channels); @@ -1169,6 +1154,18 @@ static int rz_dmac_probe(struct platform_device *pdev) goto err; } + /* Register interrupt handler for error */ + irq = platform_get_irq_byname_optional(pdev, irqname); + if (irq > 0) { + ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, + irqname, NULL); + if (ret) { + dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", + irq, ret); + goto err; + } + } + /* Register the DMAC as a DMA provider for DT. */ ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, NULL); @@ -1210,16 +1207,6 @@ static int rz_dmac_probe(struct platform_device *pdev) dma_register_err: of_dma_controller_free(pdev->dev.of_node); err: - channel_num = i ? i - 1 : 0; - for (i = 0; i < channel_num; i++) { - struct rz_dmac_chan *channel = &dmac->channels[i]; - - dma_free_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - channel->lmdesc.base, - channel->lmdesc.base_dma); - } - reset_control_assert(dmac->rstc); err_pm_runtime_put: pm_runtime_put(&pdev->dev); @@ -1232,18 +1219,9 @@ static int rz_dmac_probe(struct platform_device *pdev) static void rz_dmac_remove(struct platform_device *pdev) { struct rz_dmac *dmac = platform_get_drvdata(pdev); - unsigned int i; dma_async_device_unregister(&dmac->engine); of_dma_controller_free(pdev->dev.of_node); - for (i = 0; i < dmac->n_channels; i++) { - struct rz_dmac_chan *channel = &dmac->channels[i]; - - dma_free_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - channel->lmdesc.base, - channel->lmdesc.base_dma); - } reset_control_assert(dmac->rstc); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/drivers/dma/tegra186-gpc-dma.c b/drivers/dma/tegra186-gpc-dma.c index 5948fbf32c21e..0aa3a02b2277f 100644 --- a/drivers/dma/tegra186-gpc-dma.c +++ b/drivers/dma/tegra186-gpc-dma.c @@ -825,6 +825,13 @@ static unsigned int get_burst_size(struct tegra_dma_channel *tdc, * len to calculate the optimum burst size */ burst_byte = burst_size ? burst_size * slave_bw : len; + + /* + * Find the largest burst size that evenly divides the transfer length. + * The hardware requires the transfer length to be a multiple of the + * burst size - partial bursts are not supported. + */ + burst_byte = min(burst_byte, 1U << __ffs(len)); burst_mmio_width = burst_byte / 4; if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN) diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index cbb635db43210..8a8f2f4351235 100644 --- a/drivers/dpll/dpll_core.c +++ b/drivers/dpll/dpll_core.c @@ -913,11 +913,13 @@ __dpll_pin_unregister(struct dpll_device *dpll, struct dpll_pin *pin, const struct dpll_pin_ops *ops, void *priv, void *cookie) { ASSERT_DPLL_PIN_REGISTERED(pin); - dpll_pin_ref_sync_pair_del(pin->id); + dpll_pin_delete_ntf(pin); dpll_xa_ref_pin_del(&dpll->pin_refs, pin, ops, priv, cookie); dpll_xa_ref_dpll_del(&pin->dpll_refs, dpll, ops, priv, cookie); - if (xa_empty(&pin->dpll_refs)) + if (xa_empty(&pin->dpll_refs)) { + dpll_pin_ref_sync_pair_del(pin->id); xa_clear_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED); + } } /** @@ -939,7 +941,6 @@ void dpll_pin_unregister(struct dpll_device *dpll, struct dpll_pin *pin, return; mutex_lock(&dpll_lock); - dpll_pin_delete_ntf(pin); __dpll_pin_unregister(dpll, pin, ops, priv, NULL); mutex_unlock(&dpll_lock); } @@ -994,9 +995,9 @@ int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin, dpll_unregister: xa_for_each(&parent->dpll_refs, i, ref) if (i < stop) { + dpll_pin_delete_ntf(pin); __dpll_pin_unregister(ref->dpll, pin, ops, priv, parent); - dpll_pin_delete_ntf(pin); } dpll_xa_ref_pin_del(&pin->parent_refs, parent, ops, priv, pin); unlock: @@ -1018,14 +1019,19 @@ EXPORT_SYMBOL_GPL(dpll_pin_on_pin_register); void dpll_pin_on_pin_unregister(struct dpll_pin *parent, struct dpll_pin *pin, const struct dpll_pin_ops *ops, void *priv) { + struct dpll_pin_registration *reg; struct dpll_pin_ref *ref; unsigned long i; mutex_lock(&dpll_lock); - dpll_pin_delete_ntf(pin); - dpll_xa_ref_pin_del(&pin->parent_refs, parent, ops, priv, pin); - xa_for_each(&pin->dpll_refs, i, ref) + xa_for_each(&pin->dpll_refs, i, ref) { + reg = dpll_pin_registration_find(ref, ops, priv, parent); + if (!reg) + continue; + dpll_pin_delete_ntf(pin); __dpll_pin_unregister(ref->dpll, pin, ops, priv, parent); + } + dpll_xa_ref_pin_del(&pin->parent_refs, parent, ops, priv, pin); mutex_unlock(&dpll_lock); } EXPORT_SYMBOL_GPL(dpll_pin_on_pin_unregister); diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 63df35444214b..de6c52dbd9d25 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -79,6 +79,7 @@ static struct res_config *res_cfg; static int retry_rd_err_log; static int decoding_via_mca; static bool mem_cfg_2lm; +static bool no_adxl; static struct reg_rrl icx_reg_rrl_ddr = { .set_num = 2, @@ -1222,8 +1223,14 @@ static int __init i10nm_init(void) } rc = skx_adxl_get(); - if (rc) - goto fail; + if (rc) { + /* Decoding errors via MCA banks for 2LM isn't supported yet */ + if (rc != -ENODEV || mem_cfg_2lm) + goto fail; + i10nm_printk(KERN_INFO, "ADXL not found, falling back to MCA-based decoding.\n"); + no_adxl = true; + decoding_via_mca = true; + } opstate_init(); mce_register_decode_chain(&i10nm_mce_dec); @@ -1257,7 +1264,8 @@ static void __exit i10nm_exit(void) skx_teardown_debug(); mce_unregister_decode_chain(&i10nm_mce_dec); - skx_adxl_put(); + if (!no_adxl) + skx_adxl_put(); skx_remove(); } diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index fcb8ab44cba55..f849e3299593f 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -122,6 +122,20 @@ #define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) #define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) +struct igen6_imc { + int mc; + struct mem_ctl_info *mci; + struct pci_dev *pdev; + struct device dev; + void __iomem *window; + u64 size; + u64 ch_s_size; + int ch_l_map; + u64 dimm_s_size[NUM_CHANNELS]; + u64 dimm_l_size[NUM_CHANNELS]; + int dimm_l_map[NUM_CHANNELS]; +}; + static struct res_config { bool machine_check; /* The number of present memory controllers. */ @@ -134,12 +148,29 @@ static struct res_config { u64 reg_touud_mask; /* IBECC error log */ u64 reg_eccerrlog_addr_mask; + /* MEMSS_PMA_CR registers. */ + u32 reg_mem_config_offset; + u32 reg_mem_config_ddr_type_mask; + /* Memory controller registers. */ + u32 reg_mad_inter_size_mask[NUM_CHANNELS]; + u64 reg_mad_inter_size_granularity; + u32 reg_mad_intra_rank_mask[NUM_DIMMS]; + u32 reg_mad_intra_width_mask[NUM_DIMMS]; + u32 reg_mad_intra_density_mask[NUM_DIMMS]; u32 imc_base; u32 cmf_base; u32 cmf_size; u32 ms_hash_offset; u32 ibecc_base; u32 ibecc_error_log_offset; + /* Get memory type. */ + enum mem_type (*get_mem_type)(struct igen6_imc *imc); + /* Get DRAM chip type. */ + enum dev_type (*get_dev_type)(struct igen6_imc *imc, int chan, int dimm_l); + /* Set imc->ch_{s_size,l_map}. */ + void (*set_chan_params)(struct igen6_imc *imc); + /* Set imc->dimm_{l_size,s_size,l_map}[chan]. */ + void (*set_dimm_params)(struct igen6_imc *imc, int chan); bool (*ibecc_available)(struct pci_dev *pdev); /* Extract error address logged in IBECC */ u64 (*err_addr)(u64 ecclog); @@ -149,22 +180,9 @@ static struct res_config { u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc); } *res_cfg; -struct igen6_imc { - int mc; - struct mem_ctl_info *mci; - struct pci_dev *pdev; - struct device dev; - void __iomem *window; - u64 size; - u64 ch_s_size; - int ch_l_map; - u64 dimm_s_size[NUM_CHANNELS]; - u64 dimm_l_size[NUM_CHANNELS]; - int dimm_l_map[NUM_CHANNELS]; -}; - static struct igen6_pvt { struct igen6_imc imc[NUM_IMC]; + void __iomem *memss_pma_cr; u64 ms_hash; u64 ms_s_size; int ms_l_map; @@ -500,6 +518,119 @@ static u64 rpl_p_err_addr(u64 ecclog) return field_get(res_cfg->reg_eccerrlog_addr_mask, ecclog); } +static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc) +{ + u32 mtype, val; + + val = readl(igen6_pvt->memss_pma_cr + res_cfg->reg_mem_config_offset); + mtype = field_get(res_cfg->reg_mem_config_ddr_type_mask, val); + + edac_dbg(2, "mtype %u (reg 0x%x)\n", mtype, val); + + switch (mtype) { + case 1: + return MEM_DDR5; + case 2: + return MEM_LPDDR5; + case 3: + return MEM_LPDDR4; + default: + return MEM_UNKNOWN; + } +} + +static enum dev_type ptl_h_get_dev_type(struct igen6_imc *imc, int chan, int dimm) +{ + u32 width, val; + + val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); + width = field_get(res_cfg->reg_mad_intra_width_mask[dimm], val); + + switch (width) { + case 1: + return DEV_X8; + default: + return DEV_X16; + } +} + +static u64 ptl_h_get_chan_size(struct igen6_imc *imc, int chan) +{ + u32 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); + + return field_get(res_cfg->reg_mad_inter_size_mask[chan], val) * + res_cfg->reg_mad_inter_size_granularity; +} + +static u64 ptl_h_get_dimm_size(struct igen6_imc *imc, int chan, int dimm) +{ + u32 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); + u32 ranks = 1 << field_get(res_cfg->reg_mad_intra_rank_mask[dimm], val); + /* DRAM device density in Gb */ + u64 density = field_get(res_cfg->reg_mad_intra_density_mask[dimm], val) * 4; + + enum mem_type mtype = ptl_h_get_mem_type(imc); + enum dev_type dtype = ptl_h_get_dev_type(imc, chan, dimm); + u64 sub_ch_width, dev_num; + + switch (mtype) { + case MEM_DDR5: + sub_ch_width = 32; + break; + case MEM_LPDDR5: + case MEM_LPDDR4: + sub_ch_width = 16; + break; + default: + sub_ch_width = 0; + } + + switch (dtype) { + case DEV_X8: + dev_num = sub_ch_width / 8; + break; + case DEV_X16: + dev_num = sub_ch_width / 16; + break; + default: + dev_num = 0; + } + + edac_dbg(2, "ranks %d, density %lluGb, sub_ch_width %llu, dev_num %llu (reg 0x%x)\n", ranks, density, sub_ch_width, dev_num, val); + + return ((dev_num * density / 8) * ranks) << 30; +} + +static void ptl_h_set_chan_params(struct igen6_imc *imc) +{ + u64 ch0_size = ptl_h_get_chan_size(imc, 0); + u64 ch1_size = ptl_h_get_chan_size(imc, 1); + + if (ch0_size <= ch1_size) { + imc->ch_s_size = ch0_size; + imc->ch_l_map = 1; + } else { + imc->ch_s_size = ch1_size; + imc->ch_l_map = 0; + } +} + +static void ptl_h_set_dimm_params(struct igen6_imc *imc, int chan) +{ + u64 dimm0_size = ptl_h_get_dimm_size(imc, chan, 0); + u64 dimm1_size = ptl_h_get_dimm_size(imc, chan, 1); + + if (dimm0_size <= dimm1_size) { + imc->dimm_s_size[chan] = dimm0_size; + imc->dimm_l_size[chan] = dimm1_size; + imc->dimm_l_map[chan] = 1; + } else { + imc->dimm_s_size[chan] = dimm1_size; + imc->dimm_l_size[chan] = dimm0_size; + imc->dimm_l_map[chan] = 0; + } +} + static struct res_config ehl_cfg = { .num_imc = 1, .reg_mchbar_mask = GENMASK_ULL(38, 16), @@ -622,6 +753,36 @@ static struct res_config mtl_p_cfg = { .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, }; +static struct res_config ptl_h_cfg = { + .machine_check = true, + .num_imc = 2, + .reg_mchbar_mask = GENMASK_ULL(41, 17), + .reg_tom_mask = GENMASK_ULL(41, 20), + .reg_touud_mask = GENMASK_ULL(41, 20), + .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), + .reg_mem_config_offset = 0x13d04, + .reg_mem_config_ddr_type_mask = GENMASK(8, 6), + .reg_mad_inter_size_mask[0] = GENMASK(15, 8), + .reg_mad_inter_size_mask[1] = GENMASK(23, 16), + .reg_mad_inter_size_granularity = BIT_ULL(29), + .reg_mad_intra_rank_mask[0] = BIT(7), + .reg_mad_intra_rank_mask[1] = BIT(15), + .reg_mad_intra_width_mask[0] = BIT(6), + .reg_mad_intra_width_mask[1] = BIT(14), + .reg_mad_intra_density_mask[0] = GENMASK(3, 0), + .reg_mad_intra_density_mask[1] = GENMASK(11, 8), + .imc_base = 0xd800, + .ibecc_base = 0xd400, + .ibecc_error_log_offset = 0x170, + .get_mem_type = ptl_h_get_mem_type, + .get_dev_type = ptl_h_get_dev_type, + .set_chan_params = ptl_h_set_chan_params, + .set_dimm_params = ptl_h_set_dimm_params, + .ibecc_available = mtl_p_ibecc_available, + .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, + .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, +}; + static struct res_config wcl_cfg = { .machine_check = true, .num_imc = 1, @@ -689,46 +850,34 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&ptl_h_cfg }, { PCI_VDEVICE(INTEL, DID_WCL_SKU1), (kernel_ulong_t)&wcl_cfg }, { }, }; MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); -static enum dev_type get_width(int dimm_l, u32 mad_dimm) +static enum mem_type get_mem_type(struct igen6_imc *imc) { - u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) : - MAD_DIMM_CH_DSW(mad_dimm); + u32 val; - switch (w) { - case 0: - return DEV_X8; - case 1: - return DEV_X16; - case 2: - return DEV_X32; - default: - return DEV_UNKNOWN; - } -} + if (res_cfg->get_mem_type) + return res_cfg->get_mem_type(imc); -static enum mem_type get_memory_type(u32 mad_inter) -{ - u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter); + val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); - switch (t) { + switch (MAD_INTER_CHANNEL_DDR_TYPE(val)) { case 0: return MEM_DDR4; case 1: @@ -744,6 +893,73 @@ static enum mem_type get_memory_type(u32 mad_inter) } } +static bool large_dimm(struct igen6_imc *imc, int chan, int dimm) +{ + return dimm == imc->dimm_l_map[chan]; +} + +static enum dev_type get_dev_type(struct igen6_imc *imc, int chan, int dimm) +{ + u32 width, val; + + if (res_cfg->get_dev_type) + return res_cfg->get_dev_type(imc, chan, dimm); + + val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); + width = large_dimm(imc, chan, dimm) ? MAD_DIMM_CH_DLW(val) : + MAD_DIMM_CH_DSW(val); + + switch (width) { + case 0: + return DEV_X8; + case 1: + return DEV_X16; + case 2: + return DEV_X32; + default: + return DEV_UNKNOWN; + } +} + +static u64 get_dimm_size(struct igen6_imc *imc, int chan, int dimm) +{ + if (large_dimm(imc, chan, dimm)) + return imc->dimm_l_size[chan]; + + return imc->dimm_s_size[chan]; +} + +static void set_chan_params(struct igen6_imc *imc) +{ + u32 val; + + if (res_cfg->set_chan_params) { + res_cfg->set_chan_params(imc); + return; + } + + val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); + imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(val); + imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(val); +} + +static void set_dimm_params(struct igen6_imc *imc, int chan) +{ + u32 val; + + if (res_cfg->set_dimm_params) { + res_cfg->set_dimm_params(imc, chan); + return; + } + + val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); + imc->dimm_l_map[chan] = MAD_INTRA_CH_DIMM_L_MAP(val); + + val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); + imc->dimm_l_size[chan] = MAD_DIMM_CH_DIMM_L_SIZE(val); + imc->dimm_s_size[chan] = MAD_DIMM_CH_DIMM_S_SIZE(val); +} + static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) { u64 hash_addr = addr & mask, hash = 0; @@ -1084,7 +1300,6 @@ static bool igen6_check_ecc(struct igen6_imc *imc) static int igen6_get_dimm_config(struct mem_ctl_info *mci) { struct igen6_imc *imc = mci->pvt_info; - u32 mad_inter, mad_intra, mad_dimm; int i, j, ndimms, mc = imc->mc; struct dimm_info *dimm; enum mem_type mtype; @@ -1094,33 +1309,20 @@ static int igen6_get_dimm_config(struct mem_ctl_info *mci) edac_dbg(2, "\n"); - mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); - mtype = get_memory_type(mad_inter); + mtype = get_mem_type(imc); ecc = igen6_check_ecc(imc); - imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter); - imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter); + set_chan_params(imc); for (i = 0; i < NUM_CHANNELS; i++) { - mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4); - mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4); - - imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm); - imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm); - imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra); + set_dimm_params(imc, i); imc->size += imc->dimm_s_size[i]; imc->size += imc->dimm_l_size[i]; ndimms = 0; for (j = 0; j < NUM_DIMMS; j++) { dimm = edac_get_dimm(mci, i, j, 0); - - if (j ^ imc->dimm_l_map[i]) { - dtype = get_width(0, mad_dimm); - dsize = imc->dimm_s_size[i]; - } else { - dtype = get_width(1, mad_dimm); - dsize = imc->dimm_l_size[i]; - } + dtype = get_dev_type(imc, i, j); + dsize = get_dimm_size(imc, i, j); if (!dsize) continue; @@ -1223,6 +1425,39 @@ static void igen6_debug_setup(void) {} static void igen6_debug_teardown(void) {} #endif +static struct igen6_pvt *igen6_pvt_setup(struct pci_dev *pdev) +{ + void __iomem *memss_pma_cr; + struct igen6_pvt *pvt; + u64 mchbar; + int rc; + + pvt = kzalloc_obj(*igen6_pvt); + if (!pvt) + return NULL; + + rc = get_mchbar(pdev, &mchbar); + if (rc) { + kfree(pvt); + return NULL; + } + + memss_pma_cr = ioremap(mchbar, MCHBAR_SIZE * 2); + if (!memss_pma_cr) { + kfree(pvt); + return NULL; + } + pvt->memss_pma_cr = memss_pma_cr; + + return pvt; +} + +static void igen6_pvt_release(struct igen6_pvt *pvt) +{ + iounmap(pvt->memss_pma_cr); + kfree(pvt); +} + static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) { union { @@ -1296,6 +1531,11 @@ static bool igen6_imc_absent(void __iomem *window) return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0; } +static void imc_release(struct device *dev) +{ + /* Nothing to do, the 'imc' owns the 'dev' and will also release it. */ +} + static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev) { struct edac_mc_layer layers[2]; @@ -1334,6 +1574,7 @@ static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev mci->pvt_info = &igen6_pvt->imc[mc]; imc = mci->pvt_info; + imc->dev.release = imc_release; device_initialize(&imc->dev); /* * EDAC core uses mci->pdev(pointer of structure device) as @@ -1549,12 +1790,12 @@ static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) edac_dbg(2, "\n"); - igen6_pvt = kzalloc_obj(*igen6_pvt); + res_cfg = (struct res_config *)ent->driver_data; + + igen6_pvt = igen6_pvt_setup(pdev); if (!igen6_pvt) return -ENOMEM; - res_cfg = (struct res_config *)ent->driver_data; - rc = igen6_pci_setup(pdev, &mchbar); if (rc) goto fail; @@ -1603,7 +1844,7 @@ static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) fail2: igen6_unregister_mcis(); fail: - kfree(igen6_pvt); + igen6_pvt_release(igen6_pvt); return rc; } @@ -1618,7 +1859,7 @@ static void igen6_remove(struct pci_dev *pdev) flush_work(&ecclog_work); gen_pool_destroy(ecclog_pool); igen6_unregister_mcis(); - kfree(igen6_pvt); + igen6_pvt_release(igen6_pvt); } static struct pci_driver igen6_driver = { diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index a9557c8344bc1..f15de0ea96c87 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -466,6 +466,9 @@ int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, rows = numrow(mtr); cols = imc->hbm_mc ? 6 : numcol(mtr); + if (ranks < 0 || rows < 0 || cols < 0) + return 0; + if (imc->hbm_mc) { banks = 32; mtype = MEM_HBM2; diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index b9f17fda72432..cab32cfdac423 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -324,11 +324,9 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, #define PART_INFO_EXEC_CXT_MASK GENMASK(31, 16) #define PART_INFO_PROPS_MASK GENMASK(63, 32) #define FFA_PART_INFO_GET_REGS_FIRST_REG 3 -#define FFA_PART_INFO_GET_REGS_REGS_PER_DESC 3 -#define FFA_PART_INFO_GET_REGS_MAX_DESC \ - (((sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) - \ - FFA_PART_INFO_GET_REGS_FIRST_REG) / \ - FFA_PART_INFO_GET_REGS_REGS_PER_DESC) +#define FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC 3 +#define FFA_PART_INFO_GET_REGS_NUM_REGS \ + (sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) #define PART_INFO_ID(x) ((u16)(FIELD_GET(PART_INFO_ID_MASK, (x)))) #define PART_INFO_EXEC_CXT(x) ((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x)))) #define PART_INFO_PROPERTIES(x) ((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x)))) @@ -342,7 +340,7 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, do { __le64 *regs; - int idx, nr_desc, buf_idx; + int idx, nr_desc, buf_idx, regs_per_desc, max_desc; invoke_ffa_fn((ffa_value_t){ .a0 = FFA_PARTITION_INFO_GET_REGS, @@ -365,8 +363,18 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, if (cur_idx < start_idx || cur_idx >= count) return -EINVAL; + buf_sz = PARTITION_INFO_SZ(partition_info.a2); + if (buf_sz % sizeof(*regs)) + return -EINVAL; + + regs_per_desc = buf_sz / sizeof(*regs); + if (regs_per_desc < FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC) + return -EINVAL; + nr_desc = cur_idx - start_idx + 1; - if (nr_desc > FFA_PART_INFO_GET_REGS_MAX_DESC) + max_desc = (FFA_PART_INFO_GET_REGS_NUM_REGS - + FFA_PART_INFO_GET_REGS_FIRST_REG) / regs_per_desc; + if (nr_desc > max_desc) return -EINVAL; buf_idx = buf - buffer; @@ -374,9 +382,6 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, return -EINVAL; tag = UUID_INFO_TAG(partition_info.a2); - buf_sz = PARTITION_INFO_SZ(partition_info.a2); - if (buf_sz > sizeof(*buffer)) - buf_sz = sizeof(*buffer); regs = (void *)&partition_info.a3; for (idx = 0; idx < nr_desc; idx++, buf++) { @@ -395,7 +400,7 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, buf->exec_ctxt = PART_INFO_EXEC_CXT(val); buf->properties = PART_INFO_PROPERTIES(val); uuid_copy(&buf->uuid, &uuid_regs.uuid); - regs += 3; + regs += regs_per_desc; } start_idx = cur_idx + 1; diff --git a/drivers/firmware/arm_scmi/power.c b/drivers/firmware/arm_scmi/power.c index bb5062ab8280e..28ef63a4ecc2e 100644 --- a/drivers/firmware/arm_scmi/power.c +++ b/drivers/firmware/arm_scmi/power.c @@ -204,8 +204,12 @@ scmi_power_name_get(const struct scmi_protocol_handle *ph, u32 domain) { struct scmi_power_info *pi = ph->get_priv(ph); - struct power_dom_info *dom = pi->dom_info + domain; + struct power_dom_info *dom; + + if (domain >= pi->num_domains) + return "unknown"; + dom = pi->dom_info + domain; return dom->name; } diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c index 882d55f987d22..836c294a9f42a 100644 --- a/drivers/firmware/arm_scmi/sensors.c +++ b/drivers/firmware/arm_scmi/sensors.c @@ -793,7 +793,7 @@ static int scmi_sensor_config_get(const struct scmi_protocol_handle *ph, if (!ret) { struct scmi_sensor_info *s = si->sensors + sensor_id; - *sensor_config = get_unaligned_le64(t->rx.buf); + *sensor_config = get_unaligned_le32(t->rx.buf); s->sensor_config = *sensor_config; } diff --git a/drivers/firmware/samsung/exynos-acpm.c b/drivers/firmware/samsung/exynos-acpm.c index 19db3674a28f3..65949e7fd430d 100644 --- a/drivers/firmware/samsung/exynos-acpm.c +++ b/drivers/firmware/samsung/exynos-acpm.c @@ -819,6 +819,29 @@ struct acpm_handle *devm_acpm_get_by_node(struct device *dev, } EXPORT_SYMBOL_GPL(devm_acpm_get_by_node); +/** + * devm_acpm_get_by_phandle - Resource managed lookup of the standardized + * "samsung,acpm-ipc" handle. + * @dev: consumer device + * + * Return: pointer to handle on success, ERR_PTR(-errno) otherwise. + */ +struct acpm_handle *devm_acpm_get_by_phandle(struct device *dev) +{ + struct acpm_handle *handle; + struct device_node *np; + + np = of_parse_phandle(dev->of_node, "samsung,acpm-ipc", 0); + if (!np) + return ERR_PTR(-ENODEV); + + handle = devm_acpm_get_by_node(dev, np); + of_node_put(np); + + return handle; +} +EXPORT_SYMBOL_GPL(devm_acpm_get_by_phandle); + static const struct acpm_match_data acpm_gs101 = { .initdata_base = ACPM_GS101_INITDATA_BASE, .acpm_clk_dev_name = "gs101-acpm-clk", diff --git a/drivers/firmware/smccc/soc_id.c b/drivers/firmware/smccc/soc_id.c index 2f7475e66b3ca..a909d5e6dee50 100644 --- a/drivers/firmware/smccc/soc_id.c +++ b/drivers/firmware/smccc/soc_id.c @@ -60,7 +60,7 @@ static char __init *smccc_soc_name_init(void) * to the ARM_SMCCC_ARCH_SOC_ID function. Fetch it if * available. */ - args.a0 = ARM_SMCCC_ARCH_SOC_ID; + args.a0 = ARM_SMCCC_ARCH_SOC_ID64; args.a1 = 2; /* SOC_ID name */ arm_smccc_1_2_invoke(&args, &res); diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index fbe8510f49271..af838b2dc3279 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -1450,6 +1450,34 @@ int zynqmp_pm_get_node_status(const u32 node, u32 *const status, } EXPORT_SYMBOL_GPL(zynqmp_pm_get_node_status); +/** + * zynqmp_pm_get_rpu_node_status - PM call to request a RPU node's current power state + * @node: ID of the RPU component or sub-system in question + * @status: Current operating state of the requested RPU node. + * @requirements: Current requirements asserted on the RPU node. + * @usage: Usage information, used for RPU slave nodes only: + * PM_USAGE_NO_MASTER - No master is currently using + * the node + * PM_USAGE_CURRENT_MASTER - Only requesting master is + * currently using the node + * PM_USAGE_OTHER_MASTER - Only other masters are + * currently using the node + * PM_USAGE_BOTH_MASTERS - Both the current and at least + * one other master is currently + * using the node + * + * Return: Returns status, either success or error+reason + */ +int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status, + u32 *const requirements, u32 *const usage) +{ + if (zynqmp_pm_feature(PM_GET_NODE_STATUS) < PM_API_VERSION_2) + return -EOPNOTSUPP; + + return zynqmp_pm_get_node_status(node, status, requirements, usage); +} +EXPORT_SYMBOL_GPL(zynqmp_pm_get_rpu_node_status); + /** * zynqmp_pm_force_pwrdwn - PM call to request for another PU or subsystem to * be powered down forcefully diff --git a/drivers/fpga/dfl-afu-main.c b/drivers/fpga/dfl-afu-main.c index 3bf8e7338dbef..097a97eeea66c 100644 --- a/drivers/fpga/dfl-afu-main.c +++ b/drivers/fpga/dfl-afu-main.c @@ -723,6 +723,9 @@ afu_ioctl_dma_map(struct dfl_feature_dev_data *fdata, void __user *arg) if (map.argsz < minsz || map.flags) return -EINVAL; + if (map.length >> PAGE_SHIFT > (u64)INT_MAX) + return -EINVAL; + ret = afu_dma_map_region(fdata, map.user_addr, map.length, &map.iova); if (ret) return ret; diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index 4087a36a0571a..4c63c7c8579bb 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -1132,6 +1132,8 @@ static int dfh_get_param_size(void __iomem *dfh_base, resource_size_t max) return -EINVAL; size += next * sizeof(u64); + if (size > max) + return -EINVAL; if (FIELD_GET(DFHv1_PARAM_HDR_NEXT_EOP, v)) return size; diff --git a/drivers/fpga/microchip-spi.c b/drivers/fpga/microchip-spi.c index 6134cea86ac85..cc8f6d7bb9781 100644 --- a/drivers/fpga/microchip-spi.c +++ b/drivers/fpga/microchip-spi.c @@ -116,6 +116,9 @@ static int mpf_ops_parse_header(struct fpga_manager *mgr, } header_size = *(buf + MPF_HEADER_SIZE_OFFSET); + if (!header_size) + return -EINVAL; + if (header_size > count) { info->header_size = header_size; return -EAGAIN; diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c index caa091224dc54..9107a5b461d35 100644 --- a/drivers/fpga/of-fpga-region.c +++ b/drivers/fpga/of-fpga-region.c @@ -168,11 +168,10 @@ static int child_regions_with_firmware(struct device_node *overlay) fpga_region_of_match); } - of_node_put(child_region); - if (ret) pr_err("firmware-name not allowed in child FPGA region: %pOF", child_region); + of_node_put(child_region); return ret; } diff --git a/drivers/gpib/common/gpib_os.c b/drivers/gpib/common/gpib_os.c index 5909274ddc122..fcf13432c7dd5 100644 --- a/drivers/gpib/common/gpib_os.c +++ b/drivers/gpib/common/gpib_os.c @@ -613,7 +613,7 @@ long ibioctl(struct file *filep, unsigned int cmd, unsigned long arg) unsigned int minor = iminor(file_inode(filep)); struct gpib_board *board; struct gpib_file_private *file_priv = filep->private_data; - long retval = -ENOTTY; + long retval = -EBADRQC; if (minor >= GPIB_MAX_NUM_BOARDS) { pr_err("gpib: invalid minor number of device file\n"); @@ -806,7 +806,6 @@ long ibioctl(struct file *filep, unsigned int cmd, unsigned long arg) mutex_unlock(&board->big_gpib_mutex); return write_ioctl(file_priv, board, arg); default: - retval = -ENOTTY; goto done; } @@ -1018,7 +1017,6 @@ static int command_ioctl(struct gpib_file_private *file_priv, userbuf += bytes_written; if (retval < 0) { atomic_set(&desc->io_in_progress, 0); - atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); break; diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c index 97780f27ce5b5..270cd7c888120 100644 --- a/drivers/gpio/gpio-davinci.c +++ b/drivers/gpio/gpio-davinci.c @@ -568,8 +568,10 @@ static int davinci_gpio_irq_setup(struct platform_device *pdev) sizeof(struct davinci_gpio_irq_data), GFP_KERNEL); - if (!irqdata) + if (!irqdata) { + irq_domain_remove(chips->irq_domain); return -ENOMEM; + } irqdata->regs = g; irqdata->bank_num = bank; diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c index 15cebc8b5d663..0259c65973323 100644 --- a/drivers/gpio/gpio-dwapb.c +++ b/drivers/gpio/gpio-dwapb.c @@ -118,6 +118,7 @@ struct dwapb_gpio { unsigned int flags; struct reset_control *rst; struct clk_bulk_data clks[DWAPB_NR_CLOCKS]; + bool clocks_on_for_wake; struct dwapb_gpio_port ports[] __counted_by(nr_ports); }; @@ -365,11 +366,24 @@ static int dwapb_irq_set_wake(struct irq_data *d, unsigned int enable) struct dwapb_gpio *gpio = to_dwapb_gpio(gc); struct dwapb_context *ctx = gpio->ports[0].ctx; irq_hw_number_t bit = irqd_to_hwirq(d); + u32 wake_en = ctx->wake_en; if (enable) - ctx->wake_en |= BIT(bit); + wake_en |= BIT(bit); else - ctx->wake_en &= ~BIT(bit); + wake_en &= ~BIT(bit); + +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY + if (d->parent_data && !!ctx->wake_en != !!wake_en) { + int err; + + err = irq_chip_set_wake_parent(d, enable); + if (err) + return err; + } +#endif + + ctx->wake_en = wake_en; return 0; } @@ -749,6 +763,8 @@ static int dwapb_gpio_suspend(struct device *dev) int i; scoped_guard(gpio_generic_lock_irqsave, gen_gc) { + gpio->clocks_on_for_wake = false; + for (i = 0; i < gpio->nr_ports; i++) { unsigned int offset; unsigned int idx = gpio->ports[i].idx; @@ -770,11 +786,38 @@ static int dwapb_gpio_suspend(struct device *dev) ctx->int_pol = dwapb_read(gpio, GPIO_INT_POLARITY); ctx->int_type = dwapb_read(gpio, GPIO_INTTYPE_LEVEL); ctx->int_deb = dwapb_read(gpio, GPIO_PORTA_DEBOUNCE); + } + } + } + + return 0; +} + +static int dwapb_gpio_suspend_noirq(struct device *dev) +{ + struct dwapb_gpio *gpio = dev_get_drvdata(dev); + struct gpio_generic_chip *gen_gc = &gpio->ports[0].chip; + bool wake_enabled = false; + int i; + + scoped_guard(gpio_generic_lock_irqsave, gen_gc) { + for (i = 0; i < gpio->nr_ports; i++) { + unsigned int idx = gpio->ports[i].idx; + struct dwapb_context *ctx = gpio->ports[i].ctx; - /* Mask out interrupts */ + if (idx == 0) { + wake_enabled = ctx->wake_en; dwapb_write(gpio, GPIO_INTMASK, ~ctx->wake_en); + break; } } + + gpio->clocks_on_for_wake = wake_enabled; + } + + if (wake_enabled) { + device_set_wakeup_path(dev); + return 0; } clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks); @@ -782,18 +825,27 @@ static int dwapb_gpio_suspend(struct device *dev) return 0; } -static int dwapb_gpio_resume(struct device *dev) +static int dwapb_gpio_resume_noirq(struct device *dev) { struct dwapb_gpio *gpio = dev_get_drvdata(dev); - struct gpio_chip *gc = &gpio->ports[0].chip.gc; - struct gpio_generic_chip *gen_gc = to_gpio_generic_chip(gc); - int i, err; + int err; + + if (gpio->clocks_on_for_wake) + return 0; err = clk_bulk_prepare_enable(DWAPB_NR_CLOCKS, gpio->clks); - if (err) { + if (err) dev_err(gpio->dev, "Cannot reenable APB/Debounce clocks\n"); - return err; - } + + return err; +} + +static int dwapb_gpio_resume(struct device *dev) +{ + struct dwapb_gpio *gpio = dev_get_drvdata(dev); + struct gpio_chip *gc = &gpio->ports[0].chip.gc; + struct gpio_generic_chip *gen_gc = to_gpio_generic_chip(gc); + int i; guard(gpio_generic_lock_irqsave)(gen_gc); @@ -827,8 +879,11 @@ static int dwapb_gpio_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(dwapb_gpio_pm_ops, - dwapb_gpio_suspend, dwapb_gpio_resume); +static const struct dev_pm_ops dwapb_gpio_pm_ops = { + SYSTEM_SLEEP_PM_OPS(dwapb_gpio_suspend, dwapb_gpio_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(dwapb_gpio_suspend_noirq, + dwapb_gpio_resume_noirq) +}; static struct platform_driver dwapb_gpio_driver = { .driver = { diff --git a/drivers/gpio/gpio-eic-sprd.c b/drivers/gpio/gpio-eic-sprd.c index 50fafeda8d7e2..3b7ebcf12fe7f 100644 --- a/drivers/gpio/gpio-eic-sprd.c +++ b/drivers/gpio/gpio-eic-sprd.c @@ -95,7 +95,7 @@ struct sprd_eic { struct notifier_block irq_nb; void __iomem *base[SPRD_EIC_MAX_BANK]; enum sprd_eic_type type; - spinlock_t lock; + raw_spinlock_t lock; int irq; }; @@ -149,7 +149,7 @@ static void sprd_eic_update(struct gpio_chip *chip, unsigned int offset, unsigned long flags; u32 tmp; - spin_lock_irqsave(&sprd_eic->lock, flags); + raw_spin_lock_irqsave(&sprd_eic->lock, flags); tmp = readl_relaxed(base + reg); if (val) @@ -158,7 +158,7 @@ static void sprd_eic_update(struct gpio_chip *chip, unsigned int offset, tmp &= ~BIT(SPRD_EIC_BIT(offset)); writel_relaxed(tmp, base + reg); - spin_unlock_irqrestore(&sprd_eic->lock, flags); + raw_spin_unlock_irqrestore(&sprd_eic->lock, flags); } static int sprd_eic_read(struct gpio_chip *chip, unsigned int offset, u16 reg) @@ -628,7 +628,7 @@ static int sprd_eic_probe(struct platform_device *pdev) if (!sprd_eic) return -ENOMEM; - spin_lock_init(&sprd_eic->lock); + raw_spin_lock_init(&sprd_eic->lock); sprd_eic->type = pdata->type; sprd_eic->irq = platform_get_irq(pdev, 0); diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c index 4d5b927ad70fd..fb007b9787290 100644 --- a/drivers/gpio/gpio-f7188x.c +++ b/drivers/gpio/gpio-f7188x.c @@ -48,7 +48,8 @@ /* * Nuvoton devices. */ -#define SIO_NCT6126D_ID 0xD283 /* NCT6126D chipset ID */ +#define SIO_NCT6126D_VER_A_ID 0xD283 /* NCT6126D version A chipset ID */ +#define SIO_NCT6126D_VER_B_ID 0xD284 /* NCT6126D version B chipset ID */ #define SIO_LD_GPIO_NUVOTON 0x07 /* GPIO logical device */ @@ -564,7 +565,8 @@ static int __init f7188x_find(int addr, struct f7188x_sio *sio) case SIO_F81865_ID: sio->type = f81865; break; - case SIO_NCT6126D_ID: + case SIO_NCT6126D_VER_A_ID: + case SIO_NCT6126D_VER_B_ID: sio->device = SIO_LD_GPIO_NUVOTON; sio->type = nct6126d; break; diff --git a/drivers/gpio/gpio-htc-egpio.c b/drivers/gpio/gpio-htc-egpio.c index d15423c718d03..25a4d4494f3c3 100644 --- a/drivers/gpio/gpio-htc-egpio.c +++ b/drivers/gpio/gpio-htc-egpio.c @@ -268,6 +268,7 @@ static int __init egpio_probe(struct platform_device *pdev) struct gpio_chip *chip; unsigned int irq, irq_end; int i; + int ret; /* Initialize ei data structure. */ ei = devm_kzalloc(&pdev->dev, struct_size(ei, chip, pdata->num_chips), GFP_KERNEL); @@ -326,7 +327,10 @@ static int __init egpio_probe(struct platform_device *pdev) chip->base = pdata->chip[i].gpio_base; chip->ngpio = pdata->chip[i].num_gpios; - gpiochip_add_data(chip, &ei->chip[i]); + ret = devm_gpiochip_add_data(&pdev->dev, chip, &ei->chip[i]); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to register gpiochip %d\n", i); } /* Set initial pin values */ diff --git a/drivers/gpio/gpio-mlxbf3.c b/drivers/gpio/gpio-mlxbf3.c index 4770578269bae..566326644a2c1 100644 --- a/drivers/gpio/gpio-mlxbf3.c +++ b/drivers/gpio/gpio-mlxbf3.c @@ -255,7 +255,8 @@ static int mlxbf3_gpio_probe(struct platform_device *pdev) ret = devm_gpiochip_add_data(dev, gc, gs); if (ret) - dev_err_probe(dev, ret, "Failed adding memory mapped gpiochip\n"); + return dev_err_probe(dev, ret, + "Failed adding memory mapped gpiochip\n"); return 0; } diff --git a/drivers/gpio/gpio-mt7621.c b/drivers/gpio/gpio-mt7621.c index 91230be515879..1b0b5247d3c9a 100644 --- a/drivers/gpio/gpio-mt7621.c +++ b/drivers/gpio/gpio-mt7621.c @@ -29,8 +29,8 @@ #define GPIO_REG_EDGE 0xA0 struct mtk_gc { - struct irq_chip irq_chip; struct gpio_generic_chip chip; + struct mtk *parent_priv; int bank; u32 rising; u32 falling; @@ -41,20 +41,32 @@ struct mtk_gc { /** * struct mtk - state container for * data of the platform driver. It is 3 - * separate gpio-chip each one with its - * own irq_chip. - * @dev: device instance + * separate gpio-chip having an IRQ + * linear domain shared for all of them + * @pdev: platform device instance * @base: memory base address + * @irq_domain: IRQ linear domain shared across the three gpio chips * @gpio_irq: irq number from the device tree + * @num_gpios: total number of gpio pins on the three gpio chips * @gc_map: array of the gpio chips */ struct mtk { - struct device *dev; + struct platform_device *pdev; void __iomem *base; + struct irq_domain *irq_domain; int gpio_irq; + int num_gpios; struct mtk_gc gc_map[MTK_BANK_CNT]; }; +static inline struct mtk * +mt7621_gpio_gc_to_priv(struct gpio_chip *gc) +{ + struct mtk_gc *bank = gpiochip_get_data(gc); + + return bank->parent_priv; +} + static inline struct mtk_gc * to_mediatek_gpio(struct gpio_chip *chip) { @@ -67,7 +79,7 @@ static inline void mtk_gpio_w32(struct mtk_gc *rg, u32 offset, u32 val) { struct gpio_chip *gc = &rg->chip.gc; - struct mtk *mtk = gpiochip_get_data(gc); + struct mtk *mtk = mt7621_gpio_gc_to_priv(gc); offset = (rg->bank * GPIO_BANK_STRIDE) + offset; gpio_generic_write_reg(&rg->chip, mtk->base + offset, val); @@ -77,41 +89,62 @@ static inline u32 mtk_gpio_r32(struct mtk_gc *rg, u32 offset) { struct gpio_chip *gc = &rg->chip.gc; - struct mtk *mtk = gpiochip_get_data(gc); + struct mtk *mtk = mt7621_gpio_gc_to_priv(gc); offset = (rg->bank * GPIO_BANK_STRIDE) + offset; return gpio_generic_read_reg(&rg->chip, mtk->base + offset); } -static irqreturn_t -mediatek_gpio_irq_handler(int irq, void *data) +static void +mt7621_gpio_irq_bank_handler(struct mtk_gc *bank) { - struct gpio_chip *gc = data; - struct mtk_gc *rg = to_mediatek_gpio(gc); - irqreturn_t ret = IRQ_NONE; + struct mtk *priv = bank->parent_priv; + struct irq_domain *domain = priv->irq_domain; + int hwbase = bank->chip.gc.offset; unsigned long pending; - int bit; + unsigned int offset; + + pending = mtk_gpio_r32(bank, GPIO_REG_STAT); + if (!pending) + return; + + mtk_gpio_w32(bank, GPIO_REG_STAT, pending); + + for_each_set_bit(offset, &pending, MTK_BANK_WIDTH) + generic_handle_domain_irq(domain, hwbase + offset); +} + +static void +mt7621_gpio_irq_handler(struct irq_desc *desc) +{ + struct mtk *priv = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + int i; - pending = mtk_gpio_r32(rg, GPIO_REG_STAT); + chained_irq_enter(chip, desc); + for (i = 0; i < MTK_BANK_CNT; i++) { + struct mtk_gc *bank = &priv->gc_map[i]; - for_each_set_bit(bit, &pending, MTK_BANK_WIDTH) { - generic_handle_domain_irq(gc->irq.domain, bit); - mtk_gpio_w32(rg, GPIO_REG_STAT, BIT(bit)); - ret |= IRQ_HANDLED; + mt7621_gpio_irq_bank_handler(bank); } + chained_irq_exit(chip, desc); +} - return ret; +static int +mt7621_gpio_hwirq_to_offset(irq_hw_number_t hwirq, struct mtk_gc *bank) +{ + return hwirq - bank->chip.gc.offset; } static void mediatek_gpio_irq_unmask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct mtk_gc *rg = to_mediatek_gpio(gc); - int pin = d->hwirq; + struct mtk_gc *rg = gpiochip_get_data(gc); + u32 mask = mt7621_gpio_hwirq_to_offset(d->hwirq, rg); u32 rise, fall, high, low; - gpiochip_enable_irq(gc, d->hwirq); + gpiochip_enable_irq(gc, mask); guard(gpio_generic_lock_irqsave)(&rg->chip); @@ -119,18 +152,18 @@ mediatek_gpio_irq_unmask(struct irq_data *d) fall = mtk_gpio_r32(rg, GPIO_REG_FEDGE); high = mtk_gpio_r32(rg, GPIO_REG_HLVL); low = mtk_gpio_r32(rg, GPIO_REG_LLVL); - mtk_gpio_w32(rg, GPIO_REG_REDGE, rise | (BIT(pin) & rg->rising)); - mtk_gpio_w32(rg, GPIO_REG_FEDGE, fall | (BIT(pin) & rg->falling)); - mtk_gpio_w32(rg, GPIO_REG_HLVL, high | (BIT(pin) & rg->hlevel)); - mtk_gpio_w32(rg, GPIO_REG_LLVL, low | (BIT(pin) & rg->llevel)); + mtk_gpio_w32(rg, GPIO_REG_REDGE, rise | (BIT(mask) & rg->rising)); + mtk_gpio_w32(rg, GPIO_REG_FEDGE, fall | (BIT(mask) & rg->falling)); + mtk_gpio_w32(rg, GPIO_REG_HLVL, high | (BIT(mask) & rg->hlevel)); + mtk_gpio_w32(rg, GPIO_REG_LLVL, low | (BIT(mask) & rg->llevel)); } static void mediatek_gpio_irq_mask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct mtk_gc *rg = to_mediatek_gpio(gc); - int pin = d->hwirq; + struct mtk_gc *rg = gpiochip_get_data(gc); + u32 mask = mt7621_gpio_hwirq_to_offset(d->hwirq, rg); u32 rise, fall, high, low; scoped_guard(gpio_generic_lock_irqsave, &rg->chip) { @@ -138,22 +171,23 @@ mediatek_gpio_irq_mask(struct irq_data *d) fall = mtk_gpio_r32(rg, GPIO_REG_FEDGE); high = mtk_gpio_r32(rg, GPIO_REG_HLVL); low = mtk_gpio_r32(rg, GPIO_REG_LLVL); - mtk_gpio_w32(rg, GPIO_REG_FEDGE, fall & ~BIT(pin)); - mtk_gpio_w32(rg, GPIO_REG_REDGE, rise & ~BIT(pin)); - mtk_gpio_w32(rg, GPIO_REG_HLVL, high & ~BIT(pin)); - mtk_gpio_w32(rg, GPIO_REG_LLVL, low & ~BIT(pin)); + mtk_gpio_w32(rg, GPIO_REG_FEDGE, fall & ~BIT(mask)); + mtk_gpio_w32(rg, GPIO_REG_REDGE, rise & ~BIT(mask)); + mtk_gpio_w32(rg, GPIO_REG_HLVL, high & ~BIT(mask)); + mtk_gpio_w32(rg, GPIO_REG_LLVL, low & ~BIT(mask)); } - gpiochip_disable_irq(gc, d->hwirq); + gpiochip_disable_irq(gc, mask); } static int mediatek_gpio_irq_type(struct irq_data *d, unsigned int type) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct mtk_gc *rg = to_mediatek_gpio(gc); - int pin = d->hwirq; - u32 mask = BIT(pin); + struct mtk_gc *rg = gpiochip_get_data(gc); + u32 mask = BIT(mt7621_gpio_hwirq_to_offset(d->hwirq, rg)); + + guard(gpio_generic_lock_irqsave)(&rg->chip); if (type == IRQ_TYPE_PROBE) { if ((rg->rising | rg->falling | @@ -190,6 +224,26 @@ mediatek_gpio_irq_type(struct irq_data *d, unsigned int type) return 0; } +static int +mt7621_gpio_irq_reqres(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct mtk_gc *rg = gpiochip_get_data(gc); + unsigned int irq = mt7621_gpio_hwirq_to_offset(d->hwirq, rg); + + return gpiochip_reqres_irq(gc, irq); +} + +static void +mt7621_gpio_irq_relres(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct mtk_gc *rg = gpiochip_get_data(gc); + unsigned int irq = mt7621_gpio_hwirq_to_offset(d->hwirq, rg); + + gpiochip_relres_irq(gc, irq); +} + static int mediatek_gpio_xlate(struct gpio_chip *chip, const struct of_phandle_args *spec, u32 *flags) @@ -208,14 +262,123 @@ mediatek_gpio_xlate(struct gpio_chip *chip, static const struct irq_chip mt7621_irq_chip = { .name = "mt7621-gpio", + .irq_request_resources = mt7621_gpio_irq_reqres, + .irq_release_resources = mt7621_gpio_irq_relres, .irq_mask_ack = mediatek_gpio_irq_mask, .irq_mask = mediatek_gpio_irq_mask, .irq_unmask = mediatek_gpio_irq_unmask, .irq_set_type = mediatek_gpio_irq_type, .flags = IRQCHIP_IMMUTABLE, - GPIOCHIP_IRQ_RESOURCE_HELPERS, }; +static void +mt7621_gpio_remove(void *data) +{ + struct mtk *priv = data; + int offset, virq; + + if (priv->gpio_irq > 0) + irq_set_chained_handler_and_data(priv->gpio_irq, NULL, NULL); + + /* Remove all IRQ mappings and delete the domain */ + if (priv->irq_domain) { + for (offset = 0; offset < priv->num_gpios; offset++) { + virq = irq_find_mapping(priv->irq_domain, offset); + irq_dispose_mapping(virq); + } + irq_domain_remove(priv->irq_domain); + } +} + +static struct mtk_gc * +mt7621_gpio_hwirq_to_bank(struct mtk *priv, irq_hw_number_t hwirq) +{ + int i; + + for (i = 0; i < MTK_BANK_CNT; i++) { + struct mtk_gc *bank = &priv->gc_map[i]; + + if (hwirq >= bank->chip.gc.offset && + hwirq < (bank->chip.gc.offset + bank->chip.gc.ngpio)) + return bank; + } + + return NULL; +} + +static int +mt7621_gpio_irq_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + struct mtk *priv = d->host_data; + struct mtk_gc *bank = mt7621_gpio_hwirq_to_bank(priv, hwirq); + struct platform_device *pdev = priv->pdev; + int ret; + + if (!bank) + return -EINVAL; + + dev_dbg(&pdev->dev, "Mapping irq %d for gpio line %d (bank %d)\n", + irq, (int)hwirq, bank->bank); + + ret = irq_set_chip_data(irq, &bank->chip.gc); + if (ret < 0) + return ret; + + irq_set_chip_and_handler(irq, &mt7621_irq_chip, handle_simple_irq); + irq_set_noprobe(irq); + + return 0; +} + +static void +mt7621_gpio_irq_unmap(struct irq_domain *d, unsigned int irq) +{ + irq_set_chip_and_handler(irq, NULL, NULL); + irq_set_chip_data(irq, NULL); +} + +static const struct irq_domain_ops mt7621_gpio_irq_domain_ops = { + .map = mt7621_gpio_irq_map, + .unmap = mt7621_gpio_irq_unmap, + .xlate = irq_domain_xlate_twocell, +}; + +static int +mt7621_gpio_irq_setup(struct platform_device *pdev, + struct mtk *priv) +{ + struct device *dev = &pdev->dev; + + priv->irq_domain = irq_domain_create_linear(dev_fwnode(dev), + priv->num_gpios, + &mt7621_gpio_irq_domain_ops, + priv); + if (!priv->irq_domain) { + dev_err(dev, "Couldn't allocate IRQ domain\n"); + return -ENXIO; + } + + irq_set_chained_handler_and_data(priv->gpio_irq, + mt7621_gpio_irq_handler, priv); + irq_set_status_flags(priv->gpio_irq, IRQ_DISABLE_UNLAZY); + + return 0; +} + +static int +mt7621_gpio_to_irq(struct gpio_chip *gc, unsigned int offset) +{ + struct mtk *priv = mt7621_gpio_gc_to_priv(gc); + /* gc_offset is relative to this gpio_chip; want real offset */ + int hwirq = offset + gc->offset; + + if (hwirq >= priv->num_gpios) + return -ENXIO; + + return irq_create_mapping(priv->irq_domain, hwirq); +} + static int mediatek_gpio_bank_probe(struct device *dev, int bank) { @@ -228,6 +391,7 @@ mediatek_gpio_bank_probe(struct device *dev, int bank) rg = &mtk->gc_map[bank]; memset(rg, 0, sizeof(*rg)); + rg->parent_priv = mtk; rg->bank = bank; dat = mtk->base + GPIO_REG_DATA + (rg->bank * GPIO_BANK_STRIDE); @@ -253,41 +417,17 @@ mediatek_gpio_bank_probe(struct device *dev, int bank) rg->chip.gc.of_gpio_n_cells = 2; rg->chip.gc.of_xlate = mediatek_gpio_xlate; + rg->chip.gc.ngpio = MTK_BANK_WIDTH; rg->chip.gc.label = devm_kasprintf(dev, GFP_KERNEL, "%s-bank%d", dev_name(dev), bank); if (!rg->chip.gc.label) return -ENOMEM; rg->chip.gc.offset = bank * MTK_BANK_WIDTH; + if (mtk->gpio_irq > 0) + rg->chip.gc.to_irq = mt7621_gpio_to_irq; - if (mtk->gpio_irq) { - struct gpio_irq_chip *girq; - - /* - * Directly request the irq here instead of passing - * a flow-handler because the irq is shared. - */ - ret = devm_request_irq(dev, mtk->gpio_irq, - mediatek_gpio_irq_handler, IRQF_SHARED, - rg->chip.gc.label, &rg->chip.gc); - - if (ret) { - dev_err(dev, "Error requesting IRQ %d: %d\n", - mtk->gpio_irq, ret); - return ret; - } - - girq = &rg->chip.gc.irq; - gpio_irq_chip_set_chip(girq, &mt7621_irq_chip); - /* This will let us handle the parent IRQ in the driver */ - girq->parent_handler = NULL; - girq->num_parents = 0; - girq->parents = NULL; - girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; - } - - ret = devm_gpiochip_add_data(dev, &rg->chip.gc, mtk); + ret = devm_gpiochip_add_data(dev, &rg->chip.gc, rg); if (ret < 0) { dev_err(dev, "Could not register gpio %d, ret=%d\n", rg->chip.gc.ngpio, ret); @@ -322,9 +462,20 @@ mediatek_gpio_probe(struct platform_device *pdev) if (mtk->gpio_irq < 0) return mtk->gpio_irq; - mtk->dev = dev; + mtk->pdev = pdev; + mtk->num_gpios = MTK_BANK_WIDTH * MTK_BANK_CNT; platform_set_drvdata(pdev, mtk); + if (mtk->gpio_irq > 0) { + ret = mt7621_gpio_irq_setup(pdev, mtk); + if (ret) + return ret; + } + + ret = devm_add_action_or_reset(dev, mt7621_gpio_remove, mtk); + if (ret) + return ret; + for (i = 0; i < MTK_BANK_CNT; i++) { ret = mediatek_gpio_bank_probe(dev, i); if (ret) diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index c030d1f00abca..a010604e5ff79 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -1110,6 +1110,7 @@ static void mvebu_gpio_remove_irq_domain(void *data) { struct irq_domain *domain = data; + irq_domain_remove_generic_chips(domain); irq_domain_remove(domain); } @@ -1221,7 +1222,10 @@ static int mvebu_gpio_probe(struct platform_device *pdev) BUG(); } - devm_gpiochip_add_data(&pdev->dev, &mvchip->chip, mvchip); + err = devm_gpiochip_add_data(&pdev->dev, &mvchip->chip, mvchip); + if (err) + return dev_err_probe(&pdev->dev, err, + "failed to register gpiochip\n"); /* Some MVEBU SoCs have simple PWM support for GPIO lines */ if (IS_REACHABLE(CONFIG_PWM)) { diff --git a/drivers/gpio/gpio-palmas.c b/drivers/gpio/gpio-palmas.c index e377f6dd4ccf1..e64ee04877185 100644 --- a/drivers/gpio/gpio-palmas.c +++ b/drivers/gpio/gpio-palmas.c @@ -116,6 +116,24 @@ static int palmas_gpio_input(struct gpio_chip *gc, unsigned offset) return ret; } +static int palmas_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) +{ + struct palmas_gpio *pg = gpiochip_get_data(gc); + struct palmas *palmas = pg->palmas; + unsigned int val; + unsigned int reg; + int ret; + int gpio16 = (offset/8); + + offset %= 8; + reg = (gpio16) ? PALMAS_GPIO_DATA_DIR2 : PALMAS_GPIO_DATA_DIR; + ret = palmas_read(palmas, PALMAS_GPIO_BASE, reg, &val); + if (ret) + return ret; + + return (val & BIT(offset)) ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + static int palmas_gpio_to_irq(struct gpio_chip *gc, unsigned offset) { struct palmas_gpio *pg = gpiochip_get_data(gc); @@ -165,6 +183,7 @@ static int palmas_gpio_probe(struct platform_device *pdev) palmas_gpio->gpio_chip.can_sleep = true; palmas_gpio->gpio_chip.direction_input = palmas_gpio_input; palmas_gpio->gpio_chip.direction_output = palmas_gpio_output; + palmas_gpio->gpio_chip.get_direction = palmas_gpio_get_direction; palmas_gpio->gpio_chip.to_irq = palmas_gpio_to_irq; palmas_gpio->gpio_chip.set = palmas_gpio_set; palmas_gpio->gpio_chip.get = palmas_gpio_get; diff --git a/drivers/gpio/gpio-sch.c b/drivers/gpio/gpio-sch.c index 966d16a6d5157..5e361742a11ad 100644 --- a/drivers/gpio/gpio-sch.c +++ b/drivers/gpio/gpio-sch.c @@ -39,7 +39,7 @@ struct sch_gpio { struct gpio_chip chip; void __iomem *regs; - spinlock_t lock; + raw_spinlock_t lock; unsigned short resume_base; /* GPE handling */ @@ -104,9 +104,9 @@ static int sch_gpio_direction_in(struct gpio_chip *gc, unsigned int gpio_num) struct sch_gpio *sch = gpiochip_get_data(gc); unsigned long flags; - spin_lock_irqsave(&sch->lock, flags); + raw_spin_lock_irqsave(&sch->lock, flags); sch_gpio_reg_set(sch, gpio_num, GIO, 1); - spin_unlock_irqrestore(&sch->lock, flags); + raw_spin_unlock_irqrestore(&sch->lock, flags); return 0; } @@ -122,9 +122,9 @@ static int sch_gpio_set(struct gpio_chip *gc, unsigned int gpio_num, int val) struct sch_gpio *sch = gpiochip_get_data(gc); unsigned long flags; - spin_lock_irqsave(&sch->lock, flags); + raw_spin_lock_irqsave(&sch->lock, flags); sch_gpio_reg_set(sch, gpio_num, GLV, val); - spin_unlock_irqrestore(&sch->lock, flags); + raw_spin_unlock_irqrestore(&sch->lock, flags); return 0; } @@ -135,9 +135,9 @@ static int sch_gpio_direction_out(struct gpio_chip *gc, unsigned int gpio_num, struct sch_gpio *sch = gpiochip_get_data(gc); unsigned long flags; - spin_lock_irqsave(&sch->lock, flags); + raw_spin_lock_irqsave(&sch->lock, flags); sch_gpio_reg_set(sch, gpio_num, GIO, 0); - spin_unlock_irqrestore(&sch->lock, flags); + raw_spin_unlock_irqrestore(&sch->lock, flags); /* * according to the datasheet, writing to the level register has no @@ -196,14 +196,14 @@ static int sch_irq_type(struct irq_data *d, unsigned int type) return -EINVAL; } - spin_lock_irqsave(&sch->lock, flags); + raw_spin_lock_irqsave(&sch->lock, flags); sch_gpio_reg_set(sch, gpio_num, GTPE, rising); sch_gpio_reg_set(sch, gpio_num, GTNE, falling); irq_set_handler_locked(d, handle_edge_irq); - spin_unlock_irqrestore(&sch->lock, flags); + raw_spin_unlock_irqrestore(&sch->lock, flags); return 0; } @@ -215,9 +215,9 @@ static void sch_irq_ack(struct irq_data *d) irq_hw_number_t gpio_num = irqd_to_hwirq(d); unsigned long flags; - spin_lock_irqsave(&sch->lock, flags); + raw_spin_lock_irqsave(&sch->lock, flags); sch_gpio_reg_set(sch, gpio_num, GTS, 1); - spin_unlock_irqrestore(&sch->lock, flags); + raw_spin_unlock_irqrestore(&sch->lock, flags); } static void sch_irq_mask_unmask(struct gpio_chip *gc, irq_hw_number_t gpio_num, int val) @@ -225,9 +225,9 @@ static void sch_irq_mask_unmask(struct gpio_chip *gc, irq_hw_number_t gpio_num, struct sch_gpio *sch = gpiochip_get_data(gc); unsigned long flags; - spin_lock_irqsave(&sch->lock, flags); + raw_spin_lock_irqsave(&sch->lock, flags); sch_gpio_reg_set(sch, gpio_num, GGPE, val); - spin_unlock_irqrestore(&sch->lock, flags); + raw_spin_unlock_irqrestore(&sch->lock, flags); } static void sch_irq_mask(struct irq_data *d) @@ -268,12 +268,12 @@ static u32 sch_gpio_gpe_handler(acpi_handle gpe_device, u32 gpe, void *context) int offset; u32 ret; - spin_lock_irqsave(&sch->lock, flags); + raw_spin_lock_irqsave(&sch->lock, flags); core_status = ioread32(sch->regs + CORE_BANK_OFFSET + GTS); resume_status = ioread32(sch->regs + RESUME_BANK_OFFSET + GTS); - spin_unlock_irqrestore(&sch->lock, flags); + raw_spin_unlock_irqrestore(&sch->lock, flags); pending = (resume_status << sch->resume_base) | core_status; for_each_set_bit(offset, &pending, sch->chip.ngpio) @@ -343,7 +343,7 @@ static int sch_gpio_probe(struct platform_device *pdev) sch->regs = regs; - spin_lock_init(&sch->lock); + raw_spin_lock_init(&sch->lock); sch->chip = sch_gpio_chip; sch->chip.label = dev_name(dev); sch->chip.parent = dev; diff --git a/drivers/gpio/gpio-shared-proxy.c b/drivers/gpio/gpio-shared-proxy.c index 6941e4be6cf18..52a366f0ec4de 100644 --- a/drivers/gpio/gpio-shared-proxy.c +++ b/drivers/gpio/gpio-shared-proxy.c @@ -9,8 +9,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -20,66 +22,66 @@ struct gpio_shared_proxy_data { struct gpio_chip gc; struct gpio_shared_desc *shared_desc; struct device *dev; - bool voted_high; + bool voted_change; }; static int -gpio_shared_proxy_set_unlocked(struct gpio_shared_proxy_data *proxy, - int (*set_func)(struct gpio_desc *desc, int value), - int value) +gpio_shared_proxy_set_unlocked(struct gpio_shared_proxy_data *proxy, int value) { struct gpio_shared_desc *shared_desc = proxy->shared_desc; struct gpio_desc *desc = shared_desc->desc; int ret = 0; - gpio_shared_lockdep_assert(shared_desc); + lockdep_assert_held(&shared_desc->mutex); - if (value) { - /* User wants to set value to high. */ - if (proxy->voted_high) - /* Already voted for high, nothing to do. */ + if (value != shared_desc->def_val) { + /* User wants to vote for a value change. */ + if (proxy->voted_change) + /* Already voted for a change, nothing to do. */ goto out; - /* Haven't voted for high yet. */ - if (!shared_desc->highcnt) { + /* Haven't voted for a value change yet. */ + if (!shared_desc->votecnt) { /* - * Current value is low, need to actually set value - * to high. + * Current value is default, need to actually set value + * to the opposite. */ - ret = set_func(desc, 1); + ret = gpiod_set_value_cansleep(desc, value); if (ret) goto out; } - shared_desc->highcnt++; - proxy->voted_high = true; + shared_desc->votecnt++; + proxy->voted_change = true; goto out; } - /* Desired value is low. */ - if (!proxy->voted_high) - /* We didn't vote for high, nothing to do. */ + /* Desired value is the default. */ + if (!proxy->voted_change) + /* We didn't vote for change previously, nothing to do. */ goto out; - /* We previously voted for high. */ - if (shared_desc->highcnt == 1) { - /* This is the last remaining vote for high, set value to low. */ - ret = set_func(desc, 0); + /* We previously voted for change. */ + if (shared_desc->votecnt == 1) { + /* This is the last remaining vote for change, set value to default. */ + ret = gpiod_set_value_cansleep(desc, shared_desc->def_val); if (ret) goto out; } - shared_desc->highcnt--; - proxy->voted_high = false; + shared_desc->votecnt--; + proxy->voted_change = false; out: - if (shared_desc->highcnt) + if (shared_desc->votecnt) dev_dbg(proxy->dev, - "Voted for value '%s', effective value is 'high', number of votes for 'high': %u\n", - str_high_low(value), shared_desc->highcnt); + "Voted for value '%s', effective value is '%s', number of votes: %u\n", + str_high_low(value), str_high_low(!shared_desc->def_val), + shared_desc->votecnt); else - dev_dbg(proxy->dev, "Voted for value 'low', effective value is 'low'\n"); + dev_dbg(proxy->dev, "Voted for value '%s', effective value is '%s'\n", + str_high_low(value), str_high_low(shared_desc->def_val)); return ret; } @@ -89,7 +91,7 @@ static int gpio_shared_proxy_request(struct gpio_chip *gc, unsigned int offset) struct gpio_shared_proxy_data *proxy = gpiochip_get_data(gc); struct gpio_shared_desc *shared_desc = proxy->shared_desc; - guard(gpio_shared_desc_lock)(shared_desc); + guard(mutex)(&shared_desc->mutex); proxy->shared_desc->usecnt++; @@ -105,11 +107,10 @@ static void gpio_shared_proxy_free(struct gpio_chip *gc, unsigned int offset) struct gpio_shared_desc *shared_desc = proxy->shared_desc; int ret; - guard(gpio_shared_desc_lock)(shared_desc); + guard(mutex)(&shared_desc->mutex); - if (proxy->voted_high) { - ret = gpio_shared_proxy_set_unlocked(proxy, - shared_desc->can_sleep ? gpiod_set_value_cansleep : gpiod_set_value, 0); + if (proxy->voted_change) { + ret = gpio_shared_proxy_set_unlocked(proxy, shared_desc->def_val); if (ret) dev_err(proxy->dev, "Failed to unset the shared GPIO value on release: %d\n", ret); @@ -129,7 +130,7 @@ static int gpio_shared_proxy_set_config(struct gpio_chip *gc, struct gpio_desc *desc = shared_desc->desc; int ret; - guard(gpio_shared_desc_lock)(shared_desc); + guard(mutex)(&shared_desc->mutex); if (shared_desc->usecnt > 1) { if (shared_desc->cfg != cfg) { @@ -157,7 +158,7 @@ static int gpio_shared_proxy_direction_input(struct gpio_chip *gc, struct gpio_desc *desc = shared_desc->desc; int dir; - guard(gpio_shared_desc_lock)(shared_desc); + guard(mutex)(&shared_desc->mutex); if (shared_desc->usecnt == 1) { dev_dbg(proxy->dev, @@ -187,7 +188,7 @@ static int gpio_shared_proxy_direction_output(struct gpio_chip *gc, struct gpio_desc *desc = shared_desc->desc; int ret, dir; - guard(gpio_shared_desc_lock)(shared_desc); + guard(mutex)(&shared_desc->mutex); if (shared_desc->usecnt == 1) { dev_dbg(proxy->dev, @@ -198,13 +199,9 @@ static int gpio_shared_proxy_direction_output(struct gpio_chip *gc, if (ret) return ret; - if (value) { - proxy->voted_high = true; - shared_desc->highcnt = 1; - } else { - proxy->voted_high = false; - shared_desc->highcnt = 0; - } + shared_desc->def_val = value; + shared_desc->votecnt = 0; + proxy->voted_change = false; return 0; } @@ -219,14 +216,7 @@ static int gpio_shared_proxy_direction_output(struct gpio_chip *gc, return -EPERM; } - return gpio_shared_proxy_set_unlocked(proxy, gpiod_direction_output, value); -} - -static int gpio_shared_proxy_get(struct gpio_chip *gc, unsigned int offset) -{ - struct gpio_shared_proxy_data *proxy = gpiochip_get_data(gc); - - return gpiod_get_value(proxy->shared_desc->desc); + return gpio_shared_proxy_set_unlocked(proxy, value); } static int gpio_shared_proxy_get_cansleep(struct gpio_chip *gc, @@ -237,29 +227,14 @@ static int gpio_shared_proxy_get_cansleep(struct gpio_chip *gc, return gpiod_get_value_cansleep(proxy->shared_desc->desc); } -static int gpio_shared_proxy_do_set(struct gpio_shared_proxy_data *proxy, - int (*set_func)(struct gpio_desc *desc, int value), - int value) -{ - guard(gpio_shared_desc_lock)(proxy->shared_desc); - - return gpio_shared_proxy_set_unlocked(proxy, set_func, value); -} - -static int gpio_shared_proxy_set(struct gpio_chip *gc, unsigned int offset, - int value) -{ - struct gpio_shared_proxy_data *proxy = gpiochip_get_data(gc); - - return gpio_shared_proxy_do_set(proxy, gpiod_set_value, value); -} - static int gpio_shared_proxy_set_cansleep(struct gpio_chip *gc, unsigned int offset, int value) { struct gpio_shared_proxy_data *proxy = gpiochip_get_data(gc); - return gpio_shared_proxy_do_set(proxy, gpiod_set_value_cansleep, value); + guard(mutex)(&proxy->shared_desc->mutex); + + return gpio_shared_proxy_set_unlocked(proxy, value); } static int gpio_shared_proxy_get_direction(struct gpio_chip *gc, @@ -302,20 +277,25 @@ static int gpio_shared_proxy_probe(struct auxiliary_device *adev, gc->label = dev_name(dev); gc->parent = dev; gc->owner = THIS_MODULE; - gc->can_sleep = shared_desc->can_sleep; + /* + * Under the descriptor mutex the proxy may call + * gpiod_set_config()/gpiod_direction_*(), which can reach pinctrl + * paths that take a mutex (e.g. gpiod_set_config() -> + * gpiochip_generic_config() -> pinctrl_gpio_set_config()), independent + * of the underlying chip's can_sleep. So the descriptor lock must be a + * mutex and the proxy gpiochip is therefore always sleeping; drive the + * underlying GPIO through the cansleep value accessors, which are valid + * for both sleeping and non-sleeping chips. + */ + gc->can_sleep = true; gc->request = gpio_shared_proxy_request; gc->free = gpio_shared_proxy_free; gc->set_config = gpio_shared_proxy_set_config; gc->direction_input = gpio_shared_proxy_direction_input; gc->direction_output = gpio_shared_proxy_direction_output; - if (gc->can_sleep) { - gc->set = gpio_shared_proxy_set_cansleep; - gc->get = gpio_shared_proxy_get_cansleep; - } else { - gc->set = gpio_shared_proxy_set; - gc->get = gpio_shared_proxy_get; - } + gc->set = gpio_shared_proxy_set_cansleep; + gc->get = gpio_shared_proxy_get_cansleep; gc->get_direction = gpio_shared_proxy_get_direction; gc->to_irq = gpio_shared_proxy_to_irq; diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index df06b56a2ade8..fa6c8ee92093c 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -172,18 +172,11 @@ static int tegra_gpio_direction_input(struct gpio_chip *chip, unsigned int offset) { struct tegra_gpio_info *tgi = gpiochip_get_data(chip); - int ret; tegra_gpio_mask_write(tgi, GPIO_MSK_OE(tgi, offset), offset, 0); tegra_gpio_enable(tgi, offset); - ret = pinctrl_gpio_direction_input(chip, offset); - if (ret < 0) - dev_err(tgi->dev, - "Failed to set pinctrl input direction of GPIO %d: %d", - chip->base + offset, ret); - - return ret; + return 0; } static int tegra_gpio_direction_output(struct gpio_chip *chip, @@ -191,19 +184,12 @@ static int tegra_gpio_direction_output(struct gpio_chip *chip, int value) { struct tegra_gpio_info *tgi = gpiochip_get_data(chip); - int ret; tegra_gpio_set(chip, offset, value); tegra_gpio_mask_write(tgi, GPIO_MSK_OE(tgi, offset), offset, 1); tegra_gpio_enable(tgi, offset); - ret = pinctrl_gpio_direction_output(chip, offset); - if (ret < 0) - dev_err(tgi->dev, - "Failed to set pinctrl output direction of GPIO %d: %d", - chip->base + offset, ret); - - return ret; + return 0; } static int tegra_gpio_get_direction(struct gpio_chip *chip, diff --git a/drivers/gpio/gpio-timberdale.c b/drivers/gpio/gpio-timberdale.c index f488939dd00a8..b77c9d16fd166 100644 --- a/drivers/gpio/gpio-timberdale.c +++ b/drivers/gpio/gpio-timberdale.c @@ -235,7 +235,7 @@ static int timbgpio_probe(struct platform_device *pdev) tgpio = devm_kzalloc(dev, sizeof(*tgpio), GFP_KERNEL); if (!tgpio) - return -EINVAL; + return -ENOMEM; tgpio->irq_base = pdata->irq_base; diff --git a/drivers/gpio/gpiolib-acpi-core.c b/drivers/gpio/gpiolib-acpi-core.c index 09f860200a059..eb8a40cfb7a98 100644 --- a/drivers/gpio/gpiolib-acpi-core.c +++ b/drivers/gpio/gpiolib-acpi-core.c @@ -233,12 +233,23 @@ static void acpi_gpiochip_request_irq(struct acpi_gpio_chip *acpi_gpio, event->irq_requested = true; - /* Make sure we trigger the initial state of edge-triggered IRQs */ + /* + * Make sure we trigger the initial state of ActiveBoth IRQs. + * + * According to the Microsoft GPIO documentation, triggering GPIO + * interrupts marked as ActiveBoth during initialization is correct + * as long as the associated GPIO line is already "asserted" + * (logic level low). We should not trigger edge-based GPIO + * interrupts not marked as ActiveBoth. + * + * See: https://learn.microsoft.com/en-us/windows-hardware/drivers/bringup/general-purpose-i-o--gpio- + * Section: "GPIO controllers and ActiveBoth interrupts" + */ if (acpi_gpio_need_run_edge_events_on_boot() && - (event->irqflags & (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING))) { + ((event->irqflags & (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING)) == + (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING))) { value = gpiod_get_raw_value_cansleep(event->desc); - if (((event->irqflags & IRQF_TRIGGER_RISING) && value == 1) || - ((event->irqflags & IRQF_TRIGGER_FALLING) && value == 0)) + if (value == 0) event->handler(event->irq, event); } } diff --git a/drivers/gpio/gpiolib-shared.c b/drivers/gpio/gpiolib-shared.c index de72776fb154f..495bd3d0ddf02 100644 --- a/drivers/gpio/gpiolib-shared.c +++ b/drivers/gpio/gpiolib-shared.c @@ -627,8 +627,7 @@ static void gpio_shared_release(struct kref *kref) shared_desc = entry->shared_desc; gpio_device_put(shared_desc->desc->gdev); - if (shared_desc->can_sleep) - mutex_destroy(&shared_desc->mutex); + mutex_destroy(&shared_desc->mutex); kfree(shared_desc); entry->shared_desc = NULL; } @@ -659,11 +658,7 @@ gpiod_shared_desc_create(struct gpio_shared_entry *entry) } shared_desc->desc = &gdev->descs[entry->offset]; - shared_desc->can_sleep = gpiod_cansleep(shared_desc->desc); - if (shared_desc->can_sleep) - mutex_init(&shared_desc->mutex); - else - spin_lock_init(&shared_desc->spinlock); + mutex_init(&shared_desc->mutex); return shared_desc; } diff --git a/drivers/gpio/gpiolib-shared.h b/drivers/gpio/gpiolib-shared.h index 15e72a8dcdb13..618756f6c6aaf 100644 --- a/drivers/gpio/gpiolib-shared.h +++ b/drivers/gpio/gpiolib-shared.h @@ -3,10 +3,7 @@ #ifndef __LINUX_GPIO_SHARED_H #define __LINUX_GPIO_SHARED_H -#include -#include #include -#include struct gpio_device; struct gpio_desc; @@ -42,35 +39,13 @@ static inline int gpio_shared_add_proxy_lookup(struct device *consumer, struct gpio_shared_desc { struct gpio_desc *desc; - bool can_sleep; unsigned long cfg; unsigned int usecnt; - unsigned int highcnt; - union { - struct mutex mutex; - spinlock_t spinlock; - }; + unsigned int votecnt; + int def_val; + struct mutex mutex; /* serializes all proxy operations on this descriptor */ }; struct gpio_shared_desc *devm_gpiod_shared_get(struct device *dev); -DEFINE_LOCK_GUARD_1(gpio_shared_desc_lock, struct gpio_shared_desc, - if (_T->lock->can_sleep) - mutex_lock(&_T->lock->mutex); - else - spin_lock_irqsave(&_T->lock->spinlock, _T->flags), - if (_T->lock->can_sleep) - mutex_unlock(&_T->lock->mutex); - else - spin_unlock_irqrestore(&_T->lock->spinlock, _T->flags), - unsigned long flags) - -static inline void gpio_shared_lockdep_assert(struct gpio_shared_desc *shared_desc) -{ - if (shared_desc->can_sleep) - lockdep_assert_held(&shared_desc->mutex); - else - lockdep_assert_held(&shared_desc->spinlock); -} - #endif /* __LINUX_GPIO_SHARED_H */ diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index c1f9c0d367d5c..58479931a6906 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -3786,7 +3786,7 @@ static int gpiochip_set_multiple(struct gpio_chip *gc, unsigned long *mask, unsigned long *bits) { unsigned int i; - int ret; + int ret = 0; lockdep_assert_held(&gc->gpiodev->srcu); diff --git a/drivers/gpu/buddy.c b/drivers/gpu/buddy.c index 52686672e99f5..5490dcd27f22c 100644 --- a/drivers/gpu/buddy.c +++ b/drivers/gpu/buddy.c @@ -1078,22 +1078,30 @@ static int __gpu_buddy_alloc_range(struct gpu_buddy *mm, blocks, total_allocated_on_err); } +static int __alloc_contig_aligned_retry(struct gpu_buddy *mm, + u64 unaligned_offset, + u64 size, + u64 min_block_size, + struct list_head *blocks) +{ + u64 aligned_offset = round_down(unaligned_offset, min_block_size); + + return __gpu_buddy_alloc_range(mm, aligned_offset, size, NULL, blocks); +} + static int __alloc_contig_try_harder(struct gpu_buddy *mm, u64 size, u64 min_block_size, struct list_head *blocks) { - u64 rhs_offset, lhs_offset, lhs_size, filled; + u64 rhs_offset, lhs_offset, filled; struct gpu_buddy_block *block; unsigned int tree, order; - LIST_HEAD(blocks_lhs); - unsigned long pages; u64 modify_size; int err; modify_size = rounddown_pow_of_two(size); - pages = modify_size >> ilog2(mm->chunk_size); - order = fls(pages) - 1; + order = ilog2(modify_size) - ilog2(mm->chunk_size); if (order == 0) return -ENOSPC; @@ -1109,31 +1117,48 @@ static int __alloc_contig_try_harder(struct gpu_buddy *mm, while (iter) { block = rbtree_get_free_block(iter); - /* Allocate blocks traversing RHS */ rhs_offset = gpu_buddy_block_offset(block); + + /* Allocate blocks traversing RHS */ err = __gpu_buddy_alloc_range(mm, rhs_offset, size, &filled, blocks); - if (!err || err != -ENOSPC) + if (err && err != -ENOSPC) return err; + if (!err && IS_ALIGNED(rhs_offset, min_block_size)) + return 0; + if (!err) { + /* Allocate the unaligned RHS offset using round_down */ + gpu_buddy_free_list_internal(mm, blocks); + err = __alloc_contig_aligned_retry(mm, rhs_offset, + size, + min_block_size, + blocks); + if (!err) + return 0; + if (err != -ENOSPC) { + gpu_buddy_free_list_internal(mm, blocks); + return err; + } + goto next; + } - lhs_size = max((size - filled), min_block_size); - if (!IS_ALIGNED(lhs_size, min_block_size)) - lhs_size = round_up(lhs_size, min_block_size); + if (size - filled > rhs_offset) + goto next; - /* Allocate blocks traversing LHS */ - lhs_offset = gpu_buddy_block_offset(block) - lhs_size; - err = __gpu_buddy_alloc_range(mm, lhs_offset, lhs_size, - NULL, &blocks_lhs); - if (!err) { - list_splice(&blocks_lhs, blocks); + lhs_offset = rhs_offset - (size - filled); + + /* Allocate the unaligned LHS offset using round_down */ + gpu_buddy_free_list_internal(mm, blocks); + err = __alloc_contig_aligned_retry(mm, lhs_offset, size, + min_block_size, blocks); + if (!err) return 0; - } else if (err != -ENOSPC) { + if (err != -ENOSPC) { gpu_buddy_free_list_internal(mm, blocks); return err; } - /* Free blocks for the next iteration */ +next: gpu_buddy_free_list_internal(mm, blocks); - iter = rb_prev(iter); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 72a5a29e63f6d..abc3194889692 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1914,13 +1914,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( mutex_lock(&mem->lock); - /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */ - if (mem->alloc_flags & - (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | - KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { - amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); - } - mapped_to_gpu_memory = mem->mapped_to_gpu_memory; is_imported = mem->is_imported; mutex_unlock(&mem->lock); @@ -1934,6 +1927,15 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( return -EBUSY; } + /* At this point the BO is guaranteed to be freed, so unpin the + * MMIO/DOORBELL BOs that were pinned during allocation. + */ + if (mem->alloc_flags & + (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | + KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { + amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); + } + /* Make sure restore workers don't access the BO any more */ mutex_lock(&process_info->lock); if (!list_empty(&mem->validate_list)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c index d386bc775d03c..f89f3a37a45b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c @@ -229,6 +229,7 @@ amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_inf sizing_pass = buffer == NULL; iter.data = buffer; + iter.start = 0; iter.offset = 0; iter.remain = count; @@ -553,7 +554,7 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check, coredump->rings_dw = kzalloc(total_ring_size, GFP_NOWAIT); coredump->rings = kcalloc(ring_count, sizeof(struct amdgpu_coredump_ring), GFP_NOWAIT); if (coredump->rings && coredump->rings_dw) { - for (i = 0, off = 0, idx = 0; i < adev->num_rings; i++) { + for (i = 0, off = 0, idx = 0; i < adev->num_rings && idx < ring_count; i++) { ring = adev->rings[i]; if (atomic_read(&ring->fence_drv.last_seq) == ring->fence_drv.sync_seq && diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index feab90e3efd1e..4f8489626624d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3747,6 +3747,8 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->gfx.workload_profile_mutex); mutex_init(&adev->vcn.workload_profile_mutex); + spin_lock_init(&adev->irq.lock); + amdgpu_device_init_apu_flags(adev); r = amdgpu_device_check_arguments(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index fe6d988e7f245..1120f8225ac02 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -27,6 +27,7 @@ */ #include #include +#include #include #include #include @@ -1228,13 +1229,14 @@ int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data, return ret; } -static int amdgpu_gem_align_pitch(struct amdgpu_device *adev, - int width, - int cpp, - bool tiled) +static unsigned int amdgpu_gem_align_pitch(struct amdgpu_device *adev, + unsigned int width, + unsigned int cpp, + bool tiled) { - int aligned = width; - int pitch_mask = 0; + unsigned int aligned = width; + unsigned int pitch_mask = 0; + unsigned int pitch; switch (cpp) { case 1: @@ -1249,9 +1251,12 @@ static int amdgpu_gem_align_pitch(struct amdgpu_device *adev, break; } - aligned += pitch_mask; + if (check_add_overflow(aligned, pitch_mask, &aligned)) + return 0; aligned &= ~pitch_mask; - return aligned * cpp; + if (check_mul_overflow(aligned, cpp, &pitch)) + return 0; + return pitch; } int amdgpu_mode_dumb_create(struct drm_file *file_priv, @@ -1278,8 +1283,12 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, args->pitch = amdgpu_gem_align_pitch(adev, args->width, DIV_ROUND_UP(args->bpp, 8), 0); + if (!args->pitch) + return -EINVAL; args->size = (u64)args->pitch * args->height; args->size = ALIGN(args->size, PAGE_SIZE); + if (!args->size) + return -EINVAL; domain = amdgpu_bo_get_preferred_domain(adev, amdgpu_display_supported_domains(adev, flags)); r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c index e452444b33b08..99bc9ad67d5b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c @@ -174,7 +174,6 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, const u64 max_bytes = SZ_2G; struct hmm_range *hmm_range = &range->hmm_range; - unsigned long timeout; unsigned long *pfns; unsigned long end; int r; @@ -201,15 +200,9 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, pr_debug("hmm range: start = 0x%lx, end = 0x%lx", hmm_range->start, hmm_range->end); - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - -retry: r = hmm_range_fault(hmm_range); - if (unlikely(r)) { - if (r == -EBUSY && !time_after(jiffies, timeout)) - goto retry; + if (unlikely(r)) goto out_free_pfns; - } if (hmm_range->end == end) break; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 254a4e983f40a..40b8506ac66f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -309,8 +309,6 @@ int amdgpu_irq_init(struct amdgpu_device *adev) unsigned int irq, flags; int r; - spin_lock_init(&adev->irq.lock); - /* Enable MSI if not disabled by module parameter */ adev->irq.msi_enabled = false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c index 823ba17e32af4..cc6d1a4e4c3af 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c @@ -99,6 +99,7 @@ int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev) strcpy(ras->ras_block.ras_comm.name, "mca.mp0"); ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA; + ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP0; ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->mca.mp0.ras_if = &ras->ras_block.ras_comm; @@ -123,6 +124,7 @@ int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev) strcpy(ras->ras_block.ras_comm.name, "mca.mp1"); ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA; + ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP1; ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->mca.mp1.ras_if = &ras->ras_block.ras_comm; @@ -147,6 +149,7 @@ int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev) strcpy(ras->ras_block.ras_comm.name, "mca.mpio"); ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA; + ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MPIO; ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->mca.mpio.ras_if = &ras->ras_block.ras_comm; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c index cec8012781265..be86802fef33b 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c @@ -2016,7 +2016,7 @@ static int mes_v12_1_map_test_bo(struct amdgpu_device *adev, error: amdgpu_sync_free(&sync); - return 0; + return r; } static int mes_v12_1_test_ring(struct amdgpu_device *adev, int xcc_id, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 8785f7810157e..78068b2c96856 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -2362,6 +2362,9 @@ static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd, const bool criu_resume = true; u64 offset; + if (bo_priv->idr_handle > INT_MAX) + return -EINVAL; + if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) { if (bo_bucket->size != kfd_doorbell_process_slice(pdd->dev->kfd)) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 31187ddbb79ea..ee413117b0ff6 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -3095,32 +3095,24 @@ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbel list_for_each_entry(q, &qpd->queues_list, list) { if (q->doorbell_id == doorbell_id && q->properties.is_active) { - ret = suspend_all_queues_mes(dqm); - if (ret) { - dev_err(dev, "Suspending all queues failed"); - goto out; - } + /* suspend all queues will save any good queues and mark the rest as bad */ + suspend_all_queues_mes(dqm); q->properties.is_evicted = true; q->properties.is_active = false; decrement_queue_count(dqm, qpd, q); + /* this will remove the bad queue and sched a GPU reset if needed */ ret = remove_queue_mes(dqm, q, qpd); - if (ret) { - dev_err(dev, "Removing bad queue failed"); - goto out; - } - - ret = resume_all_queues_mes(dqm); if (ret) - dev_err(dev, "Resuming all queues failed"); - + dev_err(dev, "Removing bad queue failed"); + /* resume the good queues */ + resume_all_queues_mes(dqm); break; } } } -out: dqm_unlock(dqm); kfd_unref_process(p); return ret; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index e65b323aafbf3..81900b49d9d5b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -483,6 +483,11 @@ int kfd_criu_restore_event(struct file *devkfd, } *priv_data_offset += sizeof(*ev_priv); + if (ev_priv->event_id > INT_MAX) { + ret = -EINVAL; + goto exit; + } + if (ev_priv->user_handle) { ret = kfd_kmap_event_page(p, ev_priv->user_handle); if (ret) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c index c90c0d99b1e3f..475589b924e90 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c @@ -32,6 +32,10 @@ #include "amdgpu_amdkfd.h" #include "kfd_device_queue_manager.h" +static void update_mqd(struct mqd_manager *mm, void *mqd, + struct queue_properties *q, + struct mqd_update_info *minfo); + static inline struct v12_1_compute_mqd *get_mqd(void *mqd) { return (struct v12_1_compute_mqd *)mqd; @@ -215,7 +219,7 @@ static void init_mqd(struct mqd_manager *mm, void **mqd, *mqd = m; if (gart_addr) *gart_addr = addr; - mm->update_mqd(mm, m, q, NULL); + update_mqd(mm, m, q, NULL); } static int load_mqd(struct mqd_manager *mm, void *mqd, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index d28ca581cad05..888fbaf069be8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -910,7 +910,7 @@ static void kfd_process_free_id(struct kfd_process *process) { struct kfd_process *primary_process; - if (process->context_id != KFD_CONTEXT_ID_PRIMARY) + if (process->context_id == KFD_CONTEXT_ID_PRIMARY) return; primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 3841943da5ece..0b54ad8036b64 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -4115,6 +4115,7 @@ int kfd_criu_resume_svm(struct kfd_process *p) list_for_each_entry_safe(criu_svm_md, next, &svms->criu_svm_metadata_list, list) { pr_debug("freeing criu_svm_md[]\n\tstart: 0x%llx\n", criu_svm_md->data.start_addr); + list_del(&criu_svm_md->list); kfree(criu_svm_md); } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index f8c13bad4ac23..1ecd7bef9ed85 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -12553,13 +12553,11 @@ static bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev, struct drm_plane_state *new_plane_state, *old_plane_state; drm_for_each_plane_mask(plane, dev, crtc_state->plane_mask) { - new_plane_state = drm_atomic_get_plane_state(state, plane); - old_plane_state = drm_atomic_get_plane_state(state, plane); + new_plane_state = drm_atomic_get_new_plane_state(state, plane); + old_plane_state = drm_atomic_get_old_plane_state(state, plane); - if (IS_ERR(new_plane_state) || IS_ERR(old_plane_state)) { - drm_err(dev, "Failed to get plane state for plane %s\n", plane->name); - return false; - } + if (!old_plane_state || !new_plane_state) + continue; if (old_plane_state->fb && new_plane_state->fb && get_mem_type(old_plane_state->fb) != get_mem_type(new_plane_state->fb)) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index a3cb05490dc91..ee214f7592e50 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -95,8 +95,11 @@ static u32 edid_extract_panel_id(struct edid *edid) (u32)EDID_PRODUCT_ID(edid); } -static void apply_edid_quirks(struct drm_device *dev, struct edid *edid, struct dc_edid_caps *edid_caps) +static void apply_edid_quirks(struct dc_link *link, struct edid *edid, + struct dc_edid_caps *edid_caps) { + struct amdgpu_dm_connector *aconnector = link->priv; + struct drm_device *dev = aconnector->base.dev; uint32_t panel_id = edid_extract_panel_id(edid); switch (panel_id) { @@ -126,6 +129,11 @@ static void apply_edid_quirks(struct drm_device *dev, struct edid *edid, struct drm_dbg_driver(dev, "Disabling VSC on monitor with panel id %X\n", panel_id); edid_caps->panel_patch.disable_colorimetry = true; break; + /* Workaround for monitors that get corrupted by the PHY SSC reduction */ + case drm_edid_encode_panel_id('D', 'E', 'L', 0x4147): + drm_dbg_driver(dev, "Skip PHY SSC reduction on panel id %X\n", panel_id); + link->wa_flags.skip_phy_ssc_reduction = true; + break; default: return; } @@ -147,7 +155,6 @@ enum dc_edid_status dm_helpers_parse_edid_caps( { struct amdgpu_dm_connector *aconnector = link->priv; struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; struct edid *edid_buf = edid ? (struct edid *) edid->raw_edid : NULL; struct cea_sad *sads; int sad_count = -1; @@ -181,7 +188,7 @@ enum dc_edid_status dm_helpers_parse_edid_caps( if (edid_caps->edid_hdmi) populate_hdmi_info_from_connector(&connector->display_info.hdmi, edid_caps); - apply_edid_quirks(dev, edid_buf, edid_caps); + apply_edid_quirks(link, edid_buf, edid_caps); sad_count = drm_edid_to_sad((struct edid *) edid->raw_edid, &sads); if (sad_count <= 0) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index b3530fbf32f7f..5f2c62fcba9bf 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -2956,27 +2956,16 @@ static struct surface_update_descriptor det_surface_update( elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM); } - if (u->cm || (u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) { + if (u->blend_tf || (u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) { update_flags->bits.gamma_change = 1; elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM); } - if (u->cm && (u->cm->flags.bits.lut3d_enable || u->surface->cm.flags.bits.lut3d_enable)) { + if (u->lut3d_func || u->func_shaper) { update_flags->bits.lut_3d = 1; elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM); } - if (u->cm && u->cm->flags.bits.lut3d_dma_enable != u->surface->cm.flags.bits.lut3d_dma_enable && - u->cm->flags.bits.lut3d_enable && u->surface->cm.flags.bits.lut3d_enable) { - /* Toggling 3DLUT loading between DMA and Host is illegal */ - BREAK_TO_DEBUGGER(); - } - - if (u->cm && u->cm->flags.bits.lut3d_enable && !u->cm->flags.bits.lut3d_dma_enable) { - /* Host loading 3DLUT requires full update but only stream lock */ - elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM); - } - if (u->hdr_mult.value) if (u->hdr_mult.value != u->surface->hdr_mult.value) { // TODO: Should be fast? @@ -2995,6 +2984,20 @@ static struct surface_update_descriptor det_surface_update( update_flags->bits.cm_hist_change = 1; elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM); } + if (u->cm2_params) { + if (u->cm2_params->component_settings.shaper_3dlut_setting != u->surface->mcm_shaper_3dlut_setting + || u->cm2_params->component_settings.lut1d_enable != u->surface->mcm_lut1d_enable + || u->cm2_params->cm2_luts.lut3d_data.lut3d_src != u->surface->mcm_luts.lut3d_data.lut3d_src) { + update_flags->bits.mcm_transfer_function_enable_change = 1; + elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL); + } + } + + if (update_flags->bits.lut_3d && + u->surface->mcm_luts.lut3d_data.lut3d_src != DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) { + elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL); + } + if (check_config->enable_legacy_fast_update && (update_flags->bits.gamma_change || update_flags->bits.gamut_remap_change || @@ -3250,12 +3253,24 @@ static void copy_surface_update_to_plane( sizeof(struct dc_transfer_func_distributed_points)); } - /* Shaper, 3DLUT, 1DLUT */ - if (srf_update->cm) { - memcpy(&surface->cm, srf_update->cm, - sizeof(surface->cm)); + if (srf_update->cm2_params) { + surface->mcm_shaper_3dlut_setting = srf_update->cm2_params->component_settings.shaper_3dlut_setting; + surface->mcm_lut1d_enable = srf_update->cm2_params->component_settings.lut1d_enable; + surface->mcm_luts = srf_update->cm2_params->cm2_luts; } + if (srf_update->func_shaper) { + memcpy(&surface->in_shaper_func, srf_update->func_shaper, + sizeof(surface->in_shaper_func)); + + if (surface->mcm_shaper_3dlut_setting >= DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER) + surface->mcm_luts.shaper = &surface->in_shaper_func; + } + + if (srf_update->lut3d_func) + memcpy(&surface->lut3d_func, srf_update->lut3d_func, + sizeof(surface->lut3d_func)); + if (srf_update->hdr_mult.value) surface->hdr_mult = srf_update->hdr_mult; @@ -3264,6 +3279,17 @@ static void copy_surface_update_to_plane( surface->sdr_white_level_nits = srf_update->sdr_white_level_nits; + if (srf_update->blend_tf) { + memcpy(&surface->blend_tf, srf_update->blend_tf, + sizeof(surface->blend_tf)); + + if (surface->mcm_lut1d_enable) + surface->mcm_luts.lut1d_func = &surface->blend_tf; + } + + if (srf_update->cm2_params || srf_update->blend_tf) + surface->lut_bank_a = !surface->lut_bank_a; + if (srf_update->input_csc_color_matrix) surface->input_csc_color_matrix = *srf_update->input_csc_color_matrix; @@ -4550,9 +4576,11 @@ static void commit_planes_for_stream(struct dc *dc, if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state)) continue; - if (srf_updates[i].cm && - srf_updates[i].cm->flags.bits.lut3d_enable && - srf_updates[i].cm->flags.bits.lut3d_dma_enable && + if (srf_updates[i].cm2_params && + srf_updates[i].cm2_params->cm2_luts.lut3d_data.lut3d_src == + DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM && + srf_updates[i].cm2_params->component_settings.shaper_3dlut_setting == + DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT && dc->hwss.trigger_3dlut_dma_load) dc->hwss.trigger_3dlut_dma_load(dc, pipe_ctx); @@ -5209,12 +5237,6 @@ static bool full_update_required( const struct dc_stream_update *stream_update, const struct dc_stream_state *stream) { - const union dc_plane_cm_flags blend_only_flags = { - .bits = { - .blend_enable = 1, - } - }; - if (full_update_required_weak(dc, srf_updates, surface_count, stream_update, stream)) return true; @@ -5229,10 +5251,7 @@ static bool full_update_required( srf_updates[i].in_transfer_func || srf_updates[i].surface->force_full_update || (srf_updates[i].flip_addr && - srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface) || - (srf_updates[i].cm && - ((srf_updates[i].cm->flags.all != blend_only_flags.all && srf_updates[i].cm->flags.all != 0) || - (srf_updates[i].surface->cm.flags.all != blend_only_flags.all && srf_updates[i].surface->cm.flags.all != 0))))) + srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface))) return true; } @@ -6966,7 +6985,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa struct dc_plane_state *plane_state = pipe_ctx->plane_state; /* MPCC blending tree and mode control - capture actual blend configuration */ - state->mpc.mpcc_mode[i] = (plane_state->cm.blend_func.type != TF_TYPE_BYPASS) ? 1 : 0; + state->mpc.mpcc_mode[i] = (plane_state->blend_tf.type != TF_TYPE_BYPASS) ? 1 : 0; state->mpc.mpcc_alpha_blend_mode[i] = plane_state->per_pixel_alpha ? 1 : 0; state->mpc.mpcc_alpha_multiplied_mode[i] = plane_state->pre_multiplied_alpha ? 1 : 0; state->mpc.mpcc_blnd_active_overlap_only[i] = 0; /* Default - no overlap restriction */ diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index e16de323f39c0..717451828361a 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -996,6 +996,7 @@ void dc_stream_release_3dlut_for_stream( if (rmcm_3dlut) { rmcm_3dlut->isInUse = false; rmcm_3dlut->stream = NULL; + rmcm_3dlut->protection_bits = 0; } } @@ -1007,6 +1008,7 @@ void dc_stream_init_rmcm_3dlut(struct dc *dc) for (int i = 0; i < num_rmcm; i++) { dc->res_pool->rmcm_3dlut[i].isInUse = false; dc->res_pool->rmcm_3dlut[i].stream = NULL; + dc->res_pool->rmcm_3dlut[i].protection_bits = 0; } } diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 37714d4371fba..55152f12af48b 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1407,50 +1407,15 @@ struct lut_mem_mapping { struct dc_rmcm_3dlut { bool isInUse; const struct dc_stream_state *stream; + uint8_t protection_bits; }; struct dc_3dlut { struct kref refcount; struct tetrahedral_params lut_3d; + struct fixed31_32 hdr_multiplier; union dc_3dlut_state state; }; - -/* 3DLUT DMA (Fast Load) params */ -struct dc_3dlut_dma { - struct dc_plane_address addr; - enum dc_cm_lut_swizzle swizzle; - enum dc_cm_lut_pixel_format format; - uint16_t bias; /* FP1.5.10 */ - uint16_t scale; /* FP1.5.10 */ - enum dc_cm_lut_size size; -}; - -/* color manager */ -union dc_plane_cm_flags { - unsigned int all; - struct { - unsigned int shaper_enable : 1; - unsigned int lut3d_enable : 1; - unsigned int blend_enable : 1; - /* whether legacy (lut3d_func) or DMA is valid */ - unsigned int lut3d_dma_enable : 1; - /* RMCM lut to be used instead of MCM */ - unsigned int rmcm_enable : 1; - unsigned int reserved: 27; - } bits; -}; - -struct dc_plane_cm { - struct kref refcount; - struct dc_transfer_func shaper_func; - union { - struct dc_3dlut lut3d_func; - struct dc_3dlut_dma lut3d_dma; - }; - struct dc_transfer_func blend_func; - union dc_plane_cm_flags flags; -}; - /* * This structure is filled in by dc_surface_get_status and contains * the last requested address and the currently active address so the called @@ -1528,18 +1493,14 @@ struct dc_plane_state { struct fixed31_32 hdr_mult; struct colorspace_transform gamut_remap_matrix; + // TODO: No longer used, remove + struct dc_hdr_static_metadata hdr_static_ctx; + enum dc_color_space color_space; - bool lut_bank_a; - struct dc_hdr_static_metadata hdr_static_ctx; struct dc_3dlut lut3d_func; struct dc_transfer_func in_shaper_func; struct dc_transfer_func blend_tf; - enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting; - bool mcm_lut1d_enable; - struct dc_cm2_func_luts mcm_luts; - enum mpcc_movable_cm_location mcm_location; - struct dc_plane_cm cm; struct dc_transfer_func *gamcor_tf; enum surface_pixel_format format; @@ -1576,6 +1537,11 @@ struct dc_plane_state { bool is_statically_allocated; enum chroma_cositing cositing; + enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting; + bool mcm_lut1d_enable; + struct dc_cm2_func_luts mcm_luts; + bool lut_bank_a; + enum mpcc_movable_cm_location mcm_location; struct dc_csc_transform cursor_csc_color_matrix; bool adaptive_sharpness_en; int adaptive_sharpness_policy; @@ -1968,10 +1934,6 @@ struct dc_3dlut *dc_create_3dlut_func(void); void dc_3dlut_func_release(struct dc_3dlut *lut); void dc_3dlut_func_retain(struct dc_3dlut *lut); -struct dc_plane_cm *dc_plane_cm_create(void); -void dc_plane_cm_release(struct dc_plane_cm *cm); -void dc_plane_cm_retain(struct dc_plane_cm *cm); - void dc_post_update_surfaces_to_stream( struct dc *dc); diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index c08d5c005df6e..62e35dcb585cb 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -1493,28 +1493,4 @@ struct dc_validation_dpia_set { uint32_t required_bw; }; -enum dc_cm_lut_swizzle { - CM_LUT_3D_SWIZZLE_LINEAR_RGB, - CM_LUT_3D_SWIZZLE_LINEAR_BGR, - CM_LUT_1D_PACKED_LINEAR -}; - -enum dc_cm_lut_pixel_format { - CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB, - CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB, - CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB, - CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB, - CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10, - CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10 -}; - -enum dc_cm_lut_size { - CM_LUT_SIZE_NONE, - CM_LUT_SIZE_999, - CM_LUT_SIZE_171717, - CM_LUT_SIZE_333333, - CM_LUT_SIZE_454545, - CM_LUT_SIZE_656565, -}; - #endif /* DC_TYPES_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c index 5a816442deee9..3c7a6569b692f 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c @@ -41,12 +41,12 @@ hubp2->hubp_shift->field_name, hubp2->hubp_mask->field_name void hubp401_program_3dlut_fl_addr(struct hubp *hubp, - const struct dc_plane_address *address) + const struct dc_plane_address address) { struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH, HUBP_3DLUT_ADDRESS_HIGH, address->lut3d.addr.high_part); - REG_WRITE(HUBP_3DLUT_ADDRESS_LOW, address->lut3d.addr.low_part); + REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH, HUBP_3DLUT_ADDRESS_HIGH, address.lut3d.addr.high_part); + REG_WRITE(HUBP_3DLUT_ADDRESS_LOW, address.lut3d.addr.low_part); } void hubp401_program_3dlut_fl_dlg_param(struct hubp *hubp, int refcyc_per_3dlut_group) @@ -72,169 +72,96 @@ int hubp401_get_3dlut_fl_done(struct hubp *hubp) return ret; } -static void hubp401_get_3dlut_fl_xbar_map( - const enum dc_cm_lut_pixel_format format, - enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_y_g, - enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cb_b, - enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cr_r) +void hubp401_program_3dlut_fl_addressing_mode(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode) { - switch (format) { - case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB: - case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB: - case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10: - /* BGRA */ - *bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47; - *bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; - *bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_0_15; - break; - case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB: - case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB: - case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10: - default: - /* RGBA */ - *bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15; - *bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; - *bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47; - break; - } + struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); + + REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_ADDRESSING_MODE, addr_mode); } -void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp, - const enum dc_cm_lut_pixel_format format) +void hubp401_program_3dlut_fl_width(struct hubp *hubp, enum hubp_3dlut_fl_width width) { struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g = 0; - enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b = 0; - enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r = 0; + REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_WIDTH, width); +} - hubp401_get_3dlut_fl_xbar_map(format, - &bit_slice_y_g, - &bit_slice_cb_b, - &bit_slice_cr_r); +void hubp401_program_3dlut_fl_tmz_protected(struct hubp *hubp, uint8_t protection_bits) +{ + struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - REG_UPDATE_3(HUBP_3DLUT_CONTROL, - HUBP_3DLUT_CROSSBAR_SELECT_Y_G, bit_slice_y_g, - HUBP_3DLUT_CROSSBAR_SELECT_CB_B, bit_slice_cb_b, - HUBP_3DLUT_CROSSBAR_SELECT_CR_R, bit_slice_cr_r); + REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_TMZ, protection_bits); } -static enum hubp_3dlut_fl_width hubp401_get_3dlut_fl_width( - const enum dc_cm_lut_size size, - const enum dc_cm_lut_swizzle swizzle) +void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r) { - enum hubp_3dlut_fl_width width = 0; - - switch (size) { - case CM_LUT_SIZE_333333: - ASSERT(swizzle != CM_LUT_1D_PACKED_LINEAR); - width = hubp_3dlut_fl_width_33; - break; - case CM_LUT_SIZE_171717: - if (swizzle != CM_LUT_1D_PACKED_LINEAR) { - width = hubp_3dlut_fl_width_17; - } else { - width = hubp_3dlut_fl_width_17_transformed; - } - break; - default: - width = 0; - break; - } + struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - return width; + REG_UPDATE_3(HUBP_3DLUT_CONTROL, + HUBP_3DLUT_CROSSBAR_SELECT_Y_G, bit_slice_y_g, + HUBP_3DLUT_CROSSBAR_SELECT_CB_B, bit_slice_cb_b, + HUBP_3DLUT_CROSSBAR_SELECT_CR_R, bit_slice_cr_r); } -static enum hubp_3dlut_fl_format hubp401_get_3dlut_fl_format( - const enum dc_cm_lut_pixel_format dc_format) +void hubp401_update_3dlut_fl_bias_scale(struct hubp *hubp, uint16_t bias, uint16_t scale) { - enum hubp_3dlut_fl_format hubp_format = hubp_3dlut_fl_format_unorm_12msb_bitslice; - - switch (dc_format) { - case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB: - case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB: - hubp_format = hubp_3dlut_fl_format_unorm_12msb_bitslice; - break; - case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB: - case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB: - hubp_format = hubp_3dlut_fl_format_unorm_12lsb_bitslice; - break; - case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10: - case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10: - hubp_format = hubp_3dlut_fl_format_float_fp1_5_10; - break; - default: - BREAK_TO_DEBUGGER(); - break; - } + struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - return hubp_format; + REG_UPDATE_2(_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_BIAS, bias, HUBP0_3DLUT_FL_SCALE, scale); } -static enum hubp_3dlut_fl_addressing_mode hubp401_get_3dlut_fl_addr_mode( - const enum dc_cm_lut_swizzle swizzle) +void hubp401_program_3dlut_fl_mode(struct hubp *hubp, enum hubp_3dlut_fl_mode mode) { - enum hubp_3dlut_fl_addressing_mode addr_mode; - - switch (swizzle) { - case CM_LUT_1D_PACKED_LINEAR: - addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear; - break; - case CM_LUT_3D_SWIZZLE_LINEAR_RGB: - case CM_LUT_3D_SWIZZLE_LINEAR_BGR: - default: - addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; - break; - } + struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - return addr_mode; + REG_UPDATE(_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_MODE, mode); } -static enum hubp_3dlut_fl_mode hubp401_get_3dlut_fl_mode( - const enum dc_cm_lut_swizzle swizzle) +void hubp401_program_3dlut_fl_format(struct hubp *hubp, enum hubp_3dlut_fl_format format) { - enum hubp_3dlut_fl_mode mode; - - switch (swizzle) { - case CM_LUT_3D_SWIZZLE_LINEAR_RGB: - mode = hubp_3dlut_fl_mode_native_1; - break; - case CM_LUT_3D_SWIZZLE_LINEAR_BGR: - mode = hubp_3dlut_fl_mode_native_2; - break; - case CM_LUT_1D_PACKED_LINEAR: - mode = hubp_3dlut_fl_mode_transform; - break; - default: - mode = hubp_3dlut_fl_mode_disable; - break; - } + struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - return mode; + REG_UPDATE(_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_FORMAT, format); } -void hubp401_program_3dlut_fl_config(struct hubp *hubp, - const struct dc_3dlut_dma *config) +void hubp401_program_3dlut_fl_config( + struct hubp *hubp, + struct hubp_fl_3dlut_config *cfg) { struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - enum hubp_3dlut_fl_width width = hubp401_get_3dlut_fl_width(config->size, config->swizzle); - enum hubp_3dlut_fl_format format = hubp401_get_3dlut_fl_format(config->format); - enum hubp_3dlut_fl_addressing_mode addr_mode = hubp401_get_3dlut_fl_addr_mode(config->swizzle); - enum hubp_3dlut_fl_mode mode = hubp401_get_3dlut_fl_mode(config->swizzle); + uint32_t mpc_width = {(cfg->width == 17) ? 0 : 1}; + uint32_t width = {cfg->width}; + + if (cfg->layout == DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR) + width = (cfg->width == 17) ? 4916 : 35940; REG_UPDATE_2(_3DLUT_FL_CONFIG, - HUBP0_3DLUT_FL_MODE, mode, - HUBP0_3DLUT_FL_FORMAT, format); + HUBP0_3DLUT_FL_MODE, cfg->mode, + HUBP0_3DLUT_FL_FORMAT, cfg->format); REG_UPDATE_2(_3DLUT_FL_BIAS_SCALE, - HUBP0_3DLUT_FL_BIAS, config->bias, - HUBP0_3DLUT_FL_SCALE, config->scale); - - REG_UPDATE_3(HUBP_3DLUT_CONTROL, - HUBP_3DLUT_WIDTH, width, - HUBP_3DLUT_ADDRESSING_MODE, addr_mode, - HUBP_3DLUT_TMZ, config->addr.tmz_surface); + HUBP0_3DLUT_FL_BIAS, cfg->bias, + HUBP0_3DLUT_FL_SCALE, cfg->scale); + + REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH, + HUBP_3DLUT_ADDRESS_HIGH, cfg->address.lut3d.addr.high_part); + REG_UPDATE(HUBP_3DLUT_ADDRESS_LOW, + HUBP_3DLUT_ADDRESS_LOW, cfg->address.lut3d.addr.low_part); + + //cross bar + REG_UPDATE_8(HUBP_3DLUT_CONTROL, + HUBP_3DLUT_MPC_WIDTH, mpc_width, + HUBP_3DLUT_WIDTH, width, + HUBP_3DLUT_CROSSBAR_SELECT_CR_R, cfg->crossbar_bit_slice_cr_r, + HUBP_3DLUT_CROSSBAR_SELECT_Y_G, cfg->crossbar_bit_slice_y_g, + HUBP_3DLUT_CROSSBAR_SELECT_CB_B, cfg->crossbar_bit_slice_cb_b, + HUBP_3DLUT_ADDRESSING_MODE, cfg->addr_mode, + HUBP_3DLUT_TMZ, cfg->protection_bits, + HUBP_3DLUT_ENABLE, cfg->enabled ? 1 : 0); } void hubp401_update_mall_sel(struct hubp *hubp, uint32_t mall_sel, bool c_cursor) @@ -1135,13 +1062,19 @@ static struct hubp_funcs dcn401_hubp_funcs = { .hubp_update_mall_sel = hubp401_update_mall_sel, .hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering, .hubp_program_mcache_id_and_split_coordinate = hubp401_program_mcache_id_and_split_coordinate, + .hubp_update_3dlut_fl_bias_scale = hubp401_update_3dlut_fl_bias_scale, + .hubp_program_3dlut_fl_mode = hubp401_program_3dlut_fl_mode, + .hubp_program_3dlut_fl_format = hubp401_program_3dlut_fl_format, .hubp_program_3dlut_fl_addr = hubp401_program_3dlut_fl_addr, - .hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config, .hubp_program_3dlut_fl_dlg_param = hubp401_program_3dlut_fl_dlg_param, .hubp_enable_3dlut_fl = hubp401_enable_3dlut_fl, + .hubp_program_3dlut_fl_addressing_mode = hubp401_program_3dlut_fl_addressing_mode, + .hubp_program_3dlut_fl_width = hubp401_program_3dlut_fl_width, + .hubp_program_3dlut_fl_tmz_protected = hubp401_program_3dlut_fl_tmz_protected, .hubp_program_3dlut_fl_crossbar = hubp401_program_3dlut_fl_crossbar, .hubp_get_3dlut_fl_done = hubp401_get_3dlut_fl_done, .hubp_clear_tiling = hubp401_clear_tiling, + .hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config, .hubp_read_reg_state = hubp3_read_reg_state }; diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h index 043948f64b862..4570b8016de5a 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h @@ -328,17 +328,32 @@ int hubp401_get_3dlut_fl_done(struct hubp *hubp); void hubp401_set_unbounded_requesting(struct hubp *hubp, bool enable); +void hubp401_update_3dlut_fl_bias_scale(struct hubp *hubp, uint16_t bias, uint16_t scale); + void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp, - const enum dc_cm_lut_pixel_format format); + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r); + +void hubp401_program_3dlut_fl_tmz_protected(struct hubp *hubp, uint8_t protection_bits); + +void hubp401_program_3dlut_fl_width(struct hubp *hubp, enum hubp_3dlut_fl_width width); + +void hubp401_program_3dlut_fl_addressing_mode(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode); void hubp401_enable_3dlut_fl(struct hubp *hubp, bool enable); void hubp401_program_3dlut_fl_dlg_param(struct hubp *hubp, int refcyc_per_3dlut_group); -void hubp401_program_3dlut_fl_addr(struct hubp *hubp, const struct dc_plane_address *address); +void hubp401_program_3dlut_fl_addr(struct hubp *hubp, const struct dc_plane_address address); -void hubp401_program_3dlut_fl_config(struct hubp *hubp, - const struct dc_3dlut_dma *config); +void hubp401_program_3dlut_fl_format(struct hubp *hubp, enum hubp_3dlut_fl_format format); + +void hubp401_program_3dlut_fl_mode(struct hubp *hubp, enum hubp_3dlut_fl_mode mode); + +void hubp401_program_3dlut_fl_config( + struct hubp *hubp, + struct hubp_fl_3dlut_config *cfg); void hubp401_clear_tiling(struct hubp *hubp); diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c index ad6badcceb12a..e4602c3ddc662 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c @@ -311,84 +311,19 @@ static void hubp42_program_surface_config( hubp42_program_pixel_format(hubp, format); } -static void hubp42_get_3dlut_fl_xbar_map( - const enum dc_cm_lut_pixel_format format, - enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_y_g, - enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cb_b, - enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cr_r) -{ - switch (format) { - case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB: - case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB: - case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10: - /* BGRA */ - *bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47; - *bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; - *bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_0_15; - break; - case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB: - case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB: - case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10: - default: - /* RGBA */ - *bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15; - *bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; - *bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47; - break; - } -} - void hubp42_program_3dlut_fl_crossbar(struct hubp *hubp, - const enum dc_cm_lut_pixel_format format) + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_r, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_g, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_b) { struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_g = 0; - enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_b = 0; - enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_r = 0; - - hubp42_get_3dlut_fl_xbar_map(format, - &bit_slice_g, - &bit_slice_b, - &bit_slice_r); - REG_UPDATE_3(HUBP_3DLUT_CONTROL, HUBP_3DLUT_CROSSBAR_SEL_R, bit_slice_r, HUBP_3DLUT_CROSSBAR_SEL_G, bit_slice_g, HUBP_3DLUT_CROSSBAR_SEL_B, bit_slice_b); } -static uint32_t hubp42_get_3dlut_fl_mpc_width( - const enum dc_cm_lut_size size) -{ - uint32_t width = 0; - - switch (size) { - case CM_LUT_SIZE_333333: - width = 1; - break; - case CM_LUT_SIZE_171717: - default: - width = 0; - break; - } - - return width; -} - -void hubp42_program_3dlut_fl_config(struct hubp *hubp, - const struct dc_3dlut_dma *config) -{ - struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); - - uint32_t mpc_width = hubp42_get_3dlut_fl_mpc_width(config->size); - - REG_UPDATE(HUBP_3DLUT_CONTROL, - HUBP_3DLUT_MPC_WIDTH, mpc_width); - - hubp401_program_3dlut_fl_config(hubp, config); -} - static bool hubp42_program_surface_flip_and_addr( struct hubp *hubp, const struct dc_plane_address *address, @@ -670,10 +605,15 @@ struct hubp_funcs dcn42_hubp_funcs = { .hubp_set_flip_int = hubp1_set_flip_int, .hubp_in_blank = hubp1_in_blank, .program_extended_blank = hubp31_program_extended_blank_value, + .hubp_update_3dlut_fl_bias_scale = hubp401_update_3dlut_fl_bias_scale, + .hubp_program_3dlut_fl_mode = hubp401_program_3dlut_fl_mode, + .hubp_program_3dlut_fl_format = hubp401_program_3dlut_fl_format, .hubp_program_3dlut_fl_addr = hubp401_program_3dlut_fl_addr, - .hubp_program_3dlut_fl_config = hubp42_program_3dlut_fl_config, .hubp_program_3dlut_fl_dlg_param = hubp401_program_3dlut_fl_dlg_param, .hubp_enable_3dlut_fl = hubp401_enable_3dlut_fl, + .hubp_program_3dlut_fl_addressing_mode = hubp401_program_3dlut_fl_addressing_mode, + .hubp_program_3dlut_fl_width = hubp401_program_3dlut_fl_width, + .hubp_program_3dlut_fl_tmz_protected = hubp401_program_3dlut_fl_tmz_protected, .hubp_program_3dlut_fl_crossbar = hubp42_program_3dlut_fl_crossbar, .hubp_get_3dlut_fl_done = hubp401_get_3dlut_fl_done, .hubp_clear_tiling = hubp3_clear_tiling, diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h index 88bb1337ab9d6..f0e614136228c 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h @@ -58,11 +58,11 @@ bool hubp42_construct( const struct dcn_hubp2_shift *hubp_shift, const struct dcn_hubp2_mask *hubp_mask); -void hubp42_program_3dlut_fl_crossbar(struct hubp *hubp, - const enum dc_cm_lut_pixel_format format); - -void hubp42_program_3dlut_fl_config(struct hubp *hubp, - const struct dc_3dlut_dma *config); +void hubp42_program_3dlut_fl_crossbar( + struct hubp *hubp, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_r, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_g, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_b); void hubp42_read_state(struct hubp *hubp); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c index 9768b8f6b6fb6..98e78aa2259cc 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c @@ -95,6 +95,10 @@ void dcn401_program_gamut_remap(struct pipe_ctx *pipe_ctx) unsigned int mpcc_id = pipe_ctx->plane_res.mpcc_inst; struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc; + //For now assert if location is not pre-blend + if (pipe_ctx->plane_state) + ASSERT(pipe_ctx->plane_state->mcm_location == MPCC_MOVABLE_CM_LOCATION_BEFORE); + // program MPCC_MCM_FIRST_GAMUT_REMAP memset(&mpc_adjust, 0, sizeof(mpc_adjust)); mpc_adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_BYPASS; @@ -374,180 +378,300 @@ void dcn401_init_hw(struct dc *dc) } } -void dcn401_trigger_3dlut_dma_load(struct dc *dc, struct pipe_ctx *pipe_ctx) +static void dcn401_get_mcm_lut_xable_from_pipe_ctx(struct dc *dc, struct pipe_ctx *pipe_ctx, + enum MCM_LUT_XABLE *shaper_xable, + enum MCM_LUT_XABLE *lut3d_xable, + enum MCM_LUT_XABLE *lut1d_xable) { - (void)dc; - struct hubp *hubp = pipe_ctx->plane_res.hubp; + enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL; + bool lut1d_enable = false; + struct mpc *mpc = dc->res_pool->mpc; + int mpcc_id = pipe_ctx->plane_res.hubp->inst; - if (hubp->funcs->hubp_enable_3dlut_fl) { - hubp->funcs->hubp_enable_3dlut_fl(hubp, true); + if (!pipe_ctx->plane_state) + return; + shaper_3dlut_setting = pipe_ctx->plane_state->mcm_shaper_3dlut_setting; + lut1d_enable = pipe_ctx->plane_state->mcm_lut1d_enable; + mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id); + pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE; + + *lut1d_xable = lut1d_enable ? MCM_LUT_ENABLE : MCM_LUT_DISABLE; + + switch (shaper_3dlut_setting) { + case DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL: + *lut3d_xable = *shaper_xable = MCM_LUT_DISABLE; + break; + case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER: + *lut3d_xable = MCM_LUT_DISABLE; + *shaper_xable = MCM_LUT_ENABLE; + break; + case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT: + *lut3d_xable = *shaper_xable = MCM_LUT_ENABLE; + break; } } -bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx, - const struct dc_plane_state *plane_state) +void dcn401_populate_mcm_luts(struct dc *dc, + struct pipe_ctx *pipe_ctx, + struct dc_cm2_func_luts mcm_luts, + bool lut_bank_a) { - struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc; struct dpp *dpp_base = pipe_ctx->plane_res.dpp; struct hubp *hubp = pipe_ctx->plane_res.hubp; - const struct dc_plane_cm *cm = &plane_state->cm; int mpcc_id = hubp->inst; struct mpc *mpc = dc->res_pool->mpc; union mcm_lut_params m_lut_params; - struct dc_3dlut_dma lut3d_dma; - bool lut_enable; - bool lut_bank_a; + enum dc_cm2_transfer_func_source lut3d_src = mcm_luts.lut3d_data.lut3d_src; + enum hubp_3dlut_fl_format format = 0; + enum hubp_3dlut_fl_mode mode; + enum hubp_3dlut_fl_width width = 0; + enum hubp_3dlut_fl_addressing_mode addr_mode; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = 0; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = 0; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = 0; + enum MCM_LUT_XABLE shaper_xable = MCM_LUT_DISABLE; + enum MCM_LUT_XABLE lut3d_xable = MCM_LUT_DISABLE; + enum MCM_LUT_XABLE lut1d_xable = MCM_LUT_DISABLE; bool rval; - bool result = true; - /* decide LUT bank based on current in use */ - mpc->funcs->get_lut_mode(mpc, MCM_LUT_1DLUT, mpcc_id, &lut_enable, &lut_bank_a); - if (!lut_enable) { - mpc->funcs->get_lut_mode(mpc, MCM_LUT_SHAPER, mpcc_id, &lut_enable, &lut_bank_a); - } - if (!lut_enable) { - mpc->funcs->get_lut_mode(mpc, MCM_LUT_3DLUT, mpcc_id, &lut_enable, &lut_bank_a); - } - - /* switch to the next bank */ - if (lut_enable) { - lut_bank_a = !lut_bank_a; - } - - /* MCM location fixed to pre-blend */ - mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id); + dcn401_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable); /* 1D LUT */ - lut_enable = cm->flags.bits.blend_enable; - memset(&m_lut_params, 0, sizeof(m_lut_params)); - if (lut_enable) { - if (cm->blend_func.type == TF_TYPE_HWPWL) - m_lut_params.pwl = &cm->blend_func.pwl; - else if (cm->blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) { - rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, - &cm->blend_func, - &dpp_base->regamma_params, - false); + if (mcm_luts.lut1d_func) { + memset(&m_lut_params, 0, sizeof(m_lut_params)); + if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL) + m_lut_params.pwl = &mcm_luts.lut1d_func->pwl; + else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) { + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, + mcm_luts.lut1d_func, + &dpp_base->regamma_params, false); m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; } - - if (!m_lut_params.pwl) { - lut_enable = false; + if (m_lut_params.pwl) { + if (mpc->funcs->populate_lut) + mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, m_lut_params, lut_bank_a, mpcc_id); } - } else { - lut_enable = false; + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut1d_xable && m_lut_params.pwl, lut_bank_a, mpcc_id); } - if (mpc->funcs->program_lut_mode) - mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut_enable, lut_bank_a, CM_LUT_SIZE_NONE, mpcc_id); - if (lut_enable && mpc->funcs->populate_lut) - mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, &m_lut_params, lut_bank_a, mpcc_id); - /* Shaper */ - lut_enable = cm->flags.bits.shaper_enable; - if (lut_enable) { + if (mcm_luts.shaper && mcm_luts.lut3d_data.mpc_3dlut_enable) { memset(&m_lut_params, 0, sizeof(m_lut_params)); - if (cm->shaper_func.type == TF_TYPE_HWPWL) - m_lut_params.pwl = &cm->shaper_func.pwl; - else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { + if (mcm_luts.shaper->type == TF_TYPE_HWPWL) + m_lut_params.pwl = &mcm_luts.shaper->pwl; + else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) { ASSERT(false); - rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, - &cm->shaper_func, - &dpp_base->shaper_params, - true); - m_lut_params.pwl = rval ? &dpp_base->shaper_params : NULL; + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, + mcm_luts.shaper, + &dpp_base->regamma_params, true); + m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; } - if (!m_lut_params.pwl) { - lut_enable = false; + if (m_lut_params.pwl) { + if (mpc->funcs->mcm.populate_lut) + mpc->funcs->mcm.populate_lut(mpc, m_lut_params, lut_bank_a, mpcc_id); + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_ENABLE, lut_bank_a, mpcc_id); } - } else { - lut_enable = false; } - if (mpc->funcs->program_lut_mode) - mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, lut_enable, lut_bank_a, CM_LUT_SIZE_NONE, mpcc_id); - if (lut_enable && mpc->funcs->populate_lut) - mpc->funcs->populate_lut(mpc, MCM_LUT_SHAPER, &m_lut_params, lut_bank_a, mpcc_id); + /* 3DLUT */ + switch (lut3d_src) { + case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM: + memset(&m_lut_params, 0, sizeof(m_lut_params)); + if (hubp->funcs->hubp_enable_3dlut_fl) + hubp->funcs->hubp_enable_3dlut_fl(hubp, false); - /* NOTE: Toggling from DMA->Host is not supported atomically as hardware - * blocks writes until 3DLUT FL mode is cleared from HUBP on VUpdate. - * Expectation is either option is used consistently. - */ + if (mcm_luts.lut3d_data.lut3d_func && mcm_luts.lut3d_data.lut3d_func->state.bits.initialized) { + m_lut_params.lut3d = &mcm_luts.lut3d_data.lut3d_func->lut_3d; + if (mpc->funcs->populate_lut) + mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, m_lut_params, lut_bank_a, mpcc_id); + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, + mpcc_id); + } + break; + case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM: + switch (mcm_luts.lut3d_data.gpu_mem_params.size) { +#if defined(CONFIG_DRM_AMD_DC_DCN4_2) + case DC_CM2_GPU_MEM_SIZE_333333: + if (dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33) + width = hubp_3dlut_fl_width_33; + break; +#endif + case DC_CM2_GPU_MEM_SIZE_171717: + width = hubp_3dlut_fl_width_17; + break; + case DC_CM2_GPU_MEM_SIZE_TRANSFORMED: + width = hubp_3dlut_fl_width_transformed; + break; + default: + //TODO: handle default case + break; + } - /* 3DLUT */ - lut_enable = cm->flags.bits.lut3d_enable; - if (lut_enable && cm->flags.bits.lut3d_dma_enable) { - /* Fast (DMA) Load Mode */ - /* MPC */ - if (mpc->funcs->program_lut_mode) - mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut_enable, lut_bank_a, cm->lut3d_dma.size, mpcc_id); + //check for support + if (mpc->funcs->mcm.is_config_supported && + !mpc->funcs->mcm.is_config_supported(width)) + break; - /* only supports 12 bit */ if (mpc->funcs->program_lut_read_write_control) - mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, 12, mpcc_id); + mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, mpcc_id); + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, mpcc_id); - if (mpc->funcs->update_3dlut_fast_load_select) - mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst); + if (hubp->funcs->hubp_program_3dlut_fl_addr) + hubp->funcs->hubp_program_3dlut_fl_addr(hubp, mcm_luts.lut3d_data.gpu_mem_params.addr); - /* HUBP */ - if (hubp->funcs->hubp_program_3dlut_fl_config) - hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma); + if (mpc->funcs->mcm.program_bit_depth) + mpc->funcs->mcm.program_bit_depth(mpc, mcm_luts.lut3d_data.gpu_mem_params.bit_depth, mpcc_id); - if (hubp->funcs->hubp_program_3dlut_fl_crossbar) - hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp, cm->lut3d_dma.format); + switch (mcm_luts.lut3d_data.gpu_mem_params.layout) { + case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB: + mode = hubp_3dlut_fl_mode_native_1; + addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR: + mode = hubp_3dlut_fl_mode_native_2; + addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR: + mode = hubp_3dlut_fl_mode_transform; + addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear; + break; + default: + mode = hubp_3dlut_fl_mode_disable; + addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + } + if (hubp->funcs->hubp_program_3dlut_fl_mode) + hubp->funcs->hubp_program_3dlut_fl_mode(hubp, mode); - if (hubp->funcs->hubp_program_3dlut_fl_addr) - hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr); + if (hubp->funcs->hubp_program_3dlut_fl_addressing_mode) + hubp->funcs->hubp_program_3dlut_fl_addressing_mode(hubp, addr_mode); - if (hubp->funcs->hubp_enable_3dlut_fl) { - hubp->funcs->hubp_enable_3dlut_fl(hubp, true); - } else { - /* GPU memory only supports fast load path */ - BREAK_TO_DEBUGGER(); - lut_enable = false; - result = false; + switch (mcm_luts.lut3d_data.gpu_mem_params.format_params.format) { + case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB: + format = hubp_3dlut_fl_format_unorm_12msb_bitslice; + break; + case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB: + format = hubp_3dlut_fl_format_unorm_12lsb_bitslice; + break; + case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10: + format = hubp_3dlut_fl_format_float_fp1_5_10; + break; + } + if (hubp->funcs->hubp_program_3dlut_fl_format) + hubp->funcs->hubp_program_3dlut_fl_format(hubp, format); + if (hubp->funcs->hubp_update_3dlut_fl_bias_scale && + mpc->funcs->mcm.program_bias_scale) { + mpc->funcs->mcm.program_bias_scale(mpc, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale, + mpcc_id); + hubp->funcs->hubp_update_3dlut_fl_bias_scale(hubp, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale); } - } else { - /* Legacy (Host) Load Mode */ - memset(&m_lut_params, 0, sizeof(m_lut_params)); - if (cm->flags.bits.lut3d_enable && cm->lut3d_func.state.bits.initialized) { - m_lut_params.lut3d = &cm->lut3d_func.lut_3d; - } else { - lut_enable = false; + //navi 4x has a bug and r and blue are swapped and need to be worked around here in + //TODO: need to make a method for get_xbar per asic OR do the workaround in program_crossbar for 4x + switch (mcm_luts.lut3d_data.gpu_mem_params.component_order) { + case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA: + default: + crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15; + crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; + crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47; + break; } - /* MPC */ - if (mpc->funcs->program_lut_mode) - mpc->funcs->program_lut_mode(mpc, - MCM_LUT_3DLUT, - lut_enable, - lut_bank_a, - cm->lut3d_func.lut_3d.use_tetrahedral_9 ? CM_LUT_SIZE_999 : CM_LUT_SIZE_171717, - mpcc_id); - - if (lut_enable) { - if (mpc->funcs->program_lut_read_write_control) - mpc->funcs->program_lut_read_write_control(mpc, - MCM_LUT_3DLUT, - lut_bank_a, - cm->lut3d_func.lut_3d.use_12bits ? 12 : 10, - mpcc_id); + if (hubp->funcs->hubp_program_3dlut_fl_crossbar) + hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp, + crossbar_bit_slice_cr_r, + crossbar_bit_slice_y_g, + crossbar_bit_slice_cb_b); - if (mpc->funcs->update_3dlut_fast_load_select) - mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, 0xf); + if (mpc->funcs->mcm.program_lut_read_write_control) + mpc->funcs->mcm.program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, true, mpcc_id); - if (mpc->funcs->populate_lut) - mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, &m_lut_params, lut_bank_a, mpcc_id); - } + if (mpc->funcs->mcm.program_3dlut_size) + mpc->funcs->mcm.program_3dlut_size(mpc, width, mpcc_id); - /* HUBP */ - memset(&lut3d_dma, 0, sizeof(lut3d_dma)); - if (hubp->funcs->hubp_program_3dlut_fl_config) - hubp->funcs->hubp_program_3dlut_fl_config(hubp, &lut3d_dma); + if (mpc->funcs->update_3dlut_fast_load_select) + mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst); if (hubp->funcs->hubp_enable_3dlut_fl) - hubp->funcs->hubp_enable_3dlut_fl(hubp, false); + hubp->funcs->hubp_enable_3dlut_fl(hubp, true); + else { + if (mpc->funcs->program_lut_mode) { + mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_DISABLE, lut_bank_a, mpcc_id); + mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id); + mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id); + } + } + break; + + } +} + +void dcn401_trigger_3dlut_dma_load(struct dc *dc, struct pipe_ctx *pipe_ctx) +{ + (void)dc; + struct hubp *hubp = pipe_ctx->plane_res.hubp; + + if (hubp->funcs->hubp_enable_3dlut_fl) { + hubp->funcs->hubp_enable_3dlut_fl(hubp, true); + } +} + +bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx, + const struct dc_plane_state *plane_state) +{ + struct dpp *dpp_base = pipe_ctx->plane_res.dpp; + int mpcc_id = pipe_ctx->plane_res.hubp->inst; + struct dc *dc = pipe_ctx->stream_res.opp->ctx->dc; + struct mpc *mpc = dc->res_pool->mpc; + bool result; + const struct pwl_params *lut_params = NULL; + bool rval; + + if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) { + dcn401_populate_mcm_luts(dc, pipe_ctx, plane_state->mcm_luts, plane_state->lut_bank_a); + return true; + } + + mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id); + pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE; + // 1D LUT + if (plane_state->blend_tf.type == TF_TYPE_HWPWL) + lut_params = &plane_state->blend_tf.pwl; + else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->blend_tf, + &dpp_base->regamma_params, false); + lut_params = rval ? &dpp_base->regamma_params : NULL; + } + result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id); + lut_params = NULL; + + // Shaper + if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL) + lut_params = &plane_state->in_shaper_func.pwl; + else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { + // TODO: dpp_base replace + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_shaper_func, + &dpp_base->shaper_params, true); + lut_params = rval ? &dpp_base->shaper_params : NULL; + } + result &= mpc->funcs->program_shaper(mpc, lut_params, mpcc_id); + + // 3D + if (mpc->funcs->program_3dlut) { + if (plane_state->lut3d_func.state.bits.initialized == 1) + result &= mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id); + else + result &= mpc->funcs->program_3dlut(mpc, NULL, mpcc_id); } return result; @@ -1824,9 +1948,10 @@ void dcn401_perform_3dlut_wa_unlock(struct pipe_ctx *pipe_ctx) for (odm_pipe = pipe_ctx; odm_pipe != NULL; odm_pipe = odm_pipe->next_odm_pipe) { for (mpc_pipe = odm_pipe; mpc_pipe != NULL; mpc_pipe = mpc_pipe->bottom_pipe) { - if (mpc_pipe->plane_state && - mpc_pipe->plane_state->cm.flags.bits.lut3d_enable && - mpc_pipe->plane_state->cm.flags.bits.lut3d_dma_enable) { + if (mpc_pipe->plane_state && mpc_pipe->plane_state->mcm_luts.lut3d_data.lut3d_src + == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM + && mpc_pipe->plane_state->mcm_shaper_3dlut_setting + == DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT) { wa_pipes[wa_pipe_ct++] = mpc_pipe; } } diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c index 46f2f9833d9e5..53ff5a02af023 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c @@ -394,6 +394,68 @@ void dcn42_program_cm_hist( plane_state->cm_hist_control, plane_state->color_space); } +static void dc_get_lut_xbar( + enum dc_cm2_gpu_mem_pixel_component_order order, + enum hubp_3dlut_fl_crossbar_bit_slice *cr_r, + enum hubp_3dlut_fl_crossbar_bit_slice *y_g, + enum hubp_3dlut_fl_crossbar_bit_slice *cb_b) +{ + switch (order) { + case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA: + *cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47; + *y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; + *cb_b = hubp_3dlut_fl_crossbar_bit_slice_0_15; + break; + case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_BGRA: + *cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15; + *y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; + *cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47; + break; + } +} + +static void dc_get_lut_mode( + enum dc_cm2_gpu_mem_layout layout, + enum hubp_3dlut_fl_mode *mode, + enum hubp_3dlut_fl_addressing_mode *addr_mode) +{ + switch (layout) { + case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB: + *mode = hubp_3dlut_fl_mode_native_1; + *addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR: + *mode = hubp_3dlut_fl_mode_native_2; + *addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR: + *mode = hubp_3dlut_fl_mode_transform; + *addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear; + break; + default: + *mode = hubp_3dlut_fl_mode_disable; + *addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + } +} + +static void dc_get_lut_format( + enum dc_cm2_gpu_mem_format dc_format, + enum hubp_3dlut_fl_format *format) +{ + switch (dc_format) { + case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB: + *format = hubp_3dlut_fl_format_unorm_12msb_bitslice; + break; + case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB: + *format = hubp_3dlut_fl_format_unorm_12lsb_bitslice; + break; + case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10: + *format = hubp_3dlut_fl_format_float_fp1_5_10; + break; + } +} + static bool dc_is_rmcm_3dlut_supported(struct hubp *hubp, struct mpc *mpc) { if (mpc->funcs->rmcm.power_on_shaper_3dlut && @@ -404,17 +466,119 @@ static bool dc_is_rmcm_3dlut_supported(struct hubp *hubp, struct mpc *mpc) return false; } +static bool is_rmcm_3dlut_fl_supported(struct dc *dc, enum dc_cm2_gpu_mem_size size) +{ + if (!dc->caps.color.mpc.rmcm_3d_lut_caps.dma_3d_lut) + return false; + if (size == DC_CM2_GPU_MEM_SIZE_171717) + return (dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17); + else if (size == DC_CM2_GPU_MEM_SIZE_333333) + return (dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33); + return false; +} + +static void dcn42_set_mcm_location_post_blend(struct dc *dc, struct pipe_ctx *pipe_ctx, bool bPostBlend) +{ + struct mpc *mpc = dc->res_pool->mpc; + int mpcc_id = pipe_ctx->plane_res.hubp->inst; + + if (!pipe_ctx->plane_state) + return; + + mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id); + pipe_ctx->plane_state->mcm_location = (bPostBlend) ? + MPCC_MOVABLE_CM_LOCATION_AFTER : + MPCC_MOVABLE_CM_LOCATION_BEFORE; +} + +static void dcn42_get_mcm_lut_xable_from_pipe_ctx(struct dc *dc, struct pipe_ctx *pipe_ctx, + enum MCM_LUT_XABLE *shaper_xable, + enum MCM_LUT_XABLE *lut3d_xable, + enum MCM_LUT_XABLE *lut1d_xable) +{ + enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL; + bool lut1d_enable = false; + struct mpc *mpc = dc->res_pool->mpc; + int mpcc_id = pipe_ctx->plane_res.hubp->inst; + + if (!pipe_ctx->plane_state) + return; + shaper_3dlut_setting = pipe_ctx->plane_state->mcm_shaper_3dlut_setting; + lut1d_enable = pipe_ctx->plane_state->mcm_lut1d_enable; + mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id); + pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE; + + *lut1d_xable = lut1d_enable ? MCM_LUT_ENABLE : MCM_LUT_DISABLE; + + switch (shaper_3dlut_setting) { + case DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL: + *lut3d_xable = *shaper_xable = MCM_LUT_DISABLE; + break; + case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER: + *lut3d_xable = MCM_LUT_DISABLE; + *shaper_xable = MCM_LUT_ENABLE; + break; + case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT: + *lut3d_xable = *shaper_xable = MCM_LUT_ENABLE; + break; + } +} + +static void fl_get_lut_mode( + enum dc_cm2_gpu_mem_layout layout, + enum dc_cm2_gpu_mem_size size, + enum hubp_3dlut_fl_mode *mode, + enum hubp_3dlut_fl_addressing_mode *addr_mode, + enum hubp_3dlut_fl_width *width) +{ + *width = hubp_3dlut_fl_width_17; + + if (size == DC_CM2_GPU_MEM_SIZE_333333) + *width = hubp_3dlut_fl_width_33; + + switch (layout) { + case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB: + *mode = hubp_3dlut_fl_mode_native_1; + *addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR: + *mode = hubp_3dlut_fl_mode_native_2; + *addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR: + *mode = hubp_3dlut_fl_mode_transform; + *addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear; + break; + default: + *mode = hubp_3dlut_fl_mode_disable; + *addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear; + break; + } +} + bool dcn42_program_rmcm_luts( struct hubp *hubp, struct pipe_ctx *pipe_ctx, - const struct dc_plane_cm *cm, + enum dc_cm2_transfer_func_source lut3d_src, + struct dc_cm2_func_luts *mcm_luts, struct mpc *mpc, + bool lut_bank_a, int mpcc_id) { struct dpp *dpp_base = pipe_ctx->plane_res.dpp; union mcm_lut_params m_lut_params = {0}; + enum MCM_LUT_XABLE shaper_xable, lut3d_xable = MCM_LUT_DISABLE, lut1d_xable; + enum hubp_3dlut_fl_mode mode; + enum hubp_3dlut_fl_addressing_mode addr_mode; + enum hubp_3dlut_fl_format format = hubp_3dlut_fl_format_unorm_12msb_bitslice; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_0_15; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47; + enum hubp_3dlut_fl_width width = hubp_3dlut_fl_width_17; + struct dc *dc = hubp->ctx->dc; + struct hubp_fl_3dlut_config fl_config; struct mpc_fl_3dlut_config mpc_fl_config; struct dc_stream_state *stream = pipe_ctx->stream; @@ -422,23 +586,25 @@ bool dcn42_program_rmcm_luts( // true->false when it can be allocated at DI time struct dc_rmcm_3dlut *rmcm_3dlut = dc_stream_get_3dlut_for_stream(dc, stream, false); - bool lut_bank_a = true; // TODO get from HW - //check to see current pipe is part of a stream with allocated rmcm 3dlut if (!rmcm_3dlut) return false; + rmcm_3dlut->protection_bits = mcm_luts->lut3d_data.rmcm_tmz; + + dcn42_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable); + /* Shaper */ - if (cm->flags.bits.shaper_enable) { + if (mcm_luts->shaper) { memset(&m_lut_params, 0, sizeof(m_lut_params)); - if (cm->shaper_func.type == TF_TYPE_HWPWL) { - m_lut_params.pwl = &cm->shaper_func.pwl; - } else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { + if (mcm_luts->shaper->type == TF_TYPE_HWPWL) { + m_lut_params.pwl = &mcm_luts->shaper->pwl; + } else if (mcm_luts->shaper->type == TF_TYPE_DISTRIBUTED_POINTS) { ASSERT(false); cm_helper_translate_curve_to_hw_format( dc->ctx, - &cm->shaper_func, + mcm_luts->shaper, &dpp_base->shaper_params, true); m_lut_params.pwl = &dpp_base->shaper_params; } @@ -454,21 +620,58 @@ bool dcn42_program_rmcm_luts( } /* 3DLUT */ - if (!cm->flags.bits.lut3d_dma_enable) { + switch (lut3d_src) { + case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM: memset(&m_lut_params, 0, sizeof(m_lut_params)); // Don't know what to do in this case. - } else { - if (!dc_is_rmcm_3dlut_supported(hubp, mpc)) + //case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM: + break; + case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM: + fl_get_lut_mode(mcm_luts->lut3d_data.gpu_mem_params.layout, + mcm_luts->lut3d_data.gpu_mem_params.size, + &mode, + &addr_mode, + &width); + + if (!dc_is_rmcm_3dlut_supported(hubp, mpc) || + !mpc->funcs->rmcm.is_config_supported( + (width == hubp_3dlut_fl_width_17 || + width == hubp_3dlut_fl_width_transformed) ? 17 : 33)) return false; + // setting native or transformed mode, + dc_get_lut_mode(mcm_luts->lut3d_data.gpu_mem_params.layout, &mode, &addr_mode); + //seems to be only for the MCM - mpc_fl_config.enabled = cm->flags.bits.lut3d_enable; - mpc_fl_config.size = cm->lut3d_dma.size; + dc_get_lut_format(mcm_luts->lut3d_data.gpu_mem_params.format_params.format, &format); + + dc_get_lut_xbar( + mcm_luts->lut3d_data.gpu_mem_params.component_order, + &crossbar_bit_slice_cr_r, + &crossbar_bit_slice_y_g, + &crossbar_bit_slice_cb_b); + + fl_config.mode = mode; + fl_config.enabled = lut3d_xable != MCM_LUT_DISABLE; + fl_config.address = mcm_luts->lut3d_data.gpu_mem_params.addr; + fl_config.format = format; + fl_config.crossbar_bit_slice_y_g = crossbar_bit_slice_y_g; + fl_config.crossbar_bit_slice_cb_b = crossbar_bit_slice_cb_b; + fl_config.crossbar_bit_slice_cr_r = crossbar_bit_slice_cr_r; + fl_config.width = width; + fl_config.protection_bits = rmcm_3dlut->protection_bits; + fl_config.addr_mode = addr_mode; + fl_config.layout = mcm_luts->lut3d_data.gpu_mem_params.layout; + fl_config.bias = mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.bias; + fl_config.scale = mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.scale; + + mpc_fl_config.enabled = fl_config.enabled; + mpc_fl_config.width = width; mpc_fl_config.select_lut_bank_a = lut_bank_a; - mpc_fl_config.bit_depth = 0; + mpc_fl_config.bit_depth = mcm_luts->lut3d_data.gpu_mem_params.bit_depth; mpc_fl_config.hubp_index = hubp->inst; - mpc_fl_config.bias = cm->lut3d_dma.bias; - mpc_fl_config.scale = cm->lut3d_dma.scale; + mpc_fl_config.bias = mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.bias; + mpc_fl_config.scale = mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.scale; //1. power down the block mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, false); @@ -476,44 +679,268 @@ bool dcn42_program_rmcm_luts( //2. program RMCM - 3dlut reg programming mpc->funcs->rmcm.fl_3dlut_configure(mpc, &mpc_fl_config, mpcc_id); - /* HUBP */ - if (hubp->funcs->hubp_program_3dlut_fl_config) - hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma); - - if (hubp->funcs->hubp_program_3dlut_fl_addr) - hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr); + hubp->funcs->hubp_program_3dlut_fl_config(hubp, &fl_config); //3. power on the block mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, true); + + break; + default: + return false; } return true; } -bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx, - const struct dc_plane_state *plane_state) +void dcn42_populate_mcm_luts(struct dc *dc, + struct pipe_ctx *pipe_ctx, + struct dc_cm2_func_luts mcm_luts, + bool lut_bank_a) { - struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc; + struct dpp *dpp_base = pipe_ctx->plane_res.dpp; struct hubp *hubp = pipe_ctx->plane_res.hubp; - const struct dc_plane_cm *cm = &plane_state->cm; - struct mpc *mpc = dc->res_pool->mpc; int mpcc_id = hubp->inst; - bool result; - - /* MCM */ - result = dcn401_set_mcm_luts(pipe_ctx, plane_state); - - /* RMCM */ - if (cm->flags.bits.rmcm_enable && cm->flags.bits.lut3d_dma_enable) { - /* TODO - move RMCM to its own block */ + struct mpc *mpc = dc->res_pool->mpc; + union mcm_lut_params m_lut_params; + enum dc_cm2_transfer_func_source lut3d_src = mcm_luts.lut3d_data.lut3d_src; + enum hubp_3dlut_fl_format format = 0; + enum hubp_3dlut_fl_mode mode; + enum hubp_3dlut_fl_width width = 0; + enum hubp_3dlut_fl_addressing_mode addr_mode; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = 0; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = 0; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = 0; + enum MCM_LUT_XABLE shaper_xable = MCM_LUT_DISABLE; + enum MCM_LUT_XABLE lut3d_xable = MCM_LUT_DISABLE; + enum MCM_LUT_XABLE lut1d_xable = MCM_LUT_DISABLE; + bool rval; + + dcn42_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable); + + //MCM - setting its location (Before/After) blender + //set to post blend (true) + dcn42_set_mcm_location_post_blend( + dc, + pipe_ctx, + mcm_luts.lut3d_data.mpc_mcm_post_blend); + + //RMCM - 3dLUT+Shaper + if (mcm_luts.lut3d_data.rmcm_3dlut_enable && + is_rmcm_3dlut_fl_supported(dc, mcm_luts.lut3d_data.gpu_mem_params.size)) { dcn42_program_rmcm_luts( hubp, pipe_ctx, - cm, + lut3d_src, + &mcm_luts, mpc, + lut_bank_a, mpcc_id); } + /* 1D LUT */ + if (mcm_luts.lut1d_func) { + memset(&m_lut_params, 0, sizeof(m_lut_params)); + if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL) + m_lut_params.pwl = &mcm_luts.lut1d_func->pwl; + else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) { + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, + mcm_luts.lut1d_func, + &dpp_base->regamma_params, false); + m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; + } + if (m_lut_params.pwl) { + if (mpc->funcs->populate_lut) + mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, m_lut_params, lut_bank_a, mpcc_id); + } + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut1d_xable && m_lut_params.pwl, lut_bank_a, mpcc_id); + } + + /* Shaper */ + if (mcm_luts.shaper && mcm_luts.lut3d_data.mpc_3dlut_enable) { + memset(&m_lut_params, 0, sizeof(m_lut_params)); + if (mcm_luts.shaper->type == TF_TYPE_HWPWL) + m_lut_params.pwl = &mcm_luts.shaper->pwl; + else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) { + ASSERT(false); + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, + mcm_luts.shaper, + &dpp_base->regamma_params, true); + m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; + } + if (m_lut_params.pwl) { + if (mpc->funcs->mcm.populate_lut) + mpc->funcs->mcm.populate_lut(mpc, m_lut_params, lut_bank_a, mpcc_id); + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_ENABLE, lut_bank_a, mpcc_id); + } + } + + /* 3DLUT */ + switch (lut3d_src) { + case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM: + memset(&m_lut_params, 0, sizeof(m_lut_params)); + if (hubp->funcs->hubp_enable_3dlut_fl) + hubp->funcs->hubp_enable_3dlut_fl(hubp, false); + + if (mcm_luts.lut3d_data.lut3d_func && mcm_luts.lut3d_data.lut3d_func->state.bits.initialized) { + m_lut_params.lut3d = &mcm_luts.lut3d_data.lut3d_func->lut_3d; + if (mpc->funcs->populate_lut) + mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, m_lut_params, lut_bank_a, mpcc_id); + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, + mpcc_id); + } + break; + case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM: + switch (mcm_luts.lut3d_data.gpu_mem_params.size) { + case DC_CM2_GPU_MEM_SIZE_333333: + width = hubp_3dlut_fl_width_33; + break; + case DC_CM2_GPU_MEM_SIZE_171717: + width = hubp_3dlut_fl_width_17; + break; + case DC_CM2_GPU_MEM_SIZE_TRANSFORMED: + width = hubp_3dlut_fl_width_transformed; + break; + default: + //TODO: Handle default case + break; + } + + //check for support + if (mpc->funcs->mcm.is_config_supported && + !mpc->funcs->mcm.is_config_supported(width)) + break; + + if (mpc->funcs->program_lut_read_write_control) + mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, mpcc_id); + if (mpc->funcs->program_lut_mode) + mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, mpcc_id); + + if (hubp->funcs->hubp_program_3dlut_fl_addr) + hubp->funcs->hubp_program_3dlut_fl_addr(hubp, mcm_luts.lut3d_data.gpu_mem_params.addr); + + if (mpc->funcs->mcm.program_bit_depth) + mpc->funcs->mcm.program_bit_depth(mpc, mcm_luts.lut3d_data.gpu_mem_params.bit_depth, mpcc_id); + + dc_get_lut_mode(mcm_luts.lut3d_data.gpu_mem_params.layout, &mode, &addr_mode); + if (hubp->funcs->hubp_program_3dlut_fl_mode) + hubp->funcs->hubp_program_3dlut_fl_mode(hubp, mode); + + if (hubp->funcs->hubp_program_3dlut_fl_addressing_mode) + hubp->funcs->hubp_program_3dlut_fl_addressing_mode(hubp, addr_mode); + + switch (mcm_luts.lut3d_data.gpu_mem_params.format_params.format) { + case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB: + format = hubp_3dlut_fl_format_unorm_12msb_bitslice; + break; + case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB: + format = hubp_3dlut_fl_format_unorm_12lsb_bitslice; + break; + case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10: + format = hubp_3dlut_fl_format_float_fp1_5_10; + break; + } + if (hubp->funcs->hubp_program_3dlut_fl_format) + hubp->funcs->hubp_program_3dlut_fl_format(hubp, format); + if (hubp->funcs->hubp_update_3dlut_fl_bias_scale && + mpc->funcs->mcm.program_bias_scale) { + mpc->funcs->mcm.program_bias_scale(mpc, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale, + mpcc_id); + hubp->funcs->hubp_update_3dlut_fl_bias_scale(hubp, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias, + mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale); + } + + //navi 4x has a bug and r and blue are swapped and need to be worked around here in + //TODO: need to make a method for get_xbar per asic OR do the workaround in program_crossbar for 4x + dc_get_lut_xbar( + mcm_luts.lut3d_data.gpu_mem_params.component_order, + &crossbar_bit_slice_cr_r, + &crossbar_bit_slice_y_g, + &crossbar_bit_slice_cb_b); + + if (hubp->funcs->hubp_program_3dlut_fl_crossbar) + hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp, + crossbar_bit_slice_cr_r, + crossbar_bit_slice_y_g, + crossbar_bit_slice_cb_b); + + if (mpc->funcs->mcm.program_lut_read_write_control) + mpc->funcs->mcm.program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, true, mpcc_id); + + if (mpc->funcs->mcm.program_3dlut_size) + mpc->funcs->mcm.program_3dlut_size(mpc, width, mpcc_id); + + if (mpc->funcs->update_3dlut_fast_load_select) + mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst); + + if (hubp->funcs->hubp_enable_3dlut_fl) + hubp->funcs->hubp_enable_3dlut_fl(hubp, true); + else { + if (mpc->funcs->program_lut_mode) { + mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_DISABLE, lut_bank_a, mpcc_id); + mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id); + mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id); + } + } + break; + } +} + +bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx, + const struct dc_plane_state *plane_state) +{ + struct dpp *dpp_base = pipe_ctx->plane_res.dpp; + int mpcc_id = pipe_ctx->plane_res.hubp->inst; + struct dc *dc = pipe_ctx->stream_res.opp->ctx->dc; + struct mpc *mpc = dc->res_pool->mpc; + bool result; + const struct pwl_params *lut_params = NULL; + bool rval; + + if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) { + dcn42_populate_mcm_luts(dc, pipe_ctx, plane_state->mcm_luts, plane_state->lut_bank_a); + return true; + } + + mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id); + pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE; + // 1D LUT + if (plane_state->blend_tf.type == TF_TYPE_HWPWL) + lut_params = &plane_state->blend_tf.pwl; + else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->blend_tf, + &dpp_base->regamma_params, false); + lut_params = rval ? &dpp_base->regamma_params : NULL; + } + result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id); + lut_params = NULL; + + // Shaper + if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL) + lut_params = &plane_state->in_shaper_func.pwl; + else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { + // TODO: dpp_base replace + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_shaper_func, + &dpp_base->shaper_params, true); + lut_params = rval ? &dpp_base->shaper_params : NULL; + } + result &= mpc->funcs->program_shaper(mpc, lut_params, mpcc_id); + + // 3D + if (mpc->funcs->program_3dlut) { + if (plane_state->lut3d_func.state.bits.initialized == 1) + result &= mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id); + else + result &= mpc->funcs->program_3dlut(mpc, NULL, mpcc_id); + } + return result; } void dcn42_hardware_release(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h index c4cfeed45b190..0539ee0ffaee3 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h @@ -18,11 +18,18 @@ void dcn42_program_cm_hist( bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state); +void dcn42_populate_mcm_luts(struct dc *dc, + struct pipe_ctx *pipe_ctx, + struct dc_cm2_func_luts mcm_luts, + bool lut_bank_a); + bool dcn42_program_rmcm_luts( struct hubp *hubp, struct pipe_ctx *pipe_ctx, - const struct dc_plane_cm *cm, + enum dc_cm2_transfer_func_source lut3d_src, + struct dc_cm2_func_luts *mcm_luts, struct mpc *mpc, + bool lut_bank_a, int mpcc_id); void dcn42_hardware_release(struct dc *dc); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h index 2a5a81d159507..fc005aadfd551 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h @@ -89,7 +89,7 @@ enum hubp_3dlut_fl_addressing_mode { enum hubp_3dlut_fl_width { hubp_3dlut_fl_width_17 = 17, hubp_3dlut_fl_width_33 = 33, - hubp_3dlut_fl_width_17_transformed = 4916, //mpc default + hubp_3dlut_fl_width_transformed = 4916, //mpc default }; enum hubp_3dlut_fl_crossbar_bit_slice { @@ -99,6 +99,22 @@ enum hubp_3dlut_fl_crossbar_bit_slice { hubp_3dlut_fl_crossbar_bit_slice_48_63 = 3 }; +struct hubp_fl_3dlut_config { + bool enabled; + enum hubp_3dlut_fl_width width; + enum hubp_3dlut_fl_mode mode; + enum hubp_3dlut_fl_format format; + uint16_t bias; + uint16_t scale; + struct dc_plane_address address; + enum hubp_3dlut_fl_addressing_mode addr_mode; + enum dc_cm2_gpu_mem_layout layout; + uint8_t protection_bits; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b; + enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r; +}; + struct hubp { const struct hubp_funcs *funcs; struct dc_context *ctx; @@ -273,15 +289,24 @@ struct hubp_funcs { void (*hubp_wait_pipe_read_start)(struct hubp *hubp); void (*hubp_program_mcache_id_and_split_coordinate)(struct hubp *hubp, struct dml2_hubp_pipe_mcache_regs *mcache_regs); + void (*hubp_update_3dlut_fl_bias_scale)(struct hubp *hubp, uint16_t bias, uint16_t scale); + void (*hubp_program_3dlut_fl_mode)(struct hubp *hubp, + enum hubp_3dlut_fl_mode mode); + void (*hubp_program_3dlut_fl_format)(struct hubp *hubp, + enum hubp_3dlut_fl_format format); void (*hubp_program_3dlut_fl_addr)(struct hubp *hubp, - const struct dc_plane_address *address); - void (*hubp_program_3dlut_fl_config)(struct hubp *hubp, - const struct dc_3dlut_dma *config); + const struct dc_plane_address address); void (*hubp_program_3dlut_fl_dlg_param)(struct hubp *hubp, int refcyc_per_3dlut_group); void (*hubp_enable_3dlut_fl)(struct hubp *hubp, bool enable); + void (*hubp_program_3dlut_fl_addressing_mode)(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode); + void (*hubp_program_3dlut_fl_width)(struct hubp *hubp, enum hubp_3dlut_fl_width width); + void (*hubp_program_3dlut_fl_tmz_protected)(struct hubp *hubp, uint8_t protection_bits); void (*hubp_program_3dlut_fl_crossbar)(struct hubp *hubp, - enum dc_cm_lut_pixel_format format); + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b, + enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r); int (*hubp_get_3dlut_fl_done)(struct hubp *hubp); + void (*hubp_program_3dlut_fl_config)(struct hubp *hubp, struct hubp_fl_3dlut_config *cfg); void (*hubp_clear_tiling)(struct hubp *hubp); uint32_t (*hubp_get_current_read_line)(struct hubp *hubp); uint32_t (*hubp_get_det_config_error)(struct hubp *hubp); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h index f5617674bea8e..54eb2eba68bf4 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h @@ -54,7 +54,6 @@ #include "dc_hw_types.h" #include "hw_shared.h" #include "transform.h" -#include "dc_types.h" #define MAX_MPCC 6 #define MAX_OPP 6 @@ -102,6 +101,13 @@ enum mpcc_movable_cm_location { MPCC_MOVABLE_CM_LOCATION_AFTER, }; +enum MCM_LUT_XABLE { + MCM_LUT_DISABLE, + MCM_LUT_DISABLED = MCM_LUT_DISABLE, + MCM_LUT_ENABLE, + MCM_LUT_ENABLED = MCM_LUT_ENABLE, +}; + enum MCM_LUT_ID { MCM_LUT_3DLUT, MCM_LUT_1DLUT, @@ -110,7 +116,7 @@ enum MCM_LUT_ID { struct mpc_fl_3dlut_config { bool enabled; - enum dc_cm_lut_size size; + uint16_t width; bool select_lut_bank_a; uint16_t bit_depth; int hubp_index; @@ -1069,11 +1075,8 @@ struct mpc_funcs { * * void */ - void (*populate_lut)(struct mpc *mpc, - const enum MCM_LUT_ID id, - const union mcm_lut_params *params, - const bool lut_bank_a, - const int mpcc_id); + void (*populate_lut)(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params, + bool lut_bank_a, int mpcc_id); /** * @program_lut_read_write_control: @@ -1084,18 +1087,13 @@ struct mpc_funcs { * - [in/out] mpc - MPC context. * - [in] id * - [in] lut_bank_a - * - [in] bit_depth * - [in] mpcc_id * * Return: * * void */ - void (*program_lut_read_write_control)(struct mpc *mpc, - const enum MCM_LUT_ID id, - const bool lut_bank_a, - const unsigned int bit_depth, - const int mpcc_id); + void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id, bool lut_bank_a, int mpcc_id); /** * @program_lut_mode: @@ -1105,44 +1103,33 @@ struct mpc_funcs { * Parameters: * - [in/out] mpc - MPC context. * - [in] id - * - [in] enable + * - [in] xable * - [in] lut_bank_a - * - [in] size * - [in] mpcc_id * * Return: * * void */ - void (*program_lut_mode)(struct mpc *mpc, - const enum MCM_LUT_ID id, - const bool enable, - const bool lut_bank_a, - const enum dc_cm_lut_size size, - const int mpcc_id); - + void (*program_lut_mode)(struct mpc *mpc, const enum MCM_LUT_ID id, const enum MCM_LUT_XABLE xable, + bool lut_bank_a, int mpcc_id); /** - * @get_lut_mode: - * - * Obtains enablement and ram bank status. - * - * Parameters: - * - [in/out] mpc - MPC context. - * - [in] id - * - [in] mpcc_id - * - [out] enable - * - [out] lut_bank_a - * - * Return: - * - * void - */ - void (*get_lut_mode)(struct mpc *mpc, - const enum MCM_LUT_ID id, - const int mpcc_id, - bool *enable, - bool *lut_bank_a); + * @mcm: + * + * MPC MCM new HW sequential programming functions + */ + struct { + void (*program_3dlut_size)(struct mpc *mpc, uint32_t width, int mpcc_id); + void (*program_bias_scale)(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id); + void (*program_bit_depth)(struct mpc *mpc, uint16_t bit_depth, int mpcc_id); + bool (*is_config_supported)(uint32_t width); + void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id, + bool lut_bank_a, bool enabled, int mpcc_id); + + void (*populate_lut)(struct mpc *mpc, const union mcm_lut_params params, + bool lut_bank_a, int mpcc_id); + } mcm; /** * @rmcm: @@ -1155,11 +1142,9 @@ struct mpc_funcs { void (*update_3dlut_fast_load_select)(struct mpc *mpc, int mpcc_id, int hubp_idx); void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id, bool lut_bank_a, bool enabled, int mpcc_id); - void (*program_lut_mode)(struct mpc *mpc, - bool enable, - bool lut_bank_a, - int mpcc_id); - void (*program_3dlut_size)(struct mpc *mpc, const enum dc_cm_lut_size size, int mpcc_id); + void (*program_lut_mode)(struct mpc *mpc, const enum MCM_LUT_XABLE xable, + bool lut_bank_a, int mpcc_id); + void (*program_3dlut_size)(struct mpc *mpc, uint32_t width, int mpcc_id); void (*program_bias_scale)(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id); void (*program_bit_depth)(struct mpc *mpc, uint16_t bit_depth, int mpcc_id); bool (*is_config_supported)(uint32_t width); diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c index e0617db2d0c1d..ce1ee2062e41d 100644 --- a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c @@ -73,15 +73,56 @@ void mpc401_set_movable_cm_location(struct mpc *mpc, enum mpcc_movable_cm_locati } } -void mpc401_populate_lut(struct mpc *mpc, - const enum MCM_LUT_ID id, - const union mcm_lut_params *params, - const bool lut_bank_a, - const int mpcc_id) +static enum dc_lut_mode get3dlut_config( + struct mpc *mpc, + bool *is_17x17x17, + bool *is_12bits_color_channel, + int mpcc_id) +{ + uint32_t i_mode, i_enable_10bits, lut_size; + enum dc_lut_mode mode; + struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc); + + REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], + MPCC_MCM_3DLUT_MODE_CURRENT, &i_mode); + + REG_GET(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], + MPCC_MCM_3DLUT_30BIT_EN, &i_enable_10bits); + + switch (i_mode) { + case 0: + mode = LUT_BYPASS; + break; + case 1: + mode = LUT_RAM_A; + break; + case 2: + mode = LUT_RAM_B; + break; + default: + mode = LUT_BYPASS; + break; + } + if (i_enable_10bits > 0) + *is_12bits_color_channel = false; + else + *is_12bits_color_channel = true; + + REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, &lut_size); + + if (lut_size == 0) + *is_17x17x17 = true; + else + *is_17x17x17 = false; + + return mode; +} + +void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params, bool lut_bank_a, int mpcc_id) { const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B; - const struct pwl_params *lut1d = params->pwl; - const struct pwl_params *lut_shaper = params->pwl; + const struct pwl_params *lut1d = params.pwl; + const struct pwl_params *lut_shaper = params.pwl; bool is_17x17x17; bool is_12bits_color_channel; const struct dc_rgb *lut0; @@ -90,7 +131,7 @@ void mpc401_populate_lut(struct mpc *mpc, const struct dc_rgb *lut3; int lut_size0; int lut_size; - const struct tetrahedral_params *lut3d = params->lut3d; + const struct tetrahedral_params *lut3d = params.lut3d; switch (id) { case MCM_LUT_1DLUT: @@ -133,6 +174,8 @@ void mpc401_populate_lut(struct mpc *mpc, mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true); + get3dlut_config(mpc, &is_17x17x17, &is_12bits_color_channel, mpcc_id); + is_17x17x17 = !lut3d->use_tetrahedral_9; is_12bits_color_channel = lut3d->use_12bits; if (is_17x17x17) { @@ -155,6 +198,8 @@ void mpc401_populate_lut(struct mpc *mpc, sizeof(lut3d->tetrahedral_9.lut1[0]); } + mpc32_select_3dlut_ram(mpc, next_mode, + is_12bits_color_channel, mpcc_id); mpc32_select_3dlut_ram_mask(mpc, 0x1, mpcc_id); if (is_12bits_color_channel) mpc32_set3dlut_ram12(mpc, lut0, lut_size0, mpcc_id); @@ -187,69 +232,46 @@ void mpc401_populate_lut(struct mpc *mpc, } -static uint32_t mpc401_cm_lut_size_to_3dlut_size(const enum dc_cm_lut_size cm_size) -{ - uint32_t size = 0; - - switch (cm_size) { - case CM_LUT_SIZE_999: - size = 1; - break; - case CM_LUT_SIZE_171717: - size = 0; - break; - default: - /* invalid LUT size */ - ASSERT(false); - size = 0; - break; - } - - return size; -} - void mpc401_program_lut_mode( struct mpc *mpc, const enum MCM_LUT_ID id, - const bool enable, - const bool lut_bank_a, - const enum dc_cm_lut_size size, - const int mpcc_id) + const enum MCM_LUT_XABLE xable, + bool lut_bank_a, + int mpcc_id) { - uint32_t lut_size; struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc); switch (id) { case MCM_LUT_3DLUT: - if (enable) { - lut_size = mpc401_cm_lut_size_to_3dlut_size(size); - REG_UPDATE_2(MPCC_MCM_3DLUT_MODE[mpcc_id], - MPCC_MCM_3DLUT_MODE, lut_bank_a ? 1 : 2, - MPCC_MCM_3DLUT_SIZE, lut_size); - } else { - if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) - mpc32_power_on_shaper_3dlut(mpc, mpcc_id, false); + switch (xable) { + case MCM_LUT_DISABLE: REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE, 0); + break; + case MCM_LUT_ENABLE: + REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE, lut_bank_a ? 1 : 2); + break; } break; case MCM_LUT_SHAPER: - if (enable) { - REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2); - } else { - if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) - mpc32_power_on_shaper_3dlut(mpc, mpcc_id, false); + switch (xable) { + case MCM_LUT_DISABLE: REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, 0); + break; + case MCM_LUT_ENABLE: + REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2); + break; } break; case MCM_LUT_1DLUT: - if (enable) { - REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id], - MPCC_MCM_1DLUT_MODE, 2); - } else { - if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) - mpc32_power_on_blnd_lut(mpc, mpcc_id, false); + switch (xable) { + case MCM_LUT_DISABLE: REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id], MPCC_MCM_1DLUT_MODE, 0); + break; + case MCM_LUT_ENABLE: + REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id], + MPCC_MCM_1DLUT_MODE, 2); + break; } REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id], MPCC_MCM_1DLUT_SELECT, lut_bank_a ? 0 : 1); @@ -257,20 +279,14 @@ void mpc401_program_lut_mode( } } -void mpc401_program_lut_read_write_control(struct mpc *mpc, - const enum MCM_LUT_ID id, - const bool lut_bank_a, - const unsigned int bit_depth, - const int mpcc_id) +void mpc401_program_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id, bool lut_bank_a, int mpcc_id) { struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc); switch (id) { case MCM_LUT_3DLUT: mpc32_select_3dlut_ram_mask(mpc, 0xf, mpcc_id); - REG_UPDATE_2(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], - MPCC_MCM_3DLUT_30BIT_EN, (bit_depth == 10) ? 1 : 0, - MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1); + REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1); break; case MCM_LUT_SHAPER: mpc32_configure_shaper_lut(mpc, lut_bank_a, mpcc_id); @@ -562,44 +578,6 @@ void mpc401_get_gamut_remap(struct mpc *mpc, arr_reg_val, ARRAY_SIZE(arr_reg_val)); } -void mpc401_get_lut_mode(struct mpc *mpc, - const enum MCM_LUT_ID id, - const int mpcc_id, - bool *enable, - bool *lut_bank_a) -{ - struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc); - - uint32_t lut_mode = 0; - uint32_t lut_select = 0; - - *enable = false; - *lut_bank_a = true; - - switch (id) { - case MCM_LUT_SHAPER: - REG_GET(MPCC_MCM_SHAPER_CONTROL[mpcc_id], - MPCC_MCM_SHAPER_MODE_CURRENT, &lut_mode); - *enable = lut_mode != 0; - *lut_bank_a = lut_mode != 2; - break; - case MCM_LUT_1DLUT: - REG_GET_2(MPCC_MCM_1DLUT_CONTROL[mpcc_id], - MPCC_MCM_1DLUT_MODE_CURRENT, &lut_mode, - MPCC_MCM_1DLUT_SELECT_CURRENT, &lut_select); - *enable = lut_mode != 0; - *lut_bank_a = lut_mode == 0 || lut_select == 0; - break; - case MCM_LUT_3DLUT: - default: - REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], - MPCC_MCM_3DLUT_MODE_CURRENT, &lut_mode); - *enable = lut_mode != 0; - *lut_bank_a = lut_mode != 2; - break; - } -} - static const struct mpc_funcs dcn401_mpc_funcs = { .read_mpcc_state = mpc1_read_mpcc_state, .insert_plane = mpc1_insert_plane, @@ -638,7 +616,6 @@ static const struct mpc_funcs dcn401_mpc_funcs = { .populate_lut = mpc401_populate_lut, .program_lut_read_write_control = mpc401_program_lut_read_write_control, .program_lut_mode = mpc401_program_lut_mode, - .get_lut_mode = mpc401_get_lut_mode, }; diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h index c16560c84453e..6d842d7b95c7e 100644 --- a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h +++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h @@ -206,32 +206,21 @@ void dcn401_mpc_construct(struct dcn401_mpc *mpc401, int num_rmu); void mpc401_set_movable_cm_location(struct mpc *mpc, enum mpcc_movable_cm_location location, int mpcc_id); -void mpc401_populate_lut(struct mpc *mpc, - const enum MCM_LUT_ID id, - const union mcm_lut_params *params, - bool lut_bank_a, - int mpcc_id); +void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params, + bool lut_bank_a, int mpcc_id); void mpc401_program_lut_mode( struct mpc *mpc, const enum MCM_LUT_ID id, - const bool enable, - const bool lut_bank_a, - const enum dc_cm_lut_size size, - const int mpcc_id); - -void mpc401_get_lut_mode(struct mpc *mpc, - const enum MCM_LUT_ID id, - const int mpcc_id, - bool *enable, - bool *lut_bank_a); + const enum MCM_LUT_XABLE xable, + bool lut_bank_a, + int mpcc_id); void mpc401_program_lut_read_write_control( struct mpc *mpc, const enum MCM_LUT_ID id, - const bool lut_bank_a, - const unsigned int bit_depth, - const int mpcc_id); + bool lut_bank_a, + int mpcc_id); void mpc401_set_gamut_remap( struct mpc *mpc, diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c index 507dbdbea6008..38c0e8f96d40e 100644 --- a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c @@ -63,6 +63,154 @@ void mpc42_update_blending( mpcc->blnd_cfg = *blnd_cfg; } +/* Shaper functions */ +void mpc42_power_on_shaper_3dlut( + struct mpc *mpc, + uint32_t mpcc_id, + bool power_on) +{ + uint32_t power_status_shaper = 2; + uint32_t power_status_3dlut = 2; + struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); + int max_retries = 10; + + REG_SET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], 0, + MPCC_MCM_3DLUT_MEM_PWR_DIS, power_on == true ? 1:0); + REG_SET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], 0, + MPCC_MCM_SHAPER_MEM_PWR_DIS, power_on == true ? 1:0); + /* wait for memory to fully power up */ + if (power_on && mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) { + REG_WAIT(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_SHAPER_MEM_PWR_STATE, 0, 1, max_retries); + REG_WAIT(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_3DLUT_MEM_PWR_STATE, 0, 1, max_retries); + } + + /*read status is not mandatory, it is just for debugging*/ + REG_GET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_SHAPER_MEM_PWR_STATE, &power_status_shaper); + REG_GET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_3DLUT_MEM_PWR_STATE, &power_status_3dlut); + + if (power_status_shaper != 0 && power_on == true) + BREAK_TO_DEBUGGER(); + + if (power_status_3dlut != 0 && power_on == true) + BREAK_TO_DEBUGGER(); +} + +void mpc42_configure_shaper_lut( + struct mpc *mpc, + bool is_ram_a, + uint32_t mpcc_id) +{ + struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); + + REG_UPDATE(MPCC_MCM_SHAPER_SCALE_G_B[mpcc_id], + MPCC_MCM_SHAPER_SCALE_B, 0x7000); + REG_UPDATE(MPCC_MCM_SHAPER_SCALE_G_B[mpcc_id], + MPCC_MCM_SHAPER_SCALE_G, 0x7000); + REG_UPDATE(MPCC_MCM_SHAPER_SCALE_R[mpcc_id], + MPCC_MCM_SHAPER_SCALE_R, 0x7000); + REG_UPDATE(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id], + MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, 7); + REG_UPDATE(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id], + MPCC_MCM_SHAPER_LUT_WRITE_SEL, is_ram_a == true ? 0:1); + REG_SET(MPCC_MCM_SHAPER_LUT_INDEX[mpcc_id], 0, MPCC_MCM_SHAPER_LUT_INDEX, 0); +} + + +void mpc42_program_3dlut_size(struct mpc *mpc, uint32_t width, int mpcc_id) +{ + struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); + uint32_t size = 0xff; + + REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, &size); + + REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, + (width == 33) ? 2 : + (width == 17) ? 0 : 2); + + REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, &size); +} + +void mpc42_program_3dlut_fl_bias_scale(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id) +{ + struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); + + REG_UPDATE_2(MPCC_MCM_3DLUT_OUT_OFFSET_R[mpcc_id], + MPCC_MCM_3DLUT_OUT_OFFSET_R, bias, + MPCC_MCM_3DLUT_OUT_SCALE_R, scale); + + REG_UPDATE_2(MPCC_MCM_3DLUT_OUT_OFFSET_G[mpcc_id], + MPCC_MCM_3DLUT_OUT_OFFSET_G, bias, + MPCC_MCM_3DLUT_OUT_SCALE_G, scale); + + REG_UPDATE_2(MPCC_MCM_3DLUT_OUT_OFFSET_B[mpcc_id], + MPCC_MCM_3DLUT_OUT_OFFSET_B, bias, + MPCC_MCM_3DLUT_OUT_SCALE_B, scale); +} + +void mpc42_program_bit_depth(struct mpc *mpc, uint16_t bit_depth, int mpcc_id) +{ + struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); + + REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPCC_MCM_3DLUT_WRITE_EN_MASK, 0xF); + + //program bit_depth + REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], + MPCC_MCM_3DLUT_30BIT_EN, + (bit_depth == 10) ? 1 : 0); +} + +bool mpc42_is_config_supported(uint32_t width) +{ + if (width == 17) + return true; + + return false; +} + +void mpc42_populate_lut(struct mpc *mpc, const union mcm_lut_params params, + bool lut_bank_a, int mpcc_id) +{ + const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B; + const struct pwl_params *lut_shaper = params.pwl; + + if (lut_shaper == NULL) + return; + if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) + mpc42_power_on_shaper_3dlut(mpc, mpcc_id, true); + + mpc42_configure_shaper_lut(mpc, next_mode == LUT_RAM_A, mpcc_id); + + if (next_mode == LUT_RAM_A) + mpc32_program_shaper_luta_settings(mpc, lut_shaper, mpcc_id); + else + mpc32_program_shaper_lutb_settings(mpc, lut_shaper, mpcc_id); + + mpc32_program_shaper_lut( + mpc, lut_shaper->rgb_resulted, lut_shaper->hw_points_num, mpcc_id); + + mpc42_power_on_shaper_3dlut(mpc, mpcc_id, false); +} + +void mpc42_program_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id, + bool lut_bank_a, bool enabled, int mpcc_id) +{ + struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); + + switch (id) { + case MCM_LUT_3DLUT: + REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE, + (!enabled) ? 0 : + (lut_bank_a) ? 1 : 2); + REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1); + break; + case MCM_LUT_SHAPER: + mpc32_configure_shaper_lut(mpc, lut_bank_a, mpcc_id); + break; + default: + break; + } +} + /* RMCM Shaper functions */ void mpc42_power_on_rmcm_shaper_3dlut( struct mpc *mpc, @@ -526,47 +674,32 @@ void mpc42_program_rmcm_lut_read_write_control(struct mpc *mpc, const enum MCM_L } } -void mpc42_program_lut_mode(struct mpc *mpc, - bool enable, - bool lut_bank_a, - int mpcc_id) +void mpc42_program_lut_mode(struct mpc *mpc, const enum MCM_LUT_XABLE xable, + bool lut_bank_a, int mpcc_id) { struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); - if (enable) { - REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2); - } else { + switch (xable) { + case MCM_LUT_DISABLE: REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, 0); - } -} - -static uint32_t mpc42_get_rmcm_3dlut_width( - const enum dc_cm_lut_size size) -{ - uint32_t width = 0; - - switch (size) { - case CM_LUT_SIZE_333333: - width = 2; break; - case CM_LUT_SIZE_171717: - default: - width = 0; + case MCM_LUT_ENABLE: + REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2); break; } - - return width; } -void mpc42_program_rmcm_3dlut_size(struct mpc *mpc, - const enum dc_cm_lut_size size, - int mpcc_id) +void mpc42_program_rmcm_3dlut_size(struct mpc *mpc, uint32_t width, int mpcc_id) { struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); - uint32_t width = mpc42_get_rmcm_3dlut_width(size); + uint32_t size = 0xff; + + REG_GET(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_SIZE, &size); - REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id], - MPC_RMCM_3DLUT_SIZE, width); + REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_SIZE, + (width == 33) ? 2 : 0); + + REG_GET(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_SIZE, &size); } void mpc42_program_rmcm_3dlut_fast_load_bias_scale(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id) @@ -598,6 +731,14 @@ void mpc42_program_rmcm_bit_depth(struct mpc *mpc, uint16_t bit_depth, int mpcc_ (bit_depth == 10) ? 1 : 0); } +bool mpc42_is_rmcm_config_supported(uint32_t width) +{ + if (width == 17 || width == 33) + return true; + + return false; +} + void mpc42_set_fl_config( struct mpc *mpc, struct mpc_fl_3dlut_config *cfg, @@ -605,7 +746,6 @@ void mpc42_set_fl_config( { struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); - uint32_t width = mpc42_get_rmcm_3dlut_width(cfg->size); /* From: Jie Zhou @@ -646,7 +786,7 @@ void mpc42_set_fl_config( //width REG_UPDATE_2(MPC_RMCM_3DLUT_MODE[mpcc_id], - MPC_RMCM_3DLUT_SIZE, width, + MPC_RMCM_3DLUT_SIZE, (cfg->width == 33) ? 2 : 0, MPC_RMCM_3DLUT_MODE, (!cfg->enabled) ? 0 : (cfg->select_lut_bank_a) ? 1 : 2); //connect to hubp @@ -659,6 +799,182 @@ void mpc42_set_fl_config( REG_UPDATE(MPC_RMCM_CNTL[mpcc_id], MPC_RMCM_CNTL, cfg->enabled ? 0 : 0xF); } +//static void rmcm_program_gamut_remap( +// struct mpc *mpc, +// unsigned int mpcc_id, +// const uint16_t *regval, +// enum mpcc_gamut_remap_id gamut_remap_block_id, +// enum mpcc_gamut_remap_mode_select mode_select) +//{ +// struct color_matrices_reg gamut_regs; +// struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); +// +// if (gamut_remap_block_id == MPCC_OGAM_GAMUT_REMAP || +// gamut_remap_block_id == MPCC_MCM_FIRST_GAMUT_REMAP || +// gamut_remap_block_id == MPCC_MCM_SECOND_GAMUT_REMAP) { +// mpc_program_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select); +// return; +// } +// if (gamut_remap_block_id == MPCC_OGAM_GAMUT_REMAP) { +// +// if (regval == NULL || mode_select == MPCC_GAMUT_REMAP_MODE_SELECT_0) { +// REG_SET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id], 0, +// MPC_RMCM_GAMUT_REMAP_MODE, mode_select); +// return; +// } +// +// gamut_regs.shifts.csc_c11 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C11_A; +// gamut_regs.masks.csc_c11 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C11_A; +// gamut_regs.shifts.csc_c12 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C12_A; +// gamut_regs.masks.csc_c12 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C12_A; +// +// switch (mode_select) { +// case MPCC_GAMUT_REMAP_MODE_SELECT_1: +// gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_A[mpcc_id]); +// gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_A[mpcc_id]); +// break; +// case MPCC_GAMUT_REMAP_MODE_SELECT_2: +// gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_B[mpcc_id]); +// gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_B[mpcc_id]); +// break; +// default: +// break; +// } +// +// cm_helper_program_color_matrices( +// mpc->ctx, +// regval, +// &gamut_regs); +// +// //select coefficient set to use, set A (MODE_1) or set B (MODE_2) +// REG_SET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id], 0, MPC_RMCM_GAMUT_REMAP_MODE, mode_select); +// } +//} + +//static bool is_mpc_legacy_gamut_id(enum mpcc_gamut_remap_id gamut_remap_block_id) +//{ +// if (gamut_remap_block_id == MPCC_OGAM_GAMUT_REMAP || +// gamut_remap_block_id == MPCC_MCM_FIRST_GAMUT_REMAP || +// gamut_remap_block_id == MPCC_MCM_SECOND_GAMUT_REMAP) { +// return true; +// } +// return false; +//} +//static void program_gamut_remap( +// struct mpc *mpc, +// unsigned int mpcc_id, +// const uint16_t *regval, +// enum mpcc_gamut_remap_id gamut_remap_block_id, +// enum mpcc_gamut_remap_mode_select mode_select) +//{ +// if (is_mpc_legacy_gamut_id(gamut_remap_block_id)) +// mpc_program_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select); +// else +// rmcm_program_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select); +//} + +//void mpc42_set_gamut_remap( +// struct mpc *mpc, +// int mpcc_id, +// const struct mpc_grph_gamut_adjustment *adjust) +//{ +// struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); +// unsigned int i = 0; +// uint32_t mode_select = 0; +// +// if (adjust->gamut_adjust_type != GRAPHICS_GAMUT_ADJUST_TYPE_SW) { +// /* Bypass / Disable if type is bypass or hw */ +// program_gamut_remap(mpc, mpcc_id, NULL, +// adjust->mpcc_gamut_remap_block_id, MPCC_GAMUT_REMAP_MODE_SELECT_0); +// } else { +// struct fixed31_32 arr_matrix[12]; +// uint16_t arr_reg_val[12]; +// +// for (i = 0; i < 12; i++) +// arr_matrix[i] = adjust->temperature_matrix[i]; +// +// convert_float_matrix(arr_reg_val, arr_matrix, 12); +// +// if (is_mpc_legacy_gamut_id(adjust->mpcc_gamut_remap_block_id)) +// REG_GET(MPCC_GAMUT_REMAP_MODE[mpcc_id], +// MPCC_GAMUT_REMAP_MODE_CURRENT, &mode_select); +// else +// REG_GET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id], +// MPC_RMCM_GAMUT_REMAP_MODE_CURRENT, &mode_select); +// +// //If current set in use not set A (MODE_1), then use set A, otherwise use set B +// if (mode_select != MPCC_GAMUT_REMAP_MODE_SELECT_1) +// mode_select = MPCC_GAMUT_REMAP_MODE_SELECT_1; +// else +// mode_select = MPCC_GAMUT_REMAP_MODE_SELECT_2; +// +// program_gamut_remap(mpc, mpcc_id, arr_reg_val, +// adjust->mpcc_gamut_remap_block_id, mode_select); +// } +//} + +//static void read_gamut_remap(struct mpc *mpc, +// int mpcc_id, +// uint16_t *regval, +// enum mpcc_gamut_remap_id gamut_remap_block_id, +// uint32_t *mode_select) +//{ +// struct color_matrices_reg gamut_regs = {0}; +// struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc); +// +// if (is_mpc_legacy_gamut_id(gamut_remap_block_id)) { +// mpc_read_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select); +// } +// if (gamut_remap_block_id == MPCC_RMCM_GAMUT_REMAP) { +// //current coefficient set in use +// REG_GET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id], MPC_RMCM_GAMUT_REMAP_MODE, mode_select); +// +// gamut_regs.shifts.csc_c11 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C11_A; +// gamut_regs.masks.csc_c11 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C11_A; +// gamut_regs.shifts.csc_c12 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C12_A; +// gamut_regs.masks.csc_c12 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C12_A; +// +// switch (*mode_select) { +// case MPCC_GAMUT_REMAP_MODE_SELECT_1: +// gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_A[mpcc_id]); +// gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_A[mpcc_id]); +// break; +// case MPCC_GAMUT_REMAP_MODE_SELECT_2: +// gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_B[mpcc_id]); +// gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_B[mpcc_id]); +// break; +// default: +// break; +// } +// } +// +// if (*mode_select != MPCC_GAMUT_REMAP_MODE_SELECT_0) { +// cm_helper_read_color_matrices( +// mpc42->base.ctx, +// regval, +// &gamut_regs); +// } +//} + +//void mpc42_get_gamut_remap(struct mpc *mpc, +// int mpcc_id, +// struct mpc_grph_gamut_adjustment *adjust) +//{ +// uint16_t arr_reg_val[12] = {0}; +// uint32_t mode_select; +// +// read_gamut_remap(mpc, mpcc_id, arr_reg_val, adjust->mpcc_gamut_remap_block_id, &mode_select); +// +// if (mode_select == MPCC_GAMUT_REMAP_MODE_SELECT_0) { +// adjust->gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_BYPASS; +// return; +// } +// +// adjust->gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_SW; +// convert_hw_matrix(adjust->temperature_matrix, +// arr_reg_val, ARRAY_SIZE(arr_reg_val)); +//} + void mpc42_read_mpcc_state( struct mpc *mpc, int mpcc_inst, @@ -755,7 +1071,14 @@ static const struct mpc_funcs dcn42_mpc_funcs = { .populate_lut = mpc401_populate_lut, .program_lut_read_write_control = mpc401_program_lut_read_write_control, .program_lut_mode = mpc401_program_lut_mode, - .get_lut_mode = mpc401_get_lut_mode, + .mcm = { + .program_lut_read_write_control = mpc42_program_lut_read_write_control, + .program_3dlut_size = mpc42_program_3dlut_size, + .program_bias_scale = mpc42_program_3dlut_fl_bias_scale, + .program_bit_depth = mpc42_program_bit_depth, + .is_config_supported = mpc42_is_config_supported, + .populate_lut = mpc42_populate_lut, + }, .rmcm = { .enable_3dlut_fl = mpc42_enable_3dlut_fl, .update_3dlut_fast_load_select = mpc42_update_3dlut_fast_load_select, @@ -764,6 +1087,7 @@ static const struct mpc_funcs dcn42_mpc_funcs = { .program_3dlut_size = mpc42_program_rmcm_3dlut_size, .program_bias_scale = mpc42_program_rmcm_3dlut_fast_load_bias_scale, .program_bit_depth = mpc42_program_rmcm_bit_depth, + .is_config_supported = mpc42_is_rmcm_config_supported, .power_on_shaper_3dlut = mpc42_power_on_rmcm_shaper_3dlut, .populate_lut = mpc42_populate_rmcm_lut, .fl_3dlut_configure = mpc42_set_fl_config, diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h index a5f7f4f2bb3bc..12a12c28e5537 100644 --- a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h +++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h @@ -884,6 +884,49 @@ void dcn42_mpc_construct(struct dcn42_mpc *mpc401, void mpc42_init_mpcc(struct mpcc *mpcc, int mpcc_inst); +void mpc42_program_shaper_lutb_settings( + struct mpc *mpc, + const struct pwl_params *params, + uint32_t mpcc_id); +void mpc42_program_shaper_luta_settings( + struct mpc *mpc, + const struct pwl_params *params, + uint32_t mpcc_id); +void mpc42_configure_shaper_lut( + struct mpc *mpc, + bool is_ram_a, + uint32_t mpcc_id); +void mpc42_power_on_shaper_3dlut( + struct mpc *mpc, + uint32_t mpcc_id, + bool power_on); +void mpc42_program_3dlut_size( + struct mpc *mpc, + uint32_t width, + int mpcc_id); +void mpc42_program_3dlut_fl_bias_scale( + struct mpc *mpc, + uint16_t bias, + uint16_t scale, + int mpcc_id); +void mpc42_program_bit_depth( + struct mpc *mpc, + uint16_t bit_depth, + int mpcc_id); +void mpc42_populate_lut( + struct mpc *mpc, + const union mcm_lut_params params, + bool lut_bank_a, + int mpcc_id); +void mpc42_program_lut_read_write_control( + struct mpc *mpc, + const enum MCM_LUT_ID id, + bool lut_bank_a, + bool enabled, + int mpcc_id); + +bool mpc42_is_config_supported(uint32_t width); + /* RMCM */ void mpc42_program_rmcm_shaper_lut( struct mpc *mpc, @@ -927,12 +970,12 @@ void mpc42_program_rmcm_lut_read_write_control( int mpcc_id); void mpc42_program_lut_mode( struct mpc *mpc, - bool enable, + const enum MCM_LUT_XABLE xable, bool lut_bank_a, int mpcc_id); void mpc42_program_rmcm_3dlut_size( struct mpc *mpc, - const enum dc_cm_lut_size size, + uint32_t width, int mpcc_id); void mpc42_program_rmcm_3dlut_fast_load_bias_scale( struct mpc *mpc, @@ -944,6 +987,8 @@ void mpc42_program_rmcm_bit_depth( uint16_t bit_depth, int mpcc_id); +bool mpc42_is_rmcm_config_supported(uint32_t width); + void mpc42_set_fl_config( struct mpc *mpc, struct mpc_fl_3dlut_config *cfg, diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c index c0d37f00fed91..2c1f72cf1a63f 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c @@ -666,7 +666,6 @@ static const struct resource_caps res_cap_dcn42 = { .num_vmid = 16, .num_mpc_3dlut = 2, .num_dsc = 4, - .num_rmcm = 2, }; static const struct dc_plane_cap plane_cap = { diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c index 00473c6284d5c..614db22d62f38 100644 --- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c +++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c @@ -502,6 +502,8 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream, * * @stream: contains data we may need to construct VSIF (i.e. timing_3d_format, etc.) * @info_packet: output structure where to store VSIF + * @ALLMEnabled: indicates whether ALLM HF-VSIF should be generated + * @ALLMValue: ALLM bit value to advertise in HF-VSIF */ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet) diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 736304e73ca4d..952391aecf2d3 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -244,6 +244,10 @@ static ssize_t amdgpu_set_power_dpm_state(struct device *dev, enum amd_pm_state_type state; int ret; + /* Reject empty/whitespace strings - fuzzing found this is not validated */ + if (count == 0 || sysfs_streq(buf, "")) + return -EINVAL; + if (sysfs_streq(buf, "battery")) state = POWER_STATE_TYPE_BATTERY; else if (sysfs_streq(buf, "balanced")) @@ -364,6 +368,10 @@ static ssize_t amdgpu_set_power_dpm_force_performance_level(struct device *dev, enum amd_dpm_forced_level level; int ret = 0; + /* Reject empty/whitespace strings - fuzzing found this is not validated */ + if (count == 0 || sysfs_streq(buf, "")) + return -EINVAL; + if (sysfs_streq(buf, "low")) level = AMD_DPM_FORCED_LEVEL_LOW; else if (sysfs_streq(buf, "high")) @@ -902,6 +910,10 @@ static ssize_t amdgpu_set_pp_features(struct device *dev, uint64_t featuremask; int ret; + /* Reject empty/whitespace strings - fuzzing found kstrtou64 accepts "" as 0 */ + if (count == 0 || sysfs_streq(buf, "")) + return -EINVAL; + ret = kstrtou64(buf, 0, &featuremask); if (ret) return -EINVAL; @@ -1027,6 +1039,10 @@ static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask) *mask = 0; + /* Reject empty/whitespace strings - fuzzing found this is not validated */ + if (count == 0 || sysfs_streq(buf, "")) + return -EINVAL; + bytes = min(count, sizeof(buf_cpy) - 1); memcpy(buf_cpy, buf, bytes); buf_cpy[bytes] = '\0'; @@ -1378,6 +1394,10 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, long int profile_mode = 0; const char delimiter[3] = {' ', '\n', '\0'}; + /* Reject empty/whitespace strings - fuzzing found this is not validated */ + if (count == 0 || sysfs_streq(buf, "")) + return -EINVAL; + tmp[0] = *(buf); tmp[1] = '\0'; ret = kstrtol(tmp, 0, &profile_mode); @@ -2505,12 +2525,12 @@ static ssize_t amdgpu_set_pm_policy_attr(struct device *dev, .dev_attr = __ATTR(_name, 0644, amdgpu_get_pm_policy_attr, \ amdgpu_set_pm_policy_attr), \ .id = PP_PM_POLICY_##_id, \ - }; + } #define AMDGPU_PM_POLICY_ATTR_VAR(_name) pm_policy_attr_##_name.dev_attr.attr -AMDGPU_PM_POLICY_ATTR(soc_pstate, SOC_PSTATE) -AMDGPU_PM_POLICY_ATTR(xgmi_plpd, XGMI_PLPD) +AMDGPU_PM_POLICY_ATTR(soc_pstate, SOC_PSTATE); +AMDGPU_PM_POLICY_ATTR(xgmi_plpd, XGMI_PLPD); static struct attribute *pm_policy_attrs[] = { &AMDGPU_PM_POLICY_ATTR_VAR(soc_pstate), diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c index 8dee5f2fbde51..00fe1f7893fba 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c +++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c @@ -306,7 +306,9 @@ static void analogix_dp_get_adjust_training_lane(struct analogix_dp_device *dp, lane_count = dp->link_train.lane_count; for (lane = 0; lane < lane_count; lane++) { voltage_swing = drm_dp_get_adjust_request_voltage(link_status, lane); + voltage_swing >>= DP_TRAIN_VOLTAGE_SWING_SHIFT; pre_emphasis = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); + pre_emphasis >>= DP_TRAIN_PRE_EMPHASIS_SHIFT; training_lane = DPCD_VOLTAGE_SWING_SET(voltage_swing) | DPCD_PRE_EMPHASIS_SET(pre_emphasis); @@ -352,7 +354,9 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) for (lane = 0; lane < lane_count; lane++) { training_lane = analogix_dp_get_lane_link_training(dp, lane); voltage_swing = drm_dp_get_adjust_request_voltage(link_status, lane); + voltage_swing >>= DP_TRAIN_VOLTAGE_SWING_SHIFT; pre_emphasis = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); + pre_emphasis >>= DP_TRAIN_PRE_EMPHASIS_SHIFT; if (DPCD_VOLTAGE_SWING_GET(training_lane) == voltage_swing && DPCD_PRE_EMPHASIS_GET(training_lane) == pre_emphasis) diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index 404208bf23a6b..df3c25bac761d 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -7575,6 +7575,14 @@ static void drm_parse_tiled_block(struct drm_connector *connector, u8 num_v_tile, num_h_tile; struct drm_tile_group *tg; + /* tiled block payload per spec: cap 1 + topo 3 + size 4 + bezel 5 + id 9 = 22 */ + if (block->num_bytes < 22) { + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] Unexpected tiled block size %u\n", + connector->base.id, connector->name, block->num_bytes); + return; + } + w = tile->tile_size[0] | tile->tile_size[1] << 8; h = tile->tile_size[2] | tile->tile_size[3] << 8; diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 1541fc8a9ac2d..515f699e3ecd3 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -225,16 +225,106 @@ static void drm_fb_helper_resume_worker(struct work_struct *work) console_unlock(); } +static int find_crtc_index_atomic(struct drm_fb_helper *helper) +{ + struct drm_device *dev = helper->dev; + int crtc_index = -EINVAL; + struct drm_modeset_acquire_ctx ctx; + struct drm_plane *plane; + int ret = 0; + + drm_modeset_acquire_init(&ctx, 0); + +retry: + drm_for_each_plane(plane, dev) { + const struct drm_plane_state *plane_state; + + if (plane->type != DRM_PLANE_TYPE_PRIMARY) + continue; + + ret = drm_modeset_lock(&plane->mutex, &ctx); + if (ret) + goto err_drm_modeset_lock; + plane_state = plane->state; + + if (plane_state->fb == helper->fb && plane_state->crtc) { + struct drm_crtc *crtc = plane_state->crtc; + + ret = drm_modeset_lock(&crtc->mutex, &ctx); + if (ret) + goto err_drm_modeset_lock; + if (crtc->state->active) + crtc_index = crtc->index; + drm_modeset_unlock(&crtc->mutex); + } + drm_modeset_unlock(&plane->mutex); + + if (crtc_index >= 0) + break; + } + + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + + return crtc_index; + +err_drm_modeset_lock: + if (ret == -EDEADLK) { + drm_modeset_backoff(&ctx); + goto retry; + } + return ret; +} + +static int find_crtc_index_legacy(struct drm_fb_helper *helper) +{ + struct drm_device *dev = helper->dev; + struct drm_crtc *crtc; + + drm_for_each_crtc(crtc, dev) { + struct drm_plane *plane = crtc->primary; + + if (!crtc->enabled) + continue; + if (!plane || plane->fb != helper->fb) + continue; /* CRTC doesn't display fbdev emulation */ + + return crtc->index; + } + + return -EINVAL; +} + +static int drm_fb_helper_find_crtc_index(struct drm_fb_helper *helper) +{ + struct drm_device *dev = helper->dev; + int crtc_index; + + mutex_lock(&dev->mode_config.mutex); + + if (drm_drv_uses_atomic_modeset(dev)) + crtc_index = find_crtc_index_atomic(helper); + else + crtc_index = find_crtc_index_legacy(helper); + + mutex_unlock(&dev->mode_config.mutex); + + return crtc_index; +} + static void drm_fb_helper_fb_dirty(struct drm_fb_helper *helper) { struct drm_device *dev = helper->dev; struct drm_clip_rect *clip = &helper->damage_clip; struct drm_clip_rect clip_copy; + int crtc_index; unsigned long flags; int ret; mutex_lock(&helper->lock); - drm_client_modeset_wait_for_vblank(&helper->client, 0); + crtc_index = drm_fb_helper_find_crtc_index(helper); + if (crtc_index >= 0) + drm_client_modeset_wait_for_vblank(&helper->client, crtc_index); mutex_unlock(&helper->lock); if (drm_WARN_ON_ONCE(dev, !helper->funcs->fb_dirty)) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 4b928fda5b127..7993e85c05661 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1065,6 +1065,11 @@ drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, goto err_notifier_remove; } + if (vas->vm_flags & (VM_IO | VM_PFNMAP)) { + err = -EIO; + goto err_notifier_remove; + } + range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1); if (range) goto out_mmunlock; diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 44acfe4120d24..83020b6cf9ba1 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -25,6 +25,7 @@ * */ +#include #include #include @@ -1117,6 +1118,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, gpuvm->drm = drm; gpuvm->r_obj = r_obj; + drm_dev_get(drm); drm_gem_object_get(r_obj); drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range); @@ -1160,13 +1162,15 @@ static void drm_gpuvm_free(struct kref *kref) { struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref); + struct drm_device *drm = gpuvm->drm; drm_gpuvm_fini(gpuvm); - if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free)) + if (drm_WARN_ON(drm, !gpuvm->ops->vm_free)) return; gpuvm->ops->vm_free(gpuvm); + drm_dev_put(drm); } /** @@ -1322,6 +1326,9 @@ drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec, drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { struct drm_gem_object *obj = va->gem.obj; + if (unlikely(!obj)) + continue; + ret = exec_prepare_obj(exec, obj, num_fences); if (ret) return ret; diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c index e2df4becce629..9039a39c43243 100644 --- a/drivers/gpu/drm/drm_ioctl.c +++ b/drivers/gpu/drm/drm_ioctl.c @@ -373,13 +373,25 @@ drm_setclientcap(struct drm_device *dev, void *data, struct drm_file *file_priv) return -EINVAL; file_priv->supports_virtualized_cursor_plane = req->value; break; - case DRM_CLIENT_CAP_PLANE_COLOR_PIPELINE: + case DRM_CLIENT_CAP_PLANE_COLOR_PIPELINE: { + struct drm_plane *plane; + bool has_plane_with_color_pipeline = false; + if (!file_priv->atomic) return -EINVAL; if (req->value > 1) return -EINVAL; + drm_for_each_plane(plane, dev) { + if (plane->color_pipeline_property) { + has_plane_with_color_pipeline = true; + break; + } + } + if (!has_plane_with_color_pipeline) + return -EOPNOTSUPP; file_priv->plane_color_pipeline = req->value; break; + } default: return -EINVAL; } diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index 8d9fd1917c6e6..c9dbf64c0c9f2 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -442,13 +442,15 @@ int drm_syncobj_find_fence(struct drm_file *file_private, u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT); int ret; - if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) - return -EINVAL; - if (!syncobj) return -ENOENT; - /* Waiting for userspace with locks help is illegal cause that can + if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { + ret = -EINVAL; + goto out; + } + + /* Waiting for userspace with locks held is illegal cause that can * trivial deadlock with page faults for example. Make lockdep complain * about it early on. */ diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h index f9ee7ebfec55c..f53dac256ee0b 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h @@ -69,5 +69,6 @@ int hibmc_dp_link_training(struct hibmc_dp_dev *dp); int hibmc_dp_serdes_init(struct hibmc_dp_dev *dp); int hibmc_dp_serdes_rate_switch(u8 rate, struct hibmc_dp_dev *dp); int hibmc_dp_serdes_set_tx_cfg(struct hibmc_dp_dev *dp, u8 train_set[HIBMC_DP_LANE_NUM_MAX]); +void hibmc_dp_update_caps(struct hibmc_dp_dev *dp); #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h index 31316fe1ea8db..0f3662d8737e9 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h @@ -55,6 +55,7 @@ struct hibmc_dp { struct drm_dp_aux aux; struct hibmc_dp_cbar_cfg cfg; u32 irq_status; + int phys_status; }; int hibmc_dp_hw_init(struct hibmc_dp *dp); diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c index 0726cb5b736e6..8c53f16db5160 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c @@ -325,7 +325,7 @@ static int hibmc_dp_link_downgrade_training_eq(struct hibmc_dp_dev *dp) return hibmc_dp_link_reduce_rate(dp); } -static void hibmc_dp_update_caps(struct hibmc_dp_dev *dp) +void hibmc_dp_update_caps(struct hibmc_dp_dev *dp) { dp->link.cap.link_rate = dp->dpcd[DP_MAX_LINK_RATE]; if (dp->link.cap.link_rate > DP_LINK_BW_8_1 || !dp->link.cap.link_rate) diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c index 89bed78f14666..db7fce4e8cc34 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c @@ -32,26 +32,43 @@ struct hibmc_display_panel_pll { struct hibmc_dislay_pll_config { u64 hdisplay; u64 vdisplay; + int clock; u32 pll1_config_value; u32 pll2_config_value; }; static const struct hibmc_dislay_pll_config hibmc_pll_table[] = { - {640, 480, CRT_PLL1_HS_25MHZ, CRT_PLL2_HS_25MHZ}, - {800, 600, CRT_PLL1_HS_40MHZ, CRT_PLL2_HS_40MHZ}, - {1024, 768, CRT_PLL1_HS_65MHZ, CRT_PLL2_HS_65MHZ}, - {1152, 864, CRT_PLL1_HS_80MHZ_1152, CRT_PLL2_HS_80MHZ}, - {1280, 768, CRT_PLL1_HS_80MHZ, CRT_PLL2_HS_80MHZ}, - {1280, 720, CRT_PLL1_HS_74MHZ, CRT_PLL2_HS_74MHZ}, - {1280, 960, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ}, - {1280, 1024, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ}, - {1440, 900, CRT_PLL1_HS_106MHZ, CRT_PLL2_HS_106MHZ}, - {1600, 900, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ}, - {1600, 1200, CRT_PLL1_HS_162MHZ, CRT_PLL2_HS_162MHZ}, - {1920, 1080, CRT_PLL1_HS_148MHZ, CRT_PLL2_HS_148MHZ}, - {1920, 1200, CRT_PLL1_HS_193MHZ, CRT_PLL2_HS_193MHZ}, + {640, 480, 25000, CRT_PLL1_HS_25MHZ, CRT_PLL2_HS_25MHZ}, + {800, 600, 40000, CRT_PLL1_HS_40MHZ, CRT_PLL2_HS_40MHZ}, + {1024, 768, 65000, CRT_PLL1_HS_65MHZ, CRT_PLL2_HS_65MHZ}, + {1152, 864, 78750, CRT_PLL1_HS_80MHZ_1152, CRT_PLL2_HS_80MHZ}, + {1280, 768, 80000, CRT_PLL1_HS_80MHZ, CRT_PLL2_HS_80MHZ}, + {1280, 720, 74375, CRT_PLL1_HS_74MHZ, CRT_PLL2_HS_74MHZ}, + {1280, 960, 108000, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ}, + {1280, 1024, 108000, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ}, + {1440, 900, 105952, CRT_PLL1_HS_106MHZ, CRT_PLL2_HS_106MHZ}, + {1600, 900, 108000, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ}, + {1600, 1200, 162500, CRT_PLL1_HS_162MHZ, CRT_PLL2_HS_162MHZ}, + {1920, 1080, 148750, CRT_PLL1_HS_148MHZ, CRT_PLL2_HS_148MHZ}, + {1920, 1200, 193750, CRT_PLL1_HS_193MHZ, CRT_PLL2_HS_193MHZ}, }; +static int hibmc_get_best_clock_idx(const struct drm_display_mode *mode) +{ + int i, diff; + + for (i = 0; i < ARRAY_SIZE(hibmc_pll_table); i++) { + if (hibmc_pll_table[i].hdisplay == mode->hdisplay && + hibmc_pll_table[i].vdisplay == mode->vdisplay) { + diff = abs(mode->clock - hibmc_pll_table[i].clock); + if (diff < mode->clock / 100) /* tolerance 1/100 */ + return i; + } + } + + return -MODE_CLOCK_RANGE; +} + static int hibmc_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_state *state) { @@ -214,19 +231,15 @@ static enum drm_mode_status hibmc_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode) { - size_t i = 0; int vrefresh = drm_mode_vrefresh(mode); if (vrefresh < 59 || vrefresh > 61) return MODE_NOCLOCK; - for (i = 0; i < ARRAY_SIZE(hibmc_pll_table); i++) { - if (hibmc_pll_table[i].hdisplay == mode->hdisplay && - hibmc_pll_table[i].vdisplay == mode->vdisplay) - return MODE_OK; - } + if (hibmc_get_best_clock_idx(mode) >= 0) + return MODE_OK; - return MODE_BAD; + return MODE_CLOCK_RANGE; } static u32 format_pll_reg(void) @@ -281,23 +294,20 @@ static void set_vclock_hisilicon(struct drm_device *dev, u64 pll) writel(val, priv->mmio + CRT_PLL1_HS); } -static void get_pll_config(u64 x, u64 y, u32 *pll1, u32 *pll2) +static void get_pll_config(struct drm_display_mode *mode, u32 *pll1, u32 *pll2) { - size_t i; - size_t count = ARRAY_SIZE(hibmc_pll_table); - - for (i = 0; i < count; i++) { - if (hibmc_pll_table[i].hdisplay == x && - hibmc_pll_table[i].vdisplay == y) { - *pll1 = hibmc_pll_table[i].pll1_config_value; - *pll2 = hibmc_pll_table[i].pll2_config_value; - return; - } + int idx; + + idx = hibmc_get_best_clock_idx(mode); + if (idx < 0) { + /* if found none, we use default value */ + *pll1 = CRT_PLL1_HS_25MHZ; + *pll2 = CRT_PLL2_HS_25MHZ; + return; } - /* if found none, we use default value */ - *pll1 = CRT_PLL1_HS_25MHZ; - *pll2 = CRT_PLL2_HS_25MHZ; + *pll1 = hibmc_pll_table[idx].pll1_config_value; + *pll2 = hibmc_pll_table[idx].pll2_config_value; } /* @@ -319,7 +329,7 @@ static u32 display_ctrl_adjust(struct drm_device *dev, x = mode->hdisplay; y = mode->vdisplay; - get_pll_config(x, y, &pll1, &pll2); + get_pll_config(mode, &pll1, &pll2); writel(pll2, priv->mmio + CRT_PLL2_HS); set_vclock_hisilicon(dev, pll1); diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c index 616821e3c933b..596c5bfe32d8e 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c @@ -41,6 +41,8 @@ static bool hibmc_dp_get_dpcd(struct hibmc_dp_dev *dp_dev) if (ret) return false; + hibmc_dp_update_caps(dp_dev); + dp_dev->is_branch = drm_dp_is_branch(dp_dev->dpcd); ret = drm_dp_read_desc(dp_dev->aux, &dp_dev->desc, dp_dev->is_branch); @@ -59,27 +61,38 @@ static int hibmc_dp_detect(struct drm_connector *connector, { struct hibmc_dp *dp = to_hibmc_dp(connector); struct hibmc_dp_dev *dp_dev = dp->dp_dev; - int ret; + int ret = connector_status_disconnected; if (dp->irq_status) { - if (dp_dev->hpd_status != HIBMC_HPD_IN) - return connector_status_disconnected; + if (dp_dev->hpd_status != HIBMC_HPD_IN) { + ret = connector_status_disconnected; + goto exit; + } } - if (!hibmc_dp_get_dpcd(dp_dev)) - return connector_status_disconnected; + if (!hibmc_dp_get_dpcd(dp_dev)) { + ret = connector_status_disconnected; + goto exit; + } - if (!dp_dev->is_branch) - return connector_status_connected; + if (!dp_dev->is_branch) { + ret = connector_status_connected; + goto exit; + } if (drm_dp_read_sink_count_cap(connector, dp_dev->dpcd, &dp_dev->desc) && dp_dev->downstream_ports[0] & DP_DS_PORT_HPD) { ret = drm_dp_read_sink_count(dp_dev->aux); - if (ret > 0) - return connector_status_connected; + if (ret > 0) { + ret = connector_status_connected; + goto exit; + } } - return connector_status_disconnected; +exit: + dp->phys_status = ret; + + return ret; } static int hibmc_dp_mode_valid(struct drm_connector *connector, @@ -241,5 +254,7 @@ int hibmc_dp_init(struct hibmc_drm_private *priv) connector->polled = DRM_CONNECTOR_POLL_HPD; + dp->phys_status = connector_status_disconnected; + return 0; } diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c index 289304500ab09..99b36de1fe137 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "hibmc_drm_drv.h" #include "hibmc_drm_regs.h" @@ -214,6 +215,15 @@ void hibmc_set_current_gate(struct hibmc_drm_private *priv, unsigned int gate) writel(gate, mmio + gate_reg); } +static void hibmc_display_ctrl(struct hibmc_drm_private *priv) +{ + u32 reg; + + reg = readl(priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE); + reg |= HIBMC_DISPLAY_CONTROL_PANELDATE(1); + writel(reg, priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE); +} + static void hibmc_hw_config(struct hibmc_drm_private *priv) { u32 reg; @@ -245,6 +255,8 @@ static void hibmc_hw_config(struct hibmc_drm_private *priv) reg |= HIBMC_MSCCTL_LOCALMEM_RESET(1); writel(reg, priv->mmio + HIBMC_MISC_CTRL); + + hibmc_display_ctrl(priv); } static int hibmc_hw_map(struct hibmc_drm_private *priv) @@ -355,6 +367,8 @@ static int hibmc_load(struct drm_device *dev) /* reset all the states of crtc/plane/encoder/connector */ drm_mode_config_reset(dev); + drmm_kms_helper_poll_init(dev); + return 0; err: diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h index ca8502e2760c1..cd3a3fca1fe60 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h @@ -31,6 +31,7 @@ struct hibmc_vdac { struct drm_connector connector; struct i2c_adapter adapter; struct i2c_algo_bit_data bit_data; + int phys_status; }; struct hibmc_drm_private { diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c index 841e81f47b686..b9bd6d33fb0f7 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c @@ -24,28 +24,21 @@ static int hibmc_connector_get_modes(struct drm_connector *connector) { + struct drm_mode_config *mode_config = &connector->dev->mode_config; struct hibmc_vdac *vdac = to_hibmc_vdac(connector); - const struct drm_edid *drm_edid; int count; - drm_edid = drm_edid_read_ddc(connector, &vdac->adapter); - - drm_edid_connector_update(connector, drm_edid); - - if (drm_edid) { - count = drm_edid_connector_add_modes(connector); + if (vdac->phys_status == connector_status_connected) { + count = drm_connector_helper_get_modes(connector); + } else { + drm_edid_connector_update(connector, NULL); + count = drm_add_modes_noedid(connector, + mode_config->max_width, + mode_config->max_height); if (count) - goto out; + drm_set_preferred_mode(connector, 1024, 768); } - count = drm_add_modes_noedid(connector, - connector->dev->mode_config.max_width, - connector->dev->mode_config.max_height); - drm_set_preferred_mode(connector, 1024, 768); - -out: - drm_edid_free(drm_edid); - return count; } @@ -57,10 +50,34 @@ static void hibmc_connector_destroy(struct drm_connector *connector) drm_connector_cleanup(connector); } +static int hibmc_vdac_detect(struct drm_connector *connector, + struct drm_modeset_acquire_ctx *ctx, + bool force) +{ + struct hibmc_drm_private *priv = to_hibmc_drm_private(connector->dev); + int status = drm_connector_helper_detect_from_ddc(connector, ctx, + force); + struct hibmc_vdac *vdac = to_hibmc_vdac(connector); + + if (priv->dp.phys_status == connector_status_connected) { + vdac->phys_status = status; + return status; + } + + if (status != vdac->phys_status) + ++connector->epoch_counter; + vdac->phys_status = status; + + /* When both the DP and VDAC physical status are disconnected, + * the "connected" status is returned to support KVM display. + */ + return connector_status_connected; +} + static const struct drm_connector_helper_funcs hibmc_connector_helper_funcs = { .get_modes = hibmc_connector_get_modes, - .detect_ctx = drm_connector_helper_detect_from_ddc, + .detect_ctx = hibmc_vdac_detect, }; static const struct drm_connector_funcs hibmc_connector_funcs = { @@ -71,26 +88,6 @@ static const struct drm_connector_funcs hibmc_connector_funcs = { .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, }; -static void hibmc_encoder_mode_set(struct drm_encoder *encoder, - struct drm_display_mode *mode, - struct drm_display_mode *adj_mode) -{ - u32 reg; - struct drm_device *dev = encoder->dev; - struct hibmc_drm_private *priv = to_hibmc_drm_private(dev); - - reg = readl(priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE); - reg |= HIBMC_DISPLAY_CONTROL_FPVDDEN(1); - reg |= HIBMC_DISPLAY_CONTROL_PANELDATE(1); - reg |= HIBMC_DISPLAY_CONTROL_FPEN(1); - reg |= HIBMC_DISPLAY_CONTROL_VBIASEN(1); - writel(reg, priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE); -} - -static const struct drm_encoder_helper_funcs hibmc_encoder_helper_funcs = { - .mode_set = hibmc_encoder_mode_set, -}; - int hibmc_vdac_init(struct hibmc_drm_private *priv) { struct drm_device *dev = &priv->dev; @@ -113,8 +110,6 @@ int hibmc_vdac_init(struct hibmc_drm_private *priv) goto err; } - drm_encoder_helper_add(encoder, &hibmc_encoder_helper_funcs); - ret = drm_connector_init_with_ddc(dev, connector, &hibmc_connector_funcs, DRM_MODE_CONNECTOR_VGA, @@ -130,6 +125,8 @@ int hibmc_vdac_init(struct hibmc_drm_private *priv) connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT; + vdac->phys_status = connector_status_disconnected; + return 0; err: diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c index 71b7325917b6e..a16e1cb7d41e5 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.c +++ b/drivers/gpu/drm/i915/display/intel_atomic.c @@ -289,6 +289,12 @@ static void intel_crtc_put_color_blobs(struct intel_crtc_state *crtc_state) drm_property_blob_put(crtc_state->pre_csc_lut); drm_property_blob_put(crtc_state->post_csc_lut); + + crtc_state->hw.degamma_lut = NULL; + crtc_state->hw.gamma_lut = NULL; + crtc_state->hw.ctm = NULL; + crtc_state->pre_csc_lut = NULL; + crtc_state->post_csc_lut = NULL; } void intel_crtc_free_hw_state(struct intel_crtc_state *crtc_state) diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy.c b/drivers/gpu/drm/i915/display/intel_lt_phy.c index 657ad5cb0eff9..af5391ba855f1 100644 --- a/drivers/gpu/drm/i915/display/intel_lt_phy.c +++ b/drivers/gpu/drm/i915/display/intel_lt_phy.c @@ -1223,11 +1223,7 @@ intel_lt_phy_program_port_clock_ctl(struct intel_encoder *encoder, else val |= XELPDP_DDI_CLOCK_SELECT_PREP(display, XELPDP_DDI_CLOCK_SELECT_MAXPCLK); - /* DP2.0 10G and 20G rates enable MPLLA*/ - if (port_clock == 1000000 || port_clock == 2000000) - val |= XELPDP_SSC_ENABLE_PLLA; - else - val |= ltpll->ssc_enabled ? XELPDP_SSC_ENABLE_PLLB : 0; + val |= ltpll->ssc_enabled ? XELPDP_SSC_ENABLE_PLLA : 0; intel_de_rmw(display, XELPDP_PORT_CLOCK_CTL(display, encoder->port), XELPDP_LANE1_PHY_CLOCK_SELECT | XELPDP_FORWARD_CLOCK_UNGATE | diff --git a/drivers/gpu/drm/imagination/pvr_drv.c b/drivers/gpu/drm/imagination/pvr_drv.c index 268900464ab69..cfb67f77bde6c 100644 --- a/drivers/gpu/drm/imagination/pvr_drv.c +++ b/drivers/gpu/drm/imagination/pvr_drv.c @@ -514,7 +514,8 @@ pvr_dev_query_quirks_get(struct pvr_device *pvr_dev, if (err < 0) return err; - args->size = sizeof(query); + if (args->size > sizeof(query)) + args->size = sizeof(query); return 0; } @@ -595,7 +596,8 @@ pvr_dev_query_enhancements_get(struct pvr_device *pvr_dev, if (err < 0) return err; - args->size = sizeof(query); + if (args->size > sizeof(query)) + args->size = sizeof(query); return 0; } diff --git a/drivers/gpu/drm/imagination/pvr_fw_trace.c b/drivers/gpu/drm/imagination/pvr_fw_trace.c index 6193811ef7beb..5b2218a7cd663 100644 --- a/drivers/gpu/drm/imagination/pvr_fw_trace.c +++ b/drivers/gpu/drm/imagination/pvr_fw_trace.c @@ -71,7 +71,7 @@ pvr_fw_trace_init_mask_set(const char *val, const struct kernel_param *kp) return 0; } -const struct kernel_param_ops pvr_fw_trace_init_mask_ops = { +static const struct kernel_param_ops pvr_fw_trace_init_mask_ops = { .set = pvr_fw_trace_init_mask_set, .get = param_get_hexint, }; diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c index e1ec60f34b6e6..396d349fb6ce4 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.c +++ b/drivers/gpu/drm/imagination/pvr_vm.c @@ -1019,7 +1019,8 @@ pvr_static_data_areas_get(const struct pvr_device *pvr_dev, if (err < 0) return err; - args->size = sizeof(query); + if (args->size > sizeof(query)) + args->size = sizeof(query); return 0; } @@ -1069,7 +1070,8 @@ pvr_heap_info_get(const struct pvr_device *pvr_dev, if (err < 0) return err; - args->size = sizeof(query); + if (args->size > sizeof(query)) + args->size = sizeof(query); return 0; } diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 1b44b9e21ad86..cab4c46c6cf2e 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -2357,7 +2357,12 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) goto err_mmio; } } else if (adreno_is_a8xx(adreno_gpu)) { - gmu->rscc = gmu->mmio + 0x19000; + /* + * On a8xx , RSCC lives at GPU base + 0x50000, which falls + * inside the GPU's kgsl_3d0_reg_memory range rather than the + * GMU's. + */ + gmu->rscc = gpu->mmio + 0x50000; } else { gmu->rscc = gmu->mmio + 0x23000; } diff --git a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c index ccfccc45133fd..9b99ec5ceeb58 100644 --- a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c @@ -886,17 +886,22 @@ void a8xx_recover(struct msm_gpu *gpu) adreno_dump_info(gpu); - if (hang_debug) - a8xx_dump(gpu); - /* * To handle recovery specific sequences during the rpm suspend we are * about to trigger */ a6xx_gpu->hung = true; - /* Halt SQE first */ - gpu_write(gpu, REG_A8XX_CP_SQE_CNTL, 3); + if (adreno_gpu->funcs->gx_is_on(adreno_gpu)) { + /* + * Sometimes crashstate capture is skipped, so SQE should be + * halted here again + */ + gpu_write(gpu, REG_A8XX_CP_SQE_CNTL, 3); + + if (hang_debug) + a8xx_dump(gpu); + } pm_runtime_dont_use_autosuspend(&gpu->pdev->dev); diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index d2124d6254855..cf2415635c426 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -38,9 +38,9 @@ enum { ISR_DISCONNECTED, ISR_CONNECT_PENDING, ISR_CONNECTED, - ISR_HPD_REPLUG_COUNT, + ISR_HPD_IO_GLITCH_COUNT, ISR_IRQ_HPD_PULSE_COUNT, - ISR_HPD_LO_GLITH_COUNT, + ISR_HPD_REPLUG_COUNT, }; /* event thread connection state */ diff --git a/drivers/gpu/drm/msm/dp/dp_reg.h b/drivers/gpu/drm/msm/dp/dp_reg.h index 7c44d4e2cf139..3689642b7fc06 100644 --- a/drivers/gpu/drm/msm/dp/dp_reg.h +++ b/drivers/gpu/drm/msm/dp/dp_reg.h @@ -68,8 +68,8 @@ #define DP_DP_IRQ_HPD_INT_ACK (0x00000002) #define DP_DP_HPD_REPLUG_INT_ACK (0x00000004) #define DP_DP_HPD_UNPLUG_INT_ACK (0x00000008) -#define DP_DP_HPD_STATE_STATUS_BITS_MASK (0x0000000F) -#define DP_DP_HPD_STATE_STATUS_BITS_SHIFT (0x1C) +#define DP_DP_HPD_STATE_STATUS_BITS_MASK (0x00000007) +#define DP_DP_HPD_STATE_STATUS_BITS_SHIFT (0x1D) #define REG_DP_DP_HPD_INT_MASK (0x0000000C) #define DP_DP_HPD_PLUG_INT_MASK (0x00000001) diff --git a/drivers/gpu/drm/msm/msm_mdss.c b/drivers/gpu/drm/msm/msm_mdss.c index 90c3fa0681a06..4a6acd468bc84 100644 --- a/drivers/gpu/drm/msm/msm_mdss.c +++ b/drivers/gpu/drm/msm/msm_mdss.c @@ -166,22 +166,19 @@ static int _msm_mdss_irq_domain_add(struct msm_mdss *msm_mdss) return 0; } -static void msm_mdss_setup_ubwc_dec_20(struct msm_mdss *msm_mdss) +static void msm_mdss_4x_setup_ubwc(struct msm_mdss *msm_mdss) { const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; - u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) | + u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle & 0x1) | MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13); - if (data->ubwc_bank_spread) - value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD; - if (data->ubwc_enc_version == UBWC_1_0) value |= MDSS_UBWC_STATIC_UBWC_MIN_ACC_LEN(1); writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC); } -static void msm_mdss_setup_ubwc_dec_30(struct msm_mdss *msm_mdss) +static void msm_mdss_5x_setup_ubwc(struct msm_mdss *msm_mdss) { const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle & 0x1) | @@ -199,11 +196,12 @@ static void msm_mdss_setup_ubwc_dec_30(struct msm_mdss *msm_mdss) writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC); } -static void msm_mdss_setup_ubwc_dec_40(struct msm_mdss *msm_mdss) +static void msm_mdss_6x_setup_ubwc(struct msm_mdss *msm_mdss) { const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) | MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13); + u32 ver, prediction_mode; if (data->ubwc_bank_spread) value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD; @@ -211,45 +209,42 @@ static void msm_mdss_setup_ubwc_dec_40(struct msm_mdss *msm_mdss) if (data->macrotile_mode) value |= MDSS_UBWC_STATIC_MACROTILE_MODE; - writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC); - - if (data->ubwc_enc_version == UBWC_3_0) { - writel_relaxed(1, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2); - writel_relaxed(0, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE); - } else { - if (data->ubwc_dec_version == UBWC_4_3) - writel_relaxed(3, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2); - else - writel_relaxed(2, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2); - writel_relaxed(1, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE); - } -} - -static void msm_mdss_setup_ubwc_dec_50(struct msm_mdss *msm_mdss) -{ - const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; - u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) | - MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13); - - if (data->ubwc_bank_spread) - value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD; - - if (data->macrotile_mode) - value |= MDSS_UBWC_STATIC_MACROTILE_MODE; + if (data->ubwc_enc_version == UBWC_1_0) + value |= MDSS_UBWC_STATIC_UBWC_MIN_ACC_LEN(1); writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC); - if (data->ubwc_dec_version == UBWC_6_0) - writel_relaxed(5, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2); + if (data->ubwc_enc_version < UBWC_4_0) + prediction_mode = 0; else - writel_relaxed(4, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2); - - writel_relaxed(1, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE); + prediction_mode = 1; + + if (data->ubwc_enc_version >= UBWC_6_0) + ver = 5; + else if (data->ubwc_enc_version >= UBWC_5_0) + ver = 4; + else if (data->ubwc_enc_version >= UBWC_4_3) + ver = 3; + else if (data->ubwc_enc_version >= UBWC_4_0) + ver = 2; + else if (data->ubwc_enc_version >= UBWC_3_0) + ver = 1; + else /* UBWC 1.0 and 2.0 */ + ver = 0; + + writel_relaxed(ver, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2); + writel_relaxed(prediction_mode, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE); } +#define MDSS_HW_VER(major, minor, step) \ + ((((major) & 0xf) << 28) | \ + (((minor) & 0xfff) << 16) | \ + ((step) & 0xffff)) + static int msm_mdss_enable(struct msm_mdss *msm_mdss) { int ret, i; + u32 hw_rev; /* * Several components have AXI clocks that can only be turned on if @@ -283,43 +278,15 @@ static int msm_mdss_enable(struct msm_mdss *msm_mdss) if (msm_mdss->is_mdp5 || !msm_mdss->mdss_data) return 0; - /* - * ubwc config is part of the "mdss" region which is not accessible - * from the rest of the driver. hardcode known configurations here - * - * Decoder version can be read from the UBWC_DEC_HW_VERSION reg, - * UBWC_n and the rest of params comes from hw data. - */ - switch (msm_mdss->mdss_data->ubwc_dec_version) { - case 0: /* no UBWC */ - case UBWC_1_0: - /* do nothing */ - break; - case UBWC_2_0: - msm_mdss_setup_ubwc_dec_20(msm_mdss); - break; - case UBWC_3_0: - msm_mdss_setup_ubwc_dec_30(msm_mdss); - break; - case UBWC_4_0: - case UBWC_4_3: - msm_mdss_setup_ubwc_dec_40(msm_mdss); - break; - case UBWC_5_0: - msm_mdss_setup_ubwc_dec_50(msm_mdss); - break; - case UBWC_6_0: - msm_mdss_setup_ubwc_dec_50(msm_mdss); - break; - default: - dev_err(msm_mdss->dev, "Unsupported UBWC decoder version %x\n", - msm_mdss->mdss_data->ubwc_dec_version); - dev_err(msm_mdss->dev, "HW_REV: 0x%x\n", - readl_relaxed(msm_mdss->mmio + REG_MDSS_HW_VERSION)); - dev_err(msm_mdss->dev, "UBWC_DEC_HW_VERSION: 0x%x\n", - readl_relaxed(msm_mdss->mmio + REG_MDSS_UBWC_DEC_HW_VERSION)); - break; - } + hw_rev = readl_relaxed(msm_mdss->mmio + REG_MDSS_HW_VERSION); + + if (hw_rev >= MDSS_HW_VER(6, 0, 0)) + msm_mdss_6x_setup_ubwc(msm_mdss); + else if (hw_rev >= MDSS_HW_VER(5, 0, 0)) + msm_mdss_5x_setup_ubwc(msm_mdss); + else if (hw_rev >= MDSS_HW_VER(4, 0, 0)) + msm_mdss_4x_setup_ubwc(msm_mdss); + /* else UBWC 1.0 or none, no params to program */ return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c index d5411d176e3a8..0d9e6cdd61193 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c @@ -65,13 +65,14 @@ pramin_init(struct nvkm_bios *bios, const char *name) /* we can't get the bios image pointer without PDISP */ if (device->card_type >= GA100) - addr = device->chipset == 0x170; /*XXX: find the fuse reg for this */ + addr = nvkm_rd32(device, 0x820c04); else if (device->card_type >= GM100) addr = nvkm_rd32(device, 0x021c04); else if (device->card_type >= NV_C0) addr = nvkm_rd32(device, 0x022500); + if (addr & 0x00000001) { nvkm_debug(subdev, "... display disabled\n"); return ERR_PTR(-ENODEV); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 958fd78080bdb..c2d854c7af99c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -230,29 +230,26 @@ nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, * covered by a number of LPTEs, the LPTEs once again take * control over their address range. * - * Determine how many LPTEs need to transition state. + * Transition each LPTE individually as each may have a + * different target state (sparse, invalid, or valid). */ - pgt->pte[ptei].s.spte_valid = false; - for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { + for (ptei++; ptei < lpti; ptei++) { if (pgt->pte[ptei].s.sptes) break; - pgt->pte[ptei].s.spte_valid = false; } - if (pgt->pte[pteb].s.sparse) { - TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes); - pair->func->sparse(vmm, pgt->pt[0], pteb, ptes); - } else if (!pgt->pte[pteb].s.lpte_valid) { - if (pair->func->invalid) { - /* If the MMU supports it, restore the LPTE to the - * INVALID state to tell the MMU there is no point - * trying to fetch the corresponding SPTEs. - */ - TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes); - pair->func->invalid(vmm, pgt->pt[0], pteb, ptes); + while (pteb < ptei) { + pgt->pte[pteb].s.spte_valid = false; + if (pgt->pte[pteb].s.sparse) { + TRA(it, "LPTE %05x: U -> S", pteb); + pair->func->sparse(vmm, pgt->pt[0], pteb, 1); + } else if (!pgt->pte[pteb].s.lpte_valid) { + if (pair->func->invalid) { + TRA(it, "LPTE %05x: U -> I", pteb); + pair->func->invalid(vmm, pgt->pt[0], pteb, 1); + } } - } else { - TRA(it, "LPTE %05x: V %d PTEs", pteb, ptes); + pteb++; } } } diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig index d592f4f4b939a..b2153e04a59af 100644 --- a/drivers/gpu/drm/panel/Kconfig +++ b/drivers/gpu/drm/panel/Kconfig @@ -890,10 +890,10 @@ config DRM_PANEL_SAMSUNG_S6D7AA0 config DRM_PANEL_SAMSUNG_S6E3FC2X01 tristate "Samsung S6E3FC2X01 DSI panel controller" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE - select VIDEOMODE_HELPERS help Say Y or M here if you want to enable support for the Samsung S6E3FC2 DDIC and connected MIPI DSI panel. @@ -910,11 +910,18 @@ config DRM_PANEL_SAMSUNG_S6E3HA2 config DRM_PANEL_SAMSUNG_S6E3HA8 tristate "Samsung S6E3HA8 DSI video mode panel" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE select DRM_DISPLAY_DSC_HELPER - select VIDEOMODE_HELPERS + help + Say Y or M here if you want to enable support for the + Samsung S6E3HA8 DDIC and connected MIPI DSI panel. + Currently supported panels: + + Samsung AMB577PX01 (found in the Samsung S9 smartphone) + config DRM_PANEL_SAMSUNG_S6E63J0X03 tristate "Samsung S6E63J0X03 DSI command mode panel" @@ -982,10 +989,10 @@ config DRM_PANEL_SAMSUNG_S6E8FC0 config DRM_PANEL_SAMSUNG_SOFEF00 tristate "Samsung SOFEF00 DSI panel controller" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE - select VIDEOMODE_HELPERS help Say Y or M here if you want to enable support for the Samsung AMOLED panel SOFEF00 DDIC and connected panel. diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c index 54fbb1aa07c52..4f522a912d896 100644 --- a/drivers/gpu/drm/panthor/panthor_device.c +++ b/drivers/gpu/drm/panthor/panthor_device.c @@ -18,12 +18,13 @@ #include "panthor_devfreq.h" #include "panthor_device.h" #include "panthor_fw.h" +#include "panthor_fw_regs.h" #include "panthor_gem.h" #include "panthor_gpu.h" +#include "panthor_gpu_regs.h" #include "panthor_hw.h" #include "panthor_mmu.h" #include "panthor_pwr.h" -#include "panthor_regs.h" #include "panthor_sched.h" static int panthor_gpu_coherency_init(struct panthor_device *ptdev) @@ -42,7 +43,7 @@ static int panthor_gpu_coherency_init(struct panthor_device *ptdev) /* Check if the ACE-Lite coherency protocol is actually supported by the GPU. * ACE protocol has never been supported for command stream frontend GPUs. */ - if ((gpu_read(ptdev, GPU_COHERENCY_FEATURES) & + if ((gpu_read(ptdev->iomem, GPU_COHERENCY_FEATURES) & GPU_COHERENCY_PROT_BIT(ACE_LITE))) { ptdev->gpu_info.selected_coherency = GPU_COHERENCY_ACE_LITE; return 0; @@ -231,6 +232,7 @@ int panthor_device_init(struct panthor_device *ptdev) *dummy_page_virt = 1; INIT_WORK(&ptdev->reset.work, panthor_device_reset_work); + disable_work(&ptdev->reset.work); ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0); if (!ptdev->reset.wq) return -ENOMEM; @@ -305,6 +307,9 @@ int panthor_device_init(struct panthor_device *ptdev) panthor_gem_init(ptdev); + /* Now that everything is initialized, we can enable the reset work. */ + enable_work(&ptdev->reset.work); + /* ~3 frames */ pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50); pm_runtime_use_autosuspend(ptdev->base.dev); diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h index b6696f73a5367..724b4aeaf23e1 100644 --- a/drivers/gpu/drm/panthor/panthor_device.h +++ b/drivers/gpu/drm/panthor/panthor_device.h @@ -81,6 +81,9 @@ struct panthor_irq { /** @ptdev: Panthor device */ struct panthor_device *ptdev; + /** @iomem: CPU mapping of IRQ base address */ + void __iomem *iomem; + /** @irq: IRQ number. */ int irq; @@ -415,6 +418,11 @@ panthor_exception_is_fault(u32 exception_code) const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code); +#define INT_RAWSTAT 0x0 +#define INT_CLEAR 0x4 +#define INT_MASK 0x8 +#define INT_STAT 0xc + /** * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt * registration function. @@ -425,16 +433,12 @@ const char *panthor_exception_name(struct panthor_device *ptdev, * * void (*handler)(struct panthor_device *, u32 status); */ -#define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler) \ +#define PANTHOR_IRQ_HANDLER(__name, __handler) \ static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data) \ { \ struct panthor_irq *pirq = data; \ - struct panthor_device *ptdev = pirq->ptdev; \ enum panthor_irq_state old_state; \ \ - if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT)) \ - return IRQ_NONE; \ - \ guard(spinlock_irqsave)(&pirq->mask_lock); \ old_state = atomic_cmpxchg(&pirq->state, \ PANTHOR_IRQ_STATE_ACTIVE, \ @@ -442,7 +446,14 @@ static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data) if (old_state != PANTHOR_IRQ_STATE_ACTIVE) \ return IRQ_NONE; \ \ - gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0); \ + if (!gpu_read(pirq->iomem, INT_STAT)) { \ + atomic_cmpxchg(&pirq->state, \ + PANTHOR_IRQ_STATE_PROCESSING, \ + PANTHOR_IRQ_STATE_ACTIVE); \ + return IRQ_NONE; \ + } \ + \ + gpu_write(pirq->iomem, INT_MASK, 0); \ return IRQ_WAKE_THREAD; \ } \ \ @@ -461,7 +472,7 @@ static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *da * right before the HW event kicks in. TLDR; it's all expected races we're \ * covered for. \ */ \ - u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask; \ + u32 status = gpu_read(pirq->iomem, INT_RAWSTAT) & pirq->mask; \ \ if (!status) \ break; \ @@ -477,7 +488,7 @@ static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *da PANTHOR_IRQ_STATE_PROCESSING, \ PANTHOR_IRQ_STATE_ACTIVE); \ if (old_state == PANTHOR_IRQ_STATE_PROCESSING) \ - gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ + gpu_write(pirq->iomem, INT_MASK, pirq->mask); \ } \ \ return ret; \ @@ -487,7 +498,7 @@ static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) { \ scoped_guard(spinlock_irqsave, &pirq->mask_lock) { \ atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDING); \ - gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ + gpu_write(pirq->iomem, INT_MASK, 0); \ } \ synchronize_irq(pirq->irq); \ atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDED); \ @@ -498,17 +509,18 @@ static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq) guard(spinlock_irqsave)(&pirq->mask_lock); \ \ atomic_set(&pirq->state, PANTHOR_IRQ_STATE_ACTIVE); \ - gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, pirq->mask); \ - gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ + gpu_write(pirq->iomem, INT_CLEAR, pirq->mask); \ + gpu_write(pirq->iomem, INT_MASK, pirq->mask); \ } \ \ static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \ struct panthor_irq *pirq, \ - int irq, u32 mask) \ + int irq, u32 mask, void __iomem *iomem) \ { \ pirq->ptdev = ptdev; \ pirq->irq = irq; \ pirq->mask = mask; \ + pirq->iomem = iomem; \ spin_lock_init(&pirq->mask_lock); \ panthor_ ## __name ## _irq_resume(pirq); \ \ @@ -530,7 +542,7 @@ static inline void panthor_ ## __name ## _irq_enable_events(struct panthor_irq * * If the IRQ is suspended/suspending, the mask is restored at resume time. \ */ \ if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE) \ - gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ + gpu_write(pirq->iomem, INT_MASK, pirq->mask); \ } \ \ static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq *pirq, u32 mask)\ @@ -544,80 +556,80 @@ static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq * If the IRQ is suspended/suspending, the mask is restored at resume time. \ */ \ if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE) \ - gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ + gpu_write(pirq->iomem, INT_MASK, pirq->mask); \ } extern struct workqueue_struct *panthor_cleanup_wq; -static inline void gpu_write(struct panthor_device *ptdev, u32 reg, u32 data) +static inline void gpu_write(void __iomem *iomem, u32 reg, u32 data) { - writel(data, ptdev->iomem + reg); + writel(data, iomem + reg); } -static inline u32 gpu_read(struct panthor_device *ptdev, u32 reg) +static inline u32 gpu_read(void __iomem *iomem, u32 reg) { - return readl(ptdev->iomem + reg); + return readl(iomem + reg); } -static inline u32 gpu_read_relaxed(struct panthor_device *ptdev, u32 reg) +static inline u32 gpu_read_relaxed(void __iomem *iomem, u32 reg) { - return readl_relaxed(ptdev->iomem + reg); + return readl_relaxed(iomem + reg); } -static inline void gpu_write64(struct panthor_device *ptdev, u32 reg, u64 data) +static inline void gpu_write64(void __iomem *iomem, u32 reg, u64 data) { - gpu_write(ptdev, reg, lower_32_bits(data)); - gpu_write(ptdev, reg + 4, upper_32_bits(data)); + gpu_write(iomem, reg, lower_32_bits(data)); + gpu_write(iomem, reg + 4, upper_32_bits(data)); } -static inline u64 gpu_read64(struct panthor_device *ptdev, u32 reg) +static inline u64 gpu_read64(void __iomem *iomem, u32 reg) { - return (gpu_read(ptdev, reg) | ((u64)gpu_read(ptdev, reg + 4) << 32)); + return (gpu_read(iomem, reg) | ((u64)gpu_read(iomem, reg + 4) << 32)); } -static inline u64 gpu_read64_relaxed(struct panthor_device *ptdev, u32 reg) +static inline u64 gpu_read64_relaxed(void __iomem *iomem, u32 reg) { - return (gpu_read_relaxed(ptdev, reg) | - ((u64)gpu_read_relaxed(ptdev, reg + 4) << 32)); + return (gpu_read_relaxed(iomem, reg) | + ((u64)gpu_read_relaxed(iomem, reg + 4) << 32)); } -static inline u64 gpu_read64_counter(struct panthor_device *ptdev, u32 reg) +static inline u64 gpu_read64_counter(void __iomem *iomem, u32 reg) { u32 lo, hi1, hi2; do { - hi1 = gpu_read(ptdev, reg + 4); - lo = gpu_read(ptdev, reg); - hi2 = gpu_read(ptdev, reg + 4); + hi1 = gpu_read(iomem, reg + 4); + lo = gpu_read(iomem, reg); + hi2 = gpu_read(iomem, reg + 4); } while (hi1 != hi2); return lo | ((u64)hi2 << 32); } -#define gpu_read_poll_timeout(dev, reg, val, cond, delay_us, timeout_us) \ +#define gpu_read_poll_timeout(iomem, reg, val, cond, delay_us, timeout_us) \ read_poll_timeout(gpu_read, val, cond, delay_us, timeout_us, false, \ - dev, reg) + iomem, reg) -#define gpu_read_poll_timeout_atomic(dev, reg, val, cond, delay_us, \ +#define gpu_read_poll_timeout_atomic(iomem, reg, val, cond, delay_us, \ timeout_us) \ read_poll_timeout_atomic(gpu_read, val, cond, delay_us, timeout_us, \ - false, dev, reg) + false, iomem, reg) -#define gpu_read64_poll_timeout(dev, reg, val, cond, delay_us, timeout_us) \ +#define gpu_read64_poll_timeout(iomem, reg, val, cond, delay_us, timeout_us) \ read_poll_timeout(gpu_read64, val, cond, delay_us, timeout_us, false, \ - dev, reg) + iomem, reg) -#define gpu_read64_poll_timeout_atomic(dev, reg, val, cond, delay_us, \ +#define gpu_read64_poll_timeout_atomic(iomem, reg, val, cond, delay_us, \ timeout_us) \ read_poll_timeout_atomic(gpu_read64, val, cond, delay_us, timeout_us, \ - false, dev, reg) + false, iomem, reg) -#define gpu_read_relaxed_poll_timeout_atomic(dev, reg, val, cond, delay_us, \ +#define gpu_read_relaxed_poll_timeout_atomic(iomem, reg, val, cond, delay_us, \ timeout_us) \ read_poll_timeout_atomic(gpu_read_relaxed, val, cond, delay_us, \ - timeout_us, false, dev, reg) + timeout_us, false, iomem, reg) -#define gpu_read64_relaxed_poll_timeout(dev, reg, val, cond, delay_us, \ +#define gpu_read64_relaxed_poll_timeout(iomem, reg, val, cond, delay_us, \ timeout_us) \ read_poll_timeout(gpu_read64_relaxed, val, cond, delay_us, timeout_us, \ - false, dev, reg) + false, iomem, reg) #endif diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c index 87d27c3c14569..3bf0bf3a6a2b2 100644 --- a/drivers/gpu/drm/panthor/panthor_drv.c +++ b/drivers/gpu/drm/panthor/panthor_drv.c @@ -33,9 +33,9 @@ #include "panthor_fw.h" #include "panthor_gem.h" #include "panthor_gpu.h" +#include "panthor_gpu_regs.h" #include "panthor_heap.h" #include "panthor_mmu.h" -#include "panthor_regs.h" #include "panthor_sched.h" /** @@ -838,7 +838,7 @@ static int panthor_query_timestamp_info(struct panthor_device *ptdev, } if (flags & DRM_PANTHOR_TIMESTAMP_GPU_OFFSET) - arg->timestamp_offset = gpu_read64(ptdev, GPU_TIMESTAMP_OFFSET); + arg->timestamp_offset = gpu_read64(ptdev->iomem, GPU_TIMESTAMP_OFFSET); else arg->timestamp_offset = 0; @@ -853,7 +853,7 @@ static int panthor_query_timestamp_info(struct panthor_device *ptdev, query_start_time = 0; if (flags & DRM_PANTHOR_TIMESTAMP_GPU) - arg->current_timestamp = gpu_read64_counter(ptdev, GPU_TIMESTAMP); + arg->current_timestamp = gpu_read64_counter(ptdev->iomem, GPU_TIMESTAMP); else arg->current_timestamp = 0; @@ -869,7 +869,7 @@ static int panthor_query_timestamp_info(struct panthor_device *ptdev, } if (flags & DRM_PANTHOR_TIMESTAMP_GPU_CYCLE_COUNT) - arg->cycle_count = gpu_read64_counter(ptdev, GPU_CYCLE_COUNT); + arg->cycle_count = gpu_read64_counter(ptdev->iomem, GPU_CYCLE_COUNT); else arg->cycle_count = 0; diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c index 8886002e1d316..834c48e368fc3 100644 --- a/drivers/gpu/drm/panthor/panthor_fw.c +++ b/drivers/gpu/drm/panthor/panthor_fw.c @@ -20,11 +20,11 @@ #include "panthor_device.h" #include "panthor_fw.h" +#include "panthor_fw_regs.h" #include "panthor_gem.h" #include "panthor_gpu.h" #include "panthor_hw.h" #include "panthor_mmu.h" -#include "panthor_regs.h" #include "panthor_sched.h" #include "panthor_trace.h" @@ -1052,7 +1052,7 @@ static void panthor_fw_init_global_iface(struct panthor_device *ptdev) GLB_CFG_POWEROFF_TIMER | GLB_CFG_PROGRESS_TIMER); - gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); + gpu_write(ptdev->iomem, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); /* Kick the watchdog. */ mod_delayed_work(ptdev->reset.wq, &ptdev->fw->watchdog.ping_work, @@ -1067,7 +1067,7 @@ static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status) if (tracepoint_enabled(gpu_job_irq)) start = ktime_get_ns(); - gpu_write(ptdev, JOB_INT_CLEAR, status); + gpu_write(ptdev->iomem, JOB_INT_CLEAR, status); if (!ptdev->fw->booted && (status & JOB_INT_GLOBAL_IF)) ptdev->fw->booted = true; @@ -1086,7 +1086,7 @@ static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status) trace_gpu_job_irq(ptdev->base.dev, status, duration); } } -PANTHOR_IRQ_HANDLER(job, JOB, panthor_job_irq_handler); +PANTHOR_IRQ_HANDLER(job, panthor_job_irq_handler); static int panthor_fw_start(struct panthor_device *ptdev) { @@ -1095,13 +1095,13 @@ static int panthor_fw_start(struct panthor_device *ptdev) ptdev->fw->booted = false; panthor_job_irq_enable_events(&ptdev->fw->irq, ~0); panthor_job_irq_resume(&ptdev->fw->irq); - gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_AUTO); + gpu_write(ptdev->iomem, MCU_CONTROL, MCU_CONTROL_AUTO); if (!wait_event_timeout(ptdev->fw->req_waitqueue, ptdev->fw->booted, msecs_to_jiffies(1000))) { if (!ptdev->fw->booted && - !(gpu_read(ptdev, JOB_INT_STAT) & JOB_INT_GLOBAL_IF)) + !(gpu_read(ptdev->iomem, JOB_INT_STAT) & JOB_INT_GLOBAL_IF)) timedout = true; } @@ -1112,7 +1112,7 @@ static int panthor_fw_start(struct panthor_device *ptdev) [MCU_STATUS_HALT] = "halt", [MCU_STATUS_FATAL] = "fatal", }; - u32 status = gpu_read(ptdev, MCU_STATUS); + u32 status = gpu_read(ptdev->iomem, MCU_STATUS); drm_err(&ptdev->base, "Failed to boot MCU (status=%s)", status < ARRAY_SIZE(status_str) ? status_str[status] : "unknown"); @@ -1126,8 +1126,8 @@ static void panthor_fw_stop(struct panthor_device *ptdev) { u32 status; - gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_DISABLE); - if (gpu_read_poll_timeout(ptdev, MCU_STATUS, status, + gpu_write(ptdev->iomem, MCU_CONTROL, MCU_CONTROL_DISABLE); + if (gpu_read_poll_timeout(ptdev->iomem, MCU_STATUS, status, status == MCU_STATUS_DISABLED, 10, 100000)) drm_err(&ptdev->base, "Failed to stop MCU"); } @@ -1137,7 +1137,7 @@ static bool panthor_fw_mcu_halted(struct panthor_device *ptdev) struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); bool halted; - halted = gpu_read(ptdev, MCU_STATUS) == MCU_STATUS_HALT; + halted = gpu_read(ptdev->iomem, MCU_STATUS) == MCU_STATUS_HALT; if (panthor_fw_has_glb_state(ptdev)) halted &= (GLB_STATE_GET(glb_iface->output->ack) == GLB_STATE_HALT); @@ -1154,7 +1154,7 @@ static void panthor_fw_halt_mcu(struct panthor_device *ptdev) else panthor_fw_update_reqs(glb_iface, req, GLB_HALT, GLB_HALT); - gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); + gpu_write(ptdev->iomem, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); } static bool panthor_fw_wait_mcu_halted(struct panthor_device *ptdev) @@ -1412,7 +1412,7 @@ void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_mask) struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); panthor_fw_toggle_reqs(glb_iface, doorbell_req, doorbell_ack, csg_mask); - gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); + gpu_write(ptdev->iomem, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); } static void panthor_fw_ping_work(struct work_struct *work) @@ -1427,7 +1427,7 @@ static void panthor_fw_ping_work(struct work_struct *work) return; panthor_fw_toggle_reqs(glb_iface, req, ack, GLB_PING); - gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); + gpu_write(ptdev->iomem, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); ret = panthor_fw_glb_wait_acks(ptdev, GLB_PING, &acked, 100); if (ret) { @@ -1463,7 +1463,8 @@ int panthor_fw_init(struct panthor_device *ptdev) if (irq <= 0) return -ENODEV; - ret = panthor_request_job_irq(ptdev, &fw->irq, irq, 0); + ret = panthor_request_job_irq(ptdev, &fw->irq, irq, 0, + ptdev->iomem + JOB_INT_BASE); if (ret) { drm_err(&ptdev->base, "failed to request job irq"); return ret; diff --git a/drivers/gpu/drm/panthor/panthor_fw_regs.h b/drivers/gpu/drm/panthor/panthor_fw_regs.h new file mode 100644 index 0000000000000..eeb41aff249bd --- /dev/null +++ b/drivers/gpu/drm/panthor/panthor_fw_regs.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 or MIT */ +/* Copyright 2026 ARM Limited. All rights reserved. */ + +#ifndef __PANTHOR_FW_REGS_H__ +#define __PANTHOR_FW_REGS_H__ + +#define MCU_CONTROL 0x700 +#define MCU_CONTROL_ENABLE 1 +#define MCU_CONTROL_AUTO 2 +#define MCU_CONTROL_DISABLE 0 + +#define MCU_STATUS 0x704 +#define MCU_STATUS_DISABLED 0 +#define MCU_STATUS_ENABLED 1 +#define MCU_STATUS_HALT 2 +#define MCU_STATUS_FATAL 3 + +#define JOB_INT_BASE 0x1000 + +#define JOB_INT_RAWSTAT 0x1000 +#define JOB_INT_CLEAR 0x1004 +#define JOB_INT_MASK 0x1008 +#define JOB_INT_STAT 0x100c +#define JOB_INT_GLOBAL_IF BIT(31) +#define JOB_INT_CSG_IF(x) BIT(x) + +#define CSF_GPU_LATEST_FLUSH_ID 0x10000 + +#define CSF_DOORBELL(i) (0x80000 + ((i) * 0x10000)) +#define CSF_GLB_DOORBELL_ID 0 + +#endif /* __PANTHOR_FW_REGS_H__ */ diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c index 2ab444ee8c710..d8e5283e9a1c8 100644 --- a/drivers/gpu/drm/panthor/panthor_gpu.c +++ b/drivers/gpu/drm/panthor/panthor_gpu.c @@ -19,8 +19,8 @@ #include "panthor_device.h" #include "panthor_gpu.h" +#include "panthor_gpu_regs.h" #include "panthor_hw.h" -#include "panthor_regs.h" #define CREATE_TRACE_POINTS #include "panthor_trace.h" @@ -56,7 +56,7 @@ struct panthor_gpu { static void panthor_gpu_coherency_set(struct panthor_device *ptdev) { - gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, + gpu_write(ptdev->iomem, GPU_COHERENCY_PROTOCOL, ptdev->gpu_info.selected_coherency); } @@ -75,26 +75,26 @@ static void panthor_gpu_l2_config_set(struct panthor_device *ptdev) } for (i = 0; i < ARRAY_SIZE(data->asn_hash); i++) - gpu_write(ptdev, GPU_ASN_HASH(i), data->asn_hash[i]); + gpu_write(ptdev->iomem, GPU_ASN_HASH(i), data->asn_hash[i]); - l2_config = gpu_read(ptdev, GPU_L2_CONFIG); + l2_config = gpu_read(ptdev->iomem, GPU_L2_CONFIG); l2_config |= GPU_L2_CONFIG_ASN_HASH_ENABLE; - gpu_write(ptdev, GPU_L2_CONFIG, l2_config); + gpu_write(ptdev->iomem, GPU_L2_CONFIG, l2_config); } static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) { - gpu_write(ptdev, GPU_INT_CLEAR, status); + gpu_write(ptdev->iomem, GPU_INT_CLEAR, status); if (tracepoint_enabled(gpu_power_status) && (status & GPU_POWER_INTERRUPTS_MASK)) trace_gpu_power_status(ptdev->base.dev, - gpu_read64(ptdev, SHADER_READY), - gpu_read64(ptdev, TILER_READY), - gpu_read64(ptdev, L2_READY)); + gpu_read64(ptdev->iomem, SHADER_READY), + gpu_read64(ptdev->iomem, TILER_READY), + gpu_read64(ptdev->iomem, L2_READY)); if (status & GPU_IRQ_FAULT) { - u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS); - u64 address = gpu_read64(ptdev, GPU_FAULT_ADDR); + u32 fault_status = gpu_read(ptdev->iomem, GPU_FAULT_STATUS); + u64 address = gpu_read64(ptdev->iomem, GPU_FAULT_ADDR); drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n", fault_status, panthor_exception_name(ptdev, fault_status & 0xFF), @@ -110,7 +110,7 @@ static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) } spin_unlock(&ptdev->gpu->reqs_lock); } -PANTHOR_IRQ_HANDLER(gpu, GPU, panthor_gpu_irq_handler); +PANTHOR_IRQ_HANDLER(gpu, panthor_gpu_irq_handler); /** * panthor_gpu_unplug() - Called when the GPU is unplugged. @@ -162,7 +162,9 @@ int panthor_gpu_init(struct panthor_device *ptdev) if (irq < 0) return irq; - ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK); + ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, + GPU_INTERRUPTS_MASK, + ptdev->iomem + GPU_INT_BASE); if (ret) return ret; @@ -204,7 +206,7 @@ int panthor_gpu_block_power_off(struct panthor_device *ptdev, u32 val; int ret; - ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, + ret = gpu_read64_relaxed_poll_timeout(ptdev->iomem, pwrtrans_reg, val, !(mask & val), 100, timeout_us); if (ret) { drm_err(&ptdev->base, @@ -213,9 +215,9 @@ int panthor_gpu_block_power_off(struct panthor_device *ptdev, return ret; } - gpu_write64(ptdev, pwroff_reg, mask); + gpu_write64(ptdev->iomem, pwroff_reg, mask); - ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, + ret = gpu_read64_relaxed_poll_timeout(ptdev->iomem, pwrtrans_reg, val, !(mask & val), 100, timeout_us); if (ret) { drm_err(&ptdev->base, @@ -247,7 +249,7 @@ int panthor_gpu_block_power_on(struct panthor_device *ptdev, u32 val; int ret; - ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, + ret = gpu_read64_relaxed_poll_timeout(ptdev->iomem, pwrtrans_reg, val, !(mask & val), 100, timeout_us); if (ret) { drm_err(&ptdev->base, @@ -256,9 +258,9 @@ int panthor_gpu_block_power_on(struct panthor_device *ptdev, return ret; } - gpu_write64(ptdev, pwron_reg, mask); + gpu_write64(ptdev->iomem, pwron_reg, mask); - ret = gpu_read64_relaxed_poll_timeout(ptdev, rdy_reg, val, + ret = gpu_read64_relaxed_poll_timeout(ptdev->iomem, rdy_reg, val, (mask & val) == val, 100, timeout_us); if (ret) { @@ -326,7 +328,7 @@ int panthor_gpu_flush_caches(struct panthor_device *ptdev, spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; - gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); + gpu_write(ptdev->iomem, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); } else { ret = -EIO; } @@ -340,7 +342,7 @@ int panthor_gpu_flush_caches(struct panthor_device *ptdev, msecs_to_jiffies(100))) { spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && - !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) + !(gpu_read(ptdev->iomem, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) ret = -ETIMEDOUT; else ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; @@ -370,8 +372,8 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev) if (!drm_WARN_ON(&ptdev->base, ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) { ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED; - gpu_write(ptdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED); - gpu_write(ptdev, GPU_CMD, GPU_SOFT_RESET); + gpu_write(ptdev->iomem, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED); + gpu_write(ptdev->iomem, GPU_CMD, GPU_SOFT_RESET); } spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); @@ -380,7 +382,7 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev) msecs_to_jiffies(100))) { spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 && - !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED)) + !(gpu_read(ptdev->iomem, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED)) timedout = true; else ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED; diff --git a/drivers/gpu/drm/panthor/panthor_gpu_regs.h b/drivers/gpu/drm/panthor/panthor_gpu_regs.h new file mode 100644 index 0000000000000..3f60c45985a75 --- /dev/null +++ b/drivers/gpu/drm/panthor/panthor_gpu_regs.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 or MIT */ +/* Copyright 2026 ARM Limited. All rights reserved. */ + +#ifndef __PANTHOR_GPU_REGS_H__ +#define __PANTHOR_GPU_REGS_H__ + +#define GPU_CONTROL_BASE 0x0 + +#define GPU_ID 0x0 +#define GPU_ARCH_MAJOR(x) ((x) >> 28) +#define GPU_ARCH_MINOR(x) (((x) & GENMASK(27, 24)) >> 24) +#define GPU_ARCH_REV(x) (((x) & GENMASK(23, 20)) >> 20) +#define GPU_PROD_MAJOR(x) (((x) & GENMASK(19, 16)) >> 16) +#define GPU_VER_MAJOR(x) (((x) & GENMASK(15, 12)) >> 12) +#define GPU_VER_MINOR(x) (((x) & GENMASK(11, 4)) >> 4) +#define GPU_VER_STATUS(x) ((x) & GENMASK(3, 0)) + +#define GPU_L2_FEATURES 0x4 +#define GPU_L2_FEATURES_LINE_SIZE(x) (1 << ((x) & GENMASK(7, 0))) + +#define GPU_CORE_FEATURES 0x8 + +#define GPU_TILER_FEATURES 0xC +#define GPU_MEM_FEATURES 0x10 +#define GROUPS_L2_COHERENT BIT(0) + +#define GPU_MMU_FEATURES 0x14 +#define GPU_MMU_FEATURES_VA_BITS(x) ((x) & GENMASK(7, 0)) +#define GPU_MMU_FEATURES_PA_BITS(x) (((x) >> 8) & GENMASK(7, 0)) +#define GPU_AS_PRESENT 0x18 +#define GPU_CSF_ID 0x1C + +#define GPU_INT_BASE 0x20 +#define GPU_INT_RAWSTAT 0x20 +#define GPU_INT_CLEAR 0x24 +#define GPU_INT_MASK 0x28 +#define GPU_INT_STAT 0x2c +#define GPU_IRQ_FAULT BIT(0) +#define GPU_IRQ_PROTM_FAULT BIT(1) +#define GPU_IRQ_RESET_COMPLETED BIT(8) +#define GPU_IRQ_POWER_CHANGED BIT(9) +#define GPU_IRQ_POWER_CHANGED_ALL BIT(10) +#define GPU_IRQ_CLEAN_CACHES_COMPLETED BIT(17) +#define GPU_IRQ_DOORBELL_MIRROR BIT(18) +#define GPU_IRQ_MCU_STATUS_CHANGED BIT(19) +#define GPU_CMD 0x30 +#define GPU_CMD_DEF(type, payload) ((type) | ((payload) << 8)) +#define GPU_SOFT_RESET GPU_CMD_DEF(1, 1) +#define GPU_HARD_RESET GPU_CMD_DEF(1, 2) +#define CACHE_CLEAN BIT(0) +#define CACHE_INV BIT(1) +#define GPU_FLUSH_CACHES(l2, lsc, oth) \ + GPU_CMD_DEF(4, ((l2) << 0) | ((lsc) << 4) | ((oth) << 8)) + +#define GPU_STATUS 0x34 +#define GPU_STATUS_ACTIVE BIT(0) +#define GPU_STATUS_PWR_ACTIVE BIT(1) +#define GPU_STATUS_PAGE_FAULT BIT(4) +#define GPU_STATUS_PROTM_ACTIVE BIT(7) +#define GPU_STATUS_DBG_ENABLED BIT(8) + +#define GPU_FAULT_STATUS 0x3C +#define GPU_FAULT_ADDR 0x40 +#define GPU_L2_CONFIG 0x48 +#define GPU_L2_CONFIG_ASN_HASH_ENABLE BIT(24) + +#define GPU_PWR_KEY 0x50 +#define GPU_PWR_KEY_UNLOCK 0x2968A819 +#define GPU_PWR_OVERRIDE0 0x54 +#define GPU_PWR_OVERRIDE1 0x58 + +#define GPU_FEATURES 0x60 +#define GPU_FEATURES_RAY_INTERSECTION BIT(2) +#define GPU_FEATURES_RAY_TRAVERSAL BIT(5) + +#define GPU_TIMESTAMP_OFFSET 0x88 +#define GPU_CYCLE_COUNT 0x90 +#define GPU_TIMESTAMP 0x98 + +#define GPU_THREAD_MAX_THREADS 0xA0 +#define GPU_THREAD_MAX_WORKGROUP_SIZE 0xA4 +#define GPU_THREAD_MAX_BARRIER_SIZE 0xA8 +#define GPU_THREAD_FEATURES 0xAC + +#define GPU_TEXTURE_FEATURES(n) (0xB0 + ((n) * 4)) + +#define GPU_SHADER_PRESENT 0x100 +#define GPU_TILER_PRESENT 0x110 +#define GPU_L2_PRESENT 0x120 + +#define SHADER_READY 0x140 +#define TILER_READY 0x150 +#define L2_READY 0x160 + +#define SHADER_PWRON 0x180 +#define TILER_PWRON 0x190 +#define L2_PWRON 0x1A0 + +#define SHADER_PWROFF 0x1C0 +#define TILER_PWROFF 0x1D0 +#define L2_PWROFF 0x1E0 + +#define SHADER_PWRTRANS 0x200 +#define TILER_PWRTRANS 0x210 +#define L2_PWRTRANS 0x220 + +#define SHADER_PWRACTIVE 0x240 +#define TILER_PWRACTIVE 0x250 +#define L2_PWRACTIVE 0x260 + +#define GPU_REVID 0x280 + +#define GPU_ASN_HASH(n) (0x2C0 + ((n) * 4)) + +#define GPU_COHERENCY_FEATURES 0x300 +#define GPU_COHERENCY_PROT_BIT(name) BIT(GPU_COHERENCY_ ## name) + +#define GPU_COHERENCY_PROTOCOL 0x304 +#define GPU_COHERENCY_ACE_LITE 0 +#define GPU_COHERENCY_ACE 1 +#define GPU_COHERENCY_NONE 31 + +#endif /* __PANTHOR_GPU_REGS_H__ */ diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c index 1ee30dc7066f7..99311abdf1e91 100644 --- a/drivers/gpu/drm/panthor/panthor_heap.c +++ b/drivers/gpu/drm/panthor/panthor_heap.c @@ -9,9 +9,9 @@ #include "panthor_device.h" #include "panthor_gem.h" +#include "panthor_gpu_regs.h" #include "panthor_heap.h" #include "panthor_mmu.h" -#include "panthor_regs.h" /* * The GPU heap context is an opaque structure used by the GPU to track the diff --git a/drivers/gpu/drm/panthor/panthor_hw.c b/drivers/gpu/drm/panthor/panthor_hw.c index d135aa6724fa5..9431f16d950fd 100644 --- a/drivers/gpu/drm/panthor/panthor_hw.c +++ b/drivers/gpu/drm/panthor/panthor_hw.c @@ -8,9 +8,10 @@ #include "panthor_device.h" #include "panthor_gpu.h" +#include "panthor_gpu_regs.h" #include "panthor_hw.h" #include "panthor_pwr.h" -#include "panthor_regs.h" +#include "panthor_pwr_regs.h" #define GPU_PROD_ID_MAKE(arch_major, prod_major) \ (((arch_major) << 24) | (prod_major)) @@ -194,35 +195,38 @@ static int panthor_gpu_info_init(struct panthor_device *ptdev) { unsigned int i; - ptdev->gpu_info.csf_id = gpu_read(ptdev, GPU_CSF_ID); - ptdev->gpu_info.gpu_rev = gpu_read(ptdev, GPU_REVID); - ptdev->gpu_info.core_features = gpu_read(ptdev, GPU_CORE_FEATURES); - ptdev->gpu_info.l2_features = gpu_read(ptdev, GPU_L2_FEATURES); - ptdev->gpu_info.tiler_features = gpu_read(ptdev, GPU_TILER_FEATURES); - ptdev->gpu_info.mem_features = gpu_read(ptdev, GPU_MEM_FEATURES); - ptdev->gpu_info.mmu_features = gpu_read(ptdev, GPU_MMU_FEATURES); - ptdev->gpu_info.thread_features = gpu_read(ptdev, GPU_THREAD_FEATURES); - ptdev->gpu_info.max_threads = gpu_read(ptdev, GPU_THREAD_MAX_THREADS); - ptdev->gpu_info.thread_max_workgroup_size = gpu_read(ptdev, GPU_THREAD_MAX_WORKGROUP_SIZE); - ptdev->gpu_info.thread_max_barrier_size = gpu_read(ptdev, GPU_THREAD_MAX_BARRIER_SIZE); - ptdev->gpu_info.coherency_features = gpu_read(ptdev, GPU_COHERENCY_FEATURES); + ptdev->gpu_info.csf_id = gpu_read(ptdev->iomem, GPU_CSF_ID); + ptdev->gpu_info.gpu_rev = gpu_read(ptdev->iomem, GPU_REVID); + ptdev->gpu_info.core_features = gpu_read(ptdev->iomem, GPU_CORE_FEATURES); + ptdev->gpu_info.l2_features = gpu_read(ptdev->iomem, GPU_L2_FEATURES); + ptdev->gpu_info.tiler_features = gpu_read(ptdev->iomem, GPU_TILER_FEATURES); + ptdev->gpu_info.mem_features = gpu_read(ptdev->iomem, GPU_MEM_FEATURES); + ptdev->gpu_info.mmu_features = gpu_read(ptdev->iomem, GPU_MMU_FEATURES); + ptdev->gpu_info.thread_features = gpu_read(ptdev->iomem, GPU_THREAD_FEATURES); + ptdev->gpu_info.max_threads = gpu_read(ptdev->iomem, GPU_THREAD_MAX_THREADS); + ptdev->gpu_info.thread_max_workgroup_size = + gpu_read(ptdev->iomem, GPU_THREAD_MAX_WORKGROUP_SIZE); + ptdev->gpu_info.thread_max_barrier_size = + gpu_read(ptdev->iomem, GPU_THREAD_MAX_BARRIER_SIZE); + ptdev->gpu_info.coherency_features = gpu_read(ptdev->iomem, GPU_COHERENCY_FEATURES); for (i = 0; i < 4; i++) - ptdev->gpu_info.texture_features[i] = gpu_read(ptdev, GPU_TEXTURE_FEATURES(i)); + ptdev->gpu_info.texture_features[i] = + gpu_read(ptdev->iomem, GPU_TEXTURE_FEATURES(i)); - ptdev->gpu_info.as_present = gpu_read(ptdev, GPU_AS_PRESENT); + ptdev->gpu_info.as_present = gpu_read(ptdev->iomem, GPU_AS_PRESENT); /* Introduced in arch 11.x */ - ptdev->gpu_info.gpu_features = gpu_read64(ptdev, GPU_FEATURES); + ptdev->gpu_info.gpu_features = gpu_read64(ptdev->iomem, GPU_FEATURES); if (panthor_hw_has_pwr_ctrl(ptdev)) { /* Introduced in arch 14.x */ - ptdev->gpu_info.l2_present = gpu_read64(ptdev, PWR_L2_PRESENT); - ptdev->gpu_info.tiler_present = gpu_read64(ptdev, PWR_TILER_PRESENT); - ptdev->gpu_info.shader_present = gpu_read64(ptdev, PWR_SHADER_PRESENT); + ptdev->gpu_info.l2_present = gpu_read64(ptdev->iomem, PWR_L2_PRESENT); + ptdev->gpu_info.tiler_present = gpu_read64(ptdev->iomem, PWR_TILER_PRESENT); + ptdev->gpu_info.shader_present = gpu_read64(ptdev->iomem, PWR_SHADER_PRESENT); } else { - ptdev->gpu_info.shader_present = gpu_read64(ptdev, GPU_SHADER_PRESENT); - ptdev->gpu_info.tiler_present = gpu_read64(ptdev, GPU_TILER_PRESENT); - ptdev->gpu_info.l2_present = gpu_read64(ptdev, GPU_L2_PRESENT); + ptdev->gpu_info.shader_present = gpu_read64(ptdev->iomem, GPU_SHADER_PRESENT); + ptdev->gpu_info.tiler_present = gpu_read64(ptdev->iomem, GPU_TILER_PRESENT); + ptdev->gpu_info.l2_present = gpu_read64(ptdev->iomem, GPU_L2_PRESENT); } return overload_shader_present(ptdev); @@ -287,7 +291,7 @@ static int panthor_hw_bind_device(struct panthor_device *ptdev) static int panthor_hw_gpu_id_init(struct panthor_device *ptdev) { - ptdev->gpu_info.gpu_id = gpu_read(ptdev, GPU_ID); + ptdev->gpu_info.gpu_id = gpu_read(ptdev->iomem, GPU_ID); if (!ptdev->gpu_info.gpu_id) return -ENXIO; diff --git a/drivers/gpu/drm/panthor/panthor_hw.h b/drivers/gpu/drm/panthor/panthor_hw.h index 2c28aea828418..f797663893b2d 100644 --- a/drivers/gpu/drm/panthor/panthor_hw.h +++ b/drivers/gpu/drm/panthor/panthor_hw.h @@ -5,7 +5,7 @@ #define __PANTHOR_HW_H__ #include "panthor_device.h" -#include "panthor_regs.h" +#include "panthor_gpu_regs.h" /** * struct panthor_hw_ops - HW operations that are specific to a GPU diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index 75d98dad7b1dd..263a22f7b3934 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -31,9 +31,10 @@ #include "panthor_device.h" #include "panthor_gem.h" #include "panthor_gpu.h" +#include "panthor_gpu_regs.h" #include "panthor_heap.h" #include "panthor_mmu.h" -#include "panthor_regs.h" +#include "panthor_mmu_regs.h" #include "panthor_sched.h" #define MAX_AS_SLOTS 32 @@ -507,9 +508,8 @@ static int wait_ready(struct panthor_device *ptdev, u32 as_nr) /* Wait for the MMU status to indicate there is no active command, in * case one is pending. */ - ret = gpu_read_relaxed_poll_timeout_atomic(ptdev, AS_STATUS(as_nr), val, - !(val & AS_STATUS_AS_ACTIVE), - 10, 100000); + ret = gpu_read_relaxed_poll_timeout_atomic(ptdev->iomem, AS_STATUS(as_nr), val, + !(val & AS_STATUS_AS_ACTIVE), 10, 100000); if (ret) { panthor_device_schedule_reset(ptdev); @@ -526,7 +526,7 @@ static int as_send_cmd_and_wait(struct panthor_device *ptdev, u32 as_nr, u32 cmd /* write AS_COMMAND when MMU is ready to accept another command */ status = wait_ready(ptdev, as_nr); if (!status) { - gpu_write(ptdev, AS_COMMAND(as_nr), cmd); + gpu_write(ptdev->iomem, AS_COMMAND(as_nr), cmd); status = wait_ready(ptdev, as_nr); } @@ -569,7 +569,7 @@ static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as) /* Forward declaration to call helpers within as_enable/disable */ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status); -PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler); +PANTHOR_IRQ_HANDLER(mmu, panthor_mmu_irq_handler); static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr, u64 transtab, u64 transcfg, u64 memattr) @@ -577,9 +577,9 @@ static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr, panthor_mmu_irq_enable_events(&ptdev->mmu->irq, panthor_mmu_as_fault_mask(ptdev, as_nr)); - gpu_write64(ptdev, AS_TRANSTAB(as_nr), transtab); - gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr); - gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg); + gpu_write64(ptdev->iomem, AS_TRANSTAB(as_nr), transtab); + gpu_write64(ptdev->iomem, AS_MEMATTR(as_nr), memattr); + gpu_write64(ptdev->iomem, AS_TRANSCFG(as_nr), transcfg); return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } @@ -614,9 +614,9 @@ static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr, if (recycle_slot) return 0; - gpu_write64(ptdev, AS_TRANSTAB(as_nr), 0); - gpu_write64(ptdev, AS_MEMATTR(as_nr), 0); - gpu_write64(ptdev, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED); + gpu_write64(ptdev->iomem, AS_TRANSTAB(as_nr), 0); + gpu_write64(ptdev->iomem, AS_MEMATTR(as_nr), 0); + gpu_write64(ptdev->iomem, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED); return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } @@ -759,7 +759,7 @@ int panthor_vm_active(struct panthor_vm *vm) */ fault_mask = panthor_mmu_as_fault_mask(ptdev, as); if (ptdev->mmu->as.faulty_mask & fault_mask) { - gpu_write(ptdev, MMU_INT_CLEAR, fault_mask); + gpu_write(ptdev->iomem, MMU_INT_CLEAR, fault_mask); ptdev->mmu->as.faulty_mask &= ~fault_mask; } @@ -1670,7 +1670,7 @@ static int panthor_vm_lock_region(struct panthor_vm *vm, u64 start, u64 size) mutex_lock(&ptdev->mmu->as.slots_lock); if (vm->as.id >= 0 && size) { /* Lock the region that needs to be updated */ - gpu_write64(ptdev, AS_LOCKADDR(vm->as.id), + gpu_write64(ptdev->iomem, AS_LOCKADDR(vm->as.id), pack_region_range(ptdev, &start, &size)); /* If the lock succeeded, update the locked_region info. */ @@ -1731,8 +1731,8 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status) u32 access_type; u32 source_id; - fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as)); - addr = gpu_read64(ptdev, AS_FAULTADDRESS(as)); + fault_status = gpu_read(ptdev->iomem, AS_FAULTSTATUS(as)); + addr = gpu_read64(ptdev->iomem, AS_FAULTADDRESS(as)); /* decode the fault status */ exception_type = fault_status & 0xFF; @@ -1763,7 +1763,7 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status) * Note that COMPLETED irqs are never cleared, but this is fine * because they are always masked. */ - gpu_write(ptdev, MMU_INT_CLEAR, mask); + gpu_write(ptdev->iomem, MMU_INT_CLEAR, mask); if (ptdev->mmu->as.slots[as].vm) ptdev->mmu->as.slots[as].vm->unhandled_fault = true; @@ -2888,7 +2888,8 @@ int panthor_mmu_init(struct panthor_device *ptdev) return -ENODEV; ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq, - panthor_mmu_fault_mask(ptdev, ~0)); + panthor_mmu_fault_mask(ptdev, ~0), + ptdev->iomem + MMU_INT_BASE); if (ret) return ret; diff --git a/drivers/gpu/drm/panthor/panthor_mmu_regs.h b/drivers/gpu/drm/panthor/panthor_mmu_regs.h new file mode 100644 index 0000000000000..de460042651dc --- /dev/null +++ b/drivers/gpu/drm/panthor/panthor_mmu_regs.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 or MIT */ +/* Copyright 2026 ARM Limited. All rights reserved. */ + +#ifndef __PANTHOR_MMU_REGS_H__ +#define __PANTHOR_MMU_REGS_H__ + +/* MMU regs */ + +#define MMU_INT_BASE 0x2000 + +#define MMU_INT_RAWSTAT 0x2000 +#define MMU_INT_CLEAR 0x2004 +#define MMU_INT_MASK 0x2008 +#define MMU_INT_STAT 0x200c + +/* AS_COMMAND register commands */ + +#define MMU_BASE 0x2400 +#define MMU_AS_SHIFT 6 +#define MMU_AS(as) (MMU_BASE + ((as) << MMU_AS_SHIFT)) + +#define AS_TRANSTAB(as) (MMU_AS(as) + 0x0) +#define AS_MEMATTR(as) (MMU_AS(as) + 0x8) +#define AS_MEMATTR_AARCH64_INNER_ALLOC_IMPL (2 << 2) +#define AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(w, r) ((3 << 2) | \ + ((w) ? BIT(0) : 0) | \ + ((r) ? BIT(1) : 0)) +#define AS_MEMATTR_AARCH64_SH_MIDGARD_INNER (0 << 4) +#define AS_MEMATTR_AARCH64_SH_CPU_INNER (1 << 4) +#define AS_MEMATTR_AARCH64_SH_CPU_INNER_SHADER_COH (2 << 4) +#define AS_MEMATTR_AARCH64_SHARED (0 << 6) +#define AS_MEMATTR_AARCH64_INNER_OUTER_NC (1 << 6) +#define AS_MEMATTR_AARCH64_INNER_OUTER_WB (2 << 6) +#define AS_MEMATTR_AARCH64_FAULT (3 << 6) +#define AS_LOCKADDR(as) (MMU_AS(as) + 0x10) +#define AS_COMMAND(as) (MMU_AS(as) + 0x18) +#define AS_COMMAND_NOP 0 +#define AS_COMMAND_UPDATE 1 +#define AS_COMMAND_LOCK 2 +#define AS_COMMAND_UNLOCK 3 +#define AS_COMMAND_FLUSH_PT 4 +#define AS_COMMAND_FLUSH_MEM 5 +#define AS_LOCK_REGION_MIN_SIZE (1ULL << 15) +#define AS_FAULTSTATUS(as) (MMU_AS(as) + 0x1C) +#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << 8) +#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0 << 8) +#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1 << 8) +#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2 << 8) +#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3 << 8) +#define AS_FAULTADDRESS(as) (MMU_AS(as) + 0x20) +#define AS_STATUS(as) (MMU_AS(as) + 0x28) +#define AS_STATUS_AS_ACTIVE BIT(0) +#define AS_TRANSCFG(as) (MMU_AS(as) + 0x30) +#define AS_TRANSCFG_ADRMODE_UNMAPPED (1 << 0) +#define AS_TRANSCFG_ADRMODE_IDENTITY (2 << 0) +#define AS_TRANSCFG_ADRMODE_AARCH64_4K (6 << 0) +#define AS_TRANSCFG_ADRMODE_AARCH64_64K (8 << 0) +#define AS_TRANSCFG_INA_BITS(x) ((x) << 6) +#define AS_TRANSCFG_OUTA_BITS(x) ((x) << 14) +#define AS_TRANSCFG_SL_CONCAT BIT(22) +#define AS_TRANSCFG_PTW_MEMATTR_NC (1 << 24) +#define AS_TRANSCFG_PTW_MEMATTR_WB (2 << 24) +#define AS_TRANSCFG_PTW_SH_NS (0 << 28) +#define AS_TRANSCFG_PTW_SH_OS (2 << 28) +#define AS_TRANSCFG_PTW_SH_IS (3 << 28) +#define AS_TRANSCFG_PTW_RA BIT(30) +#define AS_TRANSCFG_DISABLE_HIER_AP BIT(33) +#define AS_TRANSCFG_DISABLE_AF_FAULT BIT(34) +#define AS_TRANSCFG_WXN BIT(35) +#define AS_TRANSCFG_XREADABLE BIT(36) +#define AS_FAULTEXTRA(as) (MMU_AS(as) + 0x38) + +#endif /* __PANTHOR_MMU_REGS_H__ */ diff --git a/drivers/gpu/drm/panthor/panthor_pwr.c b/drivers/gpu/drm/panthor/panthor_pwr.c index ed3b2b4479ca3..dda5159a5271b 100644 --- a/drivers/gpu/drm/panthor/panthor_pwr.c +++ b/drivers/gpu/drm/panthor/panthor_pwr.c @@ -11,9 +11,10 @@ #include #include "panthor_device.h" +#include "panthor_gpu_regs.h" #include "panthor_hw.h" #include "panthor_pwr.h" -#include "panthor_regs.h" +#include "panthor_pwr_regs.h" #define PWR_INTERRUPTS_MASK \ (PWR_IRQ_POWER_CHANGED_SINGLE | \ @@ -55,7 +56,7 @@ struct panthor_pwr { static void panthor_pwr_irq_handler(struct panthor_device *ptdev, u32 status) { spin_lock(&ptdev->pwr->reqs_lock); - gpu_write(ptdev, PWR_INT_CLEAR, status); + gpu_write(ptdev->iomem, PWR_INT_CLEAR, status); if (unlikely(status & PWR_IRQ_COMMAND_NOT_ALLOWED)) drm_err(&ptdev->base, "PWR_IRQ: COMMAND_NOT_ALLOWED"); @@ -69,19 +70,19 @@ static void panthor_pwr_irq_handler(struct panthor_device *ptdev, u32 status) } spin_unlock(&ptdev->pwr->reqs_lock); } -PANTHOR_IRQ_HANDLER(pwr, PWR, panthor_pwr_irq_handler); +PANTHOR_IRQ_HANDLER(pwr, panthor_pwr_irq_handler); static void panthor_pwr_write_command(struct panthor_device *ptdev, u32 command, u64 args) { if (args) - gpu_write64(ptdev, PWR_CMDARG, args); + gpu_write64(ptdev->iomem, PWR_CMDARG, args); - gpu_write(ptdev, PWR_COMMAND, command); + gpu_write(ptdev->iomem, PWR_COMMAND, command); } static bool reset_irq_raised(struct panthor_device *ptdev) { - return gpu_read(ptdev, PWR_INT_RAWSTAT) & PWR_IRQ_RESET_COMPLETED; + return gpu_read(ptdev->iomem, PWR_INT_RAWSTAT) & PWR_IRQ_RESET_COMPLETED; } static bool reset_pending(struct panthor_device *ptdev) @@ -96,7 +97,7 @@ static int panthor_pwr_reset(struct panthor_device *ptdev, u32 reset_cmd) drm_WARN(&ptdev->base, 1, "Reset already pending"); } else { ptdev->pwr->pending_reqs |= PWR_IRQ_RESET_COMPLETED; - gpu_write(ptdev, PWR_INT_CLEAR, PWR_IRQ_RESET_COMPLETED); + gpu_write(ptdev->iomem, PWR_INT_CLEAR, PWR_IRQ_RESET_COMPLETED); panthor_pwr_write_command(ptdev, reset_cmd, 0); } } @@ -185,7 +186,7 @@ static int panthor_pwr_domain_wait_transition(struct panthor_device *ptdev, u32 u64 val; int ret = 0; - ret = gpu_read64_poll_timeout(ptdev, pwrtrans_reg, val, !(PWR_ALL_CORES_MASK & val), 100, + ret = gpu_read64_poll_timeout(ptdev->iomem, pwrtrans_reg, val, !(PWR_ALL_CORES_MASK & val), 100, timeout_us); if (ret) { drm_err(&ptdev->base, "%s domain power in transition, pwrtrans(0x%llx)", @@ -198,17 +199,17 @@ static int panthor_pwr_domain_wait_transition(struct panthor_device *ptdev, u32 static void panthor_pwr_debug_info_show(struct panthor_device *ptdev) { - drm_info(&ptdev->base, "GPU_FEATURES: 0x%016llx", gpu_read64(ptdev, GPU_FEATURES)); - drm_info(&ptdev->base, "PWR_STATUS: 0x%016llx", gpu_read64(ptdev, PWR_STATUS)); - drm_info(&ptdev->base, "L2_PRESENT: 0x%016llx", gpu_read64(ptdev, PWR_L2_PRESENT)); - drm_info(&ptdev->base, "L2_PWRTRANS: 0x%016llx", gpu_read64(ptdev, PWR_L2_PWRTRANS)); - drm_info(&ptdev->base, "L2_READY: 0x%016llx", gpu_read64(ptdev, PWR_L2_READY)); - drm_info(&ptdev->base, "TILER_PRESENT: 0x%016llx", gpu_read64(ptdev, PWR_TILER_PRESENT)); - drm_info(&ptdev->base, "TILER_PWRTRANS: 0x%016llx", gpu_read64(ptdev, PWR_TILER_PWRTRANS)); - drm_info(&ptdev->base, "TILER_READY: 0x%016llx", gpu_read64(ptdev, PWR_TILER_READY)); - drm_info(&ptdev->base, "SHADER_PRESENT: 0x%016llx", gpu_read64(ptdev, PWR_SHADER_PRESENT)); - drm_info(&ptdev->base, "SHADER_PWRTRANS: 0x%016llx", gpu_read64(ptdev, PWR_SHADER_PWRTRANS)); - drm_info(&ptdev->base, "SHADER_READY: 0x%016llx", gpu_read64(ptdev, PWR_SHADER_READY)); + drm_info(&ptdev->base, "GPU_FEATURES: 0x%016llx", gpu_read64(ptdev->iomem, GPU_FEATURES)); + drm_info(&ptdev->base, "PWR_STATUS: 0x%016llx", gpu_read64(ptdev->iomem, PWR_STATUS)); + drm_info(&ptdev->base, "L2_PRESENT: 0x%016llx", gpu_read64(ptdev->iomem, PWR_L2_PRESENT)); + drm_info(&ptdev->base, "L2_PWRTRANS: 0x%016llx", gpu_read64(ptdev->iomem, PWR_L2_PWRTRANS)); + drm_info(&ptdev->base, "L2_READY: 0x%016llx", gpu_read64(ptdev->iomem, PWR_L2_READY)); + drm_info(&ptdev->base, "TILER_PRESENT: 0x%016llx", gpu_read64(ptdev->iomem, PWR_TILER_PRESENT)); + drm_info(&ptdev->base, "TILER_PWRTRANS: 0x%016llx", gpu_read64(ptdev->iomem, PWR_TILER_PWRTRANS)); + drm_info(&ptdev->base, "TILER_READY: 0x%016llx", gpu_read64(ptdev->iomem, PWR_TILER_READY)); + drm_info(&ptdev->base, "SHADER_PRESENT: 0x%016llx", gpu_read64(ptdev->iomem, PWR_SHADER_PRESENT)); + drm_info(&ptdev->base, "SHADER_PWRTRANS: 0x%016llx", gpu_read64(ptdev->iomem, PWR_SHADER_PWRTRANS)); + drm_info(&ptdev->base, "SHADER_READY: 0x%016llx", gpu_read64(ptdev->iomem, PWR_SHADER_READY)); } static int panthor_pwr_domain_transition(struct panthor_device *ptdev, u32 cmd, u32 domain, @@ -240,13 +241,13 @@ static int panthor_pwr_domain_transition(struct panthor_device *ptdev, u32 cmd, return ret; /* domain already in target state, return early */ - if ((gpu_read64(ptdev, ready_reg) & mask) == expected_val) + if ((gpu_read64(ptdev->iomem, ready_reg) & mask) == expected_val) return 0; panthor_pwr_write_command(ptdev, pwr_cmd, mask); - ret = gpu_read64_poll_timeout(ptdev, ready_reg, val, (mask & val) == expected_val, 100, - timeout_us); + ret = gpu_read64_poll_timeout(ptdev->iomem, ready_reg, val, (mask & val) == expected_val, + 100, timeout_us); if (ret) { drm_err(&ptdev->base, "timeout waiting on %s power domain transition, cmd(0x%x), arg(0x%llx)", @@ -279,7 +280,7 @@ static int panthor_pwr_domain_transition(struct panthor_device *ptdev, u32 cmd, static int retract_domain(struct panthor_device *ptdev, u32 domain) { const u32 pwr_cmd = PWR_COMMAND_DEF(PWR_COMMAND_RETRACT, domain, 0); - const u64 pwr_status = gpu_read64(ptdev, PWR_STATUS); + const u64 pwr_status = gpu_read64(ptdev->iomem, PWR_STATUS); const u64 delegated_mask = PWR_STATUS_DOMAIN_DELEGATED(domain); const u64 allow_mask = PWR_STATUS_DOMAIN_ALLOWED(domain); u64 val; @@ -288,8 +289,9 @@ static int retract_domain(struct panthor_device *ptdev, u32 domain) if (drm_WARN_ON(&ptdev->base, domain == PWR_COMMAND_DOMAIN_L2)) return -EPERM; - ret = gpu_read64_poll_timeout(ptdev, PWR_STATUS, val, !(PWR_STATUS_RETRACT_PENDING & val), - 0, PWR_RETRACT_TIMEOUT_US); + ret = gpu_read64_poll_timeout(ptdev->iomem, PWR_STATUS, val, + !(PWR_STATUS_RETRACT_PENDING & val), 0, + PWR_RETRACT_TIMEOUT_US); if (ret) { drm_err(&ptdev->base, "%s domain retract pending", get_domain_name(domain)); return ret; @@ -306,7 +308,7 @@ static int retract_domain(struct panthor_device *ptdev, u32 domain) * On successful retraction * allow-flag will be set with delegated-flag being cleared. */ - ret = gpu_read64_poll_timeout(ptdev, PWR_STATUS, val, + ret = gpu_read64_poll_timeout(ptdev->iomem, PWR_STATUS, val, ((delegated_mask | allow_mask) & val) == allow_mask, 10, PWR_TRANSITION_TIMEOUT_US); if (ret) { @@ -333,7 +335,7 @@ static int retract_domain(struct panthor_device *ptdev, u32 domain) static int delegate_domain(struct panthor_device *ptdev, u32 domain) { const u32 pwr_cmd = PWR_COMMAND_DEF(PWR_COMMAND_DELEGATE, domain, 0); - const u64 pwr_status = gpu_read64(ptdev, PWR_STATUS); + const u64 pwr_status = gpu_read64(ptdev->iomem, PWR_STATUS); const u64 allow_mask = PWR_STATUS_DOMAIN_ALLOWED(domain); const u64 delegated_mask = PWR_STATUS_DOMAIN_DELEGATED(domain); u64 val; @@ -362,7 +364,7 @@ static int delegate_domain(struct panthor_device *ptdev, u32 domain) * On successful delegation * allow-flag will be cleared with delegated-flag being set. */ - ret = gpu_read64_poll_timeout(ptdev, PWR_STATUS, val, + ret = gpu_read64_poll_timeout(ptdev->iomem, PWR_STATUS, val, ((delegated_mask | allow_mask) & val) == delegated_mask, 10, PWR_TRANSITION_TIMEOUT_US); if (ret) { @@ -410,7 +412,7 @@ static int panthor_pwr_delegate_domains(struct panthor_device *ptdev) */ static int panthor_pwr_domain_force_off(struct panthor_device *ptdev, u32 domain) { - const u64 domain_ready = gpu_read64(ptdev, get_domain_ready_reg(domain)); + const u64 domain_ready = gpu_read64(ptdev->iomem, get_domain_ready_reg(domain)); int ret; /* Domain already powered down, early exit. */ @@ -433,7 +435,8 @@ void panthor_pwr_unplug(struct panthor_device *ptdev) return; /* Make sure the IRQ handler is not running after that point. */ - panthor_pwr_irq_suspend(&ptdev->pwr->irq); + if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) + panthor_pwr_irq_suspend(&ptdev->pwr->irq); /* Wake-up all waiters. */ spin_lock_irqsave(&ptdev->pwr->reqs_lock, flags); @@ -462,7 +465,9 @@ int panthor_pwr_init(struct panthor_device *ptdev) if (irq < 0) return irq; - err = panthor_request_pwr_irq(ptdev, &pwr->irq, irq, PWR_INTERRUPTS_MASK); + err = panthor_request_pwr_irq( + ptdev, &pwr->irq, irq, PWR_INTERRUPTS_MASK, + ptdev->iomem + PWR_CONTROL_BASE); if (err) return err; @@ -471,7 +476,7 @@ int panthor_pwr_init(struct panthor_device *ptdev) int panthor_pwr_reset_soft(struct panthor_device *ptdev) { - if (!(gpu_read64(ptdev, PWR_STATUS) & PWR_STATUS_ALLOW_SOFT_RESET)) { + if (!(gpu_read64(ptdev->iomem, PWR_STATUS) & PWR_STATUS_ALLOW_SOFT_RESET)) { drm_err(&ptdev->base, "RESET_SOFT not allowed"); return -EOPNOTSUPP; } @@ -482,7 +487,7 @@ int panthor_pwr_reset_soft(struct panthor_device *ptdev) void panthor_pwr_l2_power_off(struct panthor_device *ptdev) { const u64 l2_allow_mask = PWR_STATUS_DOMAIN_ALLOWED(PWR_COMMAND_DOMAIN_L2); - const u64 pwr_status = gpu_read64(ptdev, PWR_STATUS); + const u64 pwr_status = gpu_read64(ptdev->iomem, PWR_STATUS); /* Abort if L2 power off constraints are not satisfied */ if (!(pwr_status & l2_allow_mask)) { @@ -508,7 +513,7 @@ void panthor_pwr_l2_power_off(struct panthor_device *ptdev) int panthor_pwr_l2_power_on(struct panthor_device *ptdev) { - const u32 pwr_status = gpu_read64(ptdev, PWR_STATUS); + const u32 pwr_status = gpu_read64(ptdev->iomem, PWR_STATUS); const u32 l2_allow_mask = PWR_STATUS_DOMAIN_ALLOWED(PWR_COMMAND_DOMAIN_L2); int ret; diff --git a/drivers/gpu/drm/panthor/panthor_pwr_regs.h b/drivers/gpu/drm/panthor/panthor_pwr_regs.h new file mode 100644 index 0000000000000..ad3e446971db0 --- /dev/null +++ b/drivers/gpu/drm/panthor/panthor_pwr_regs.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 or MIT */ +/* Copyright 2026 ARM Limited. All rights reserved. */ + +#ifndef __PANTHOR_PWR_REGS_H__ +#define __PANTHOR_PWR_REGS_H__ + +#define PWR_CONTROL_BASE 0x800 +#define PWR_CTRL_REG(x) (PWR_CONTROL_BASE + (x)) + +#define PWR_INT_RAWSTAT PWR_CTRL_REG(0x0) +#define PWR_INT_CLEAR PWR_CTRL_REG(0x4) +#define PWR_INT_MASK PWR_CTRL_REG(0x8) +#define PWR_INT_STAT PWR_CTRL_REG(0xc) +#define PWR_IRQ_POWER_CHANGED_SINGLE BIT(0) +#define PWR_IRQ_POWER_CHANGED_ALL BIT(1) +#define PWR_IRQ_DELEGATION_CHANGED BIT(2) +#define PWR_IRQ_RESET_COMPLETED BIT(3) +#define PWR_IRQ_RETRACT_COMPLETED BIT(4) +#define PWR_IRQ_INSPECT_COMPLETED BIT(5) +#define PWR_IRQ_COMMAND_NOT_ALLOWED BIT(30) +#define PWR_IRQ_COMMAND_INVALID BIT(31) + +#define PWR_STATUS PWR_CTRL_REG(0x20) +#define PWR_STATUS_ALLOW_L2 BIT_U64(0) +#define PWR_STATUS_ALLOW_TILER BIT_U64(1) +#define PWR_STATUS_ALLOW_SHADER BIT_U64(8) +#define PWR_STATUS_ALLOW_BASE BIT_U64(14) +#define PWR_STATUS_ALLOW_STACK BIT_U64(15) +#define PWR_STATUS_DOMAIN_ALLOWED(x) BIT_U64(x) +#define PWR_STATUS_DELEGATED_L2 BIT_U64(16) +#define PWR_STATUS_DELEGATED_TILER BIT_U64(17) +#define PWR_STATUS_DELEGATED_SHADER BIT_U64(24) +#define PWR_STATUS_DELEGATED_BASE BIT_U64(30) +#define PWR_STATUS_DELEGATED_STACK BIT_U64(31) +#define PWR_STATUS_DELEGATED_SHIFT 16 +#define PWR_STATUS_DOMAIN_DELEGATED(x) BIT_U64((x) + PWR_STATUS_DELEGATED_SHIFT) +#define PWR_STATUS_ALLOW_SOFT_RESET BIT_U64(33) +#define PWR_STATUS_ALLOW_FAST_RESET BIT_U64(34) +#define PWR_STATUS_POWER_PENDING BIT_U64(41) +#define PWR_STATUS_RESET_PENDING BIT_U64(42) +#define PWR_STATUS_RETRACT_PENDING BIT_U64(43) +#define PWR_STATUS_INSPECT_PENDING BIT_U64(44) + +#define PWR_COMMAND PWR_CTRL_REG(0x28) +#define PWR_COMMAND_POWER_UP 0x10 +#define PWR_COMMAND_POWER_DOWN 0x11 +#define PWR_COMMAND_DELEGATE 0x20 +#define PWR_COMMAND_RETRACT 0x21 +#define PWR_COMMAND_RESET_SOFT 0x31 +#define PWR_COMMAND_RESET_FAST 0x32 +#define PWR_COMMAND_INSPECT 0xF0 +#define PWR_COMMAND_DOMAIN_L2 0 +#define PWR_COMMAND_DOMAIN_TILER 1 +#define PWR_COMMAND_DOMAIN_SHADER 8 +#define PWR_COMMAND_DOMAIN_BASE 14 +#define PWR_COMMAND_DOMAIN_STACK 15 +#define PWR_COMMAND_SUBDOMAIN_RTU BIT(0) +#define PWR_COMMAND_DEF(cmd, domain, subdomain) \ + (((subdomain) << 16) | ((domain) << 8) | (cmd)) + +#define PWR_CMDARG PWR_CTRL_REG(0x30) + +#define PWR_L2_PRESENT PWR_CTRL_REG(0x100) +#define PWR_L2_READY PWR_CTRL_REG(0x108) +#define PWR_L2_PWRTRANS PWR_CTRL_REG(0x110) +#define PWR_L2_PWRACTIVE PWR_CTRL_REG(0x118) +#define PWR_TILER_PRESENT PWR_CTRL_REG(0x140) +#define PWR_TILER_READY PWR_CTRL_REG(0x148) +#define PWR_TILER_PWRTRANS PWR_CTRL_REG(0x150) +#define PWR_TILER_PWRACTIVE PWR_CTRL_REG(0x158) +#define PWR_SHADER_PRESENT PWR_CTRL_REG(0x200) +#define PWR_SHADER_READY PWR_CTRL_REG(0x208) +#define PWR_SHADER_PWRTRANS PWR_CTRL_REG(0x210) +#define PWR_SHADER_PWRACTIVE PWR_CTRL_REG(0x218) +#define PWR_BASE_PRESENT PWR_CTRL_REG(0x380) +#define PWR_BASE_READY PWR_CTRL_REG(0x388) +#define PWR_BASE_PWRTRANS PWR_CTRL_REG(0x390) +#define PWR_BASE_PWRACTIVE PWR_CTRL_REG(0x398) +#define PWR_STACK_PRESENT PWR_CTRL_REG(0x3c0) +#define PWR_STACK_READY PWR_CTRL_REG(0x3c8) +#define PWR_STACK_PWRTRANS PWR_CTRL_REG(0x3d0) + +#endif /* __PANTHOR_PWR_REGS_H__ */ diff --git a/drivers/gpu/drm/panthor/panthor_regs.h b/drivers/gpu/drm/panthor/panthor_regs.h deleted file mode 100644 index 08bf06c452d69..0000000000000 --- a/drivers/gpu/drm/panthor/panthor_regs.h +++ /dev/null @@ -1,291 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 or MIT */ -/* Copyright 2018 Marty E. Plummer */ -/* Copyright 2019 Linaro, Ltd, Rob Herring */ -/* Copyright 2023 Collabora ltd. */ -/* - * Register definitions based on mali_kbase_gpu_regmap.h and - * mali_kbase_gpu_regmap_csf.h - * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved. - */ -#ifndef __PANTHOR_REGS_H__ -#define __PANTHOR_REGS_H__ - -#define GPU_ID 0x0 -#define GPU_ARCH_MAJOR(x) ((x) >> 28) -#define GPU_ARCH_MINOR(x) (((x) & GENMASK(27, 24)) >> 24) -#define GPU_ARCH_REV(x) (((x) & GENMASK(23, 20)) >> 20) -#define GPU_PROD_MAJOR(x) (((x) & GENMASK(19, 16)) >> 16) -#define GPU_VER_MAJOR(x) (((x) & GENMASK(15, 12)) >> 12) -#define GPU_VER_MINOR(x) (((x) & GENMASK(11, 4)) >> 4) -#define GPU_VER_STATUS(x) ((x) & GENMASK(3, 0)) - -#define GPU_L2_FEATURES 0x4 -#define GPU_L2_FEATURES_LINE_SIZE(x) (1 << ((x) & GENMASK(7, 0))) - -#define GPU_CORE_FEATURES 0x8 - -#define GPU_TILER_FEATURES 0xC -#define GPU_MEM_FEATURES 0x10 -#define GROUPS_L2_COHERENT BIT(0) - -#define GPU_MMU_FEATURES 0x14 -#define GPU_MMU_FEATURES_VA_BITS(x) ((x) & GENMASK(7, 0)) -#define GPU_MMU_FEATURES_PA_BITS(x) (((x) >> 8) & GENMASK(7, 0)) -#define GPU_AS_PRESENT 0x18 -#define GPU_CSF_ID 0x1C - -#define GPU_INT_RAWSTAT 0x20 -#define GPU_INT_CLEAR 0x24 -#define GPU_INT_MASK 0x28 -#define GPU_INT_STAT 0x2c -#define GPU_IRQ_FAULT BIT(0) -#define GPU_IRQ_PROTM_FAULT BIT(1) -#define GPU_IRQ_RESET_COMPLETED BIT(8) -#define GPU_IRQ_POWER_CHANGED BIT(9) -#define GPU_IRQ_POWER_CHANGED_ALL BIT(10) -#define GPU_IRQ_CLEAN_CACHES_COMPLETED BIT(17) -#define GPU_IRQ_DOORBELL_MIRROR BIT(18) -#define GPU_IRQ_MCU_STATUS_CHANGED BIT(19) -#define GPU_CMD 0x30 -#define GPU_CMD_DEF(type, payload) ((type) | ((payload) << 8)) -#define GPU_SOFT_RESET GPU_CMD_DEF(1, 1) -#define GPU_HARD_RESET GPU_CMD_DEF(1, 2) -#define CACHE_CLEAN BIT(0) -#define CACHE_INV BIT(1) -#define GPU_FLUSH_CACHES(l2, lsc, oth) \ - GPU_CMD_DEF(4, ((l2) << 0) | ((lsc) << 4) | ((oth) << 8)) - -#define GPU_STATUS 0x34 -#define GPU_STATUS_ACTIVE BIT(0) -#define GPU_STATUS_PWR_ACTIVE BIT(1) -#define GPU_STATUS_PAGE_FAULT BIT(4) -#define GPU_STATUS_PROTM_ACTIVE BIT(7) -#define GPU_STATUS_DBG_ENABLED BIT(8) - -#define GPU_FAULT_STATUS 0x3C -#define GPU_FAULT_ADDR 0x40 -#define GPU_L2_CONFIG 0x48 -#define GPU_L2_CONFIG_ASN_HASH_ENABLE BIT(24) - -#define GPU_PWR_KEY 0x50 -#define GPU_PWR_KEY_UNLOCK 0x2968A819 -#define GPU_PWR_OVERRIDE0 0x54 -#define GPU_PWR_OVERRIDE1 0x58 - -#define GPU_FEATURES 0x60 -#define GPU_FEATURES_RAY_INTERSECTION BIT(2) -#define GPU_FEATURES_RAY_TRAVERSAL BIT(5) - -#define GPU_TIMESTAMP_OFFSET 0x88 -#define GPU_CYCLE_COUNT 0x90 -#define GPU_TIMESTAMP 0x98 - -#define GPU_THREAD_MAX_THREADS 0xA0 -#define GPU_THREAD_MAX_WORKGROUP_SIZE 0xA4 -#define GPU_THREAD_MAX_BARRIER_SIZE 0xA8 -#define GPU_THREAD_FEATURES 0xAC - -#define GPU_TEXTURE_FEATURES(n) (0xB0 + ((n) * 4)) - -#define GPU_SHADER_PRESENT 0x100 -#define GPU_TILER_PRESENT 0x110 -#define GPU_L2_PRESENT 0x120 - -#define SHADER_READY 0x140 -#define TILER_READY 0x150 -#define L2_READY 0x160 - -#define SHADER_PWRON 0x180 -#define TILER_PWRON 0x190 -#define L2_PWRON 0x1A0 - -#define SHADER_PWROFF 0x1C0 -#define TILER_PWROFF 0x1D0 -#define L2_PWROFF 0x1E0 - -#define SHADER_PWRTRANS 0x200 -#define TILER_PWRTRANS 0x210 -#define L2_PWRTRANS 0x220 - -#define SHADER_PWRACTIVE 0x240 -#define TILER_PWRACTIVE 0x250 -#define L2_PWRACTIVE 0x260 - -#define GPU_REVID 0x280 - -#define GPU_ASN_HASH(n) (0x2C0 + ((n) * 4)) - -#define GPU_COHERENCY_FEATURES 0x300 -#define GPU_COHERENCY_PROT_BIT(name) BIT(GPU_COHERENCY_ ## name) - -#define GPU_COHERENCY_PROTOCOL 0x304 -#define GPU_COHERENCY_ACE_LITE 0 -#define GPU_COHERENCY_ACE 1 -#define GPU_COHERENCY_NONE 31 - -#define MCU_CONTROL 0x700 -#define MCU_CONTROL_ENABLE 1 -#define MCU_CONTROL_AUTO 2 -#define MCU_CONTROL_DISABLE 0 - -#define MCU_STATUS 0x704 -#define MCU_STATUS_DISABLED 0 -#define MCU_STATUS_ENABLED 1 -#define MCU_STATUS_HALT 2 -#define MCU_STATUS_FATAL 3 - -/* Job Control regs */ -#define JOB_INT_RAWSTAT 0x1000 -#define JOB_INT_CLEAR 0x1004 -#define JOB_INT_MASK 0x1008 -#define JOB_INT_STAT 0x100c -#define JOB_INT_GLOBAL_IF BIT(31) -#define JOB_INT_CSG_IF(x) BIT(x) - -/* MMU regs */ -#define MMU_INT_RAWSTAT 0x2000 -#define MMU_INT_CLEAR 0x2004 -#define MMU_INT_MASK 0x2008 -#define MMU_INT_STAT 0x200c - -/* AS_COMMAND register commands */ - -#define MMU_BASE 0x2400 -#define MMU_AS_SHIFT 6 -#define MMU_AS(as) (MMU_BASE + ((as) << MMU_AS_SHIFT)) - -#define AS_TRANSTAB(as) (MMU_AS(as) + 0x0) -#define AS_MEMATTR(as) (MMU_AS(as) + 0x8) -#define AS_MEMATTR_AARCH64_INNER_ALLOC_IMPL (2 << 2) -#define AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(w, r) ((3 << 2) | \ - ((w) ? BIT(0) : 0) | \ - ((r) ? BIT(1) : 0)) -#define AS_MEMATTR_AARCH64_SH_MIDGARD_INNER (0 << 4) -#define AS_MEMATTR_AARCH64_SH_CPU_INNER (1 << 4) -#define AS_MEMATTR_AARCH64_SH_CPU_INNER_SHADER_COH (2 << 4) -#define AS_MEMATTR_AARCH64_SHARED (0 << 6) -#define AS_MEMATTR_AARCH64_INNER_OUTER_NC (1 << 6) -#define AS_MEMATTR_AARCH64_INNER_OUTER_WB (2 << 6) -#define AS_MEMATTR_AARCH64_FAULT (3 << 6) -#define AS_LOCKADDR(as) (MMU_AS(as) + 0x10) -#define AS_COMMAND(as) (MMU_AS(as) + 0x18) -#define AS_COMMAND_NOP 0 -#define AS_COMMAND_UPDATE 1 -#define AS_COMMAND_LOCK 2 -#define AS_COMMAND_UNLOCK 3 -#define AS_COMMAND_FLUSH_PT 4 -#define AS_COMMAND_FLUSH_MEM 5 -#define AS_LOCK_REGION_MIN_SIZE (1ULL << 15) -#define AS_FAULTSTATUS(as) (MMU_AS(as) + 0x1C) -#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << 8) -#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0 << 8) -#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1 << 8) -#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2 << 8) -#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3 << 8) -#define AS_FAULTADDRESS(as) (MMU_AS(as) + 0x20) -#define AS_STATUS(as) (MMU_AS(as) + 0x28) -#define AS_STATUS_AS_ACTIVE BIT(0) -#define AS_TRANSCFG(as) (MMU_AS(as) + 0x30) -#define AS_TRANSCFG_ADRMODE_UNMAPPED (1 << 0) -#define AS_TRANSCFG_ADRMODE_IDENTITY (2 << 0) -#define AS_TRANSCFG_ADRMODE_AARCH64_4K (6 << 0) -#define AS_TRANSCFG_ADRMODE_AARCH64_64K (8 << 0) -#define AS_TRANSCFG_INA_BITS(x) ((x) << 6) -#define AS_TRANSCFG_OUTA_BITS(x) ((x) << 14) -#define AS_TRANSCFG_SL_CONCAT BIT(22) -#define AS_TRANSCFG_PTW_MEMATTR_NC (1 << 24) -#define AS_TRANSCFG_PTW_MEMATTR_WB (2 << 24) -#define AS_TRANSCFG_PTW_SH_NS (0 << 28) -#define AS_TRANSCFG_PTW_SH_OS (2 << 28) -#define AS_TRANSCFG_PTW_SH_IS (3 << 28) -#define AS_TRANSCFG_PTW_RA BIT(30) -#define AS_TRANSCFG_DISABLE_HIER_AP BIT(33) -#define AS_TRANSCFG_DISABLE_AF_FAULT BIT(34) -#define AS_TRANSCFG_WXN BIT(35) -#define AS_TRANSCFG_XREADABLE BIT(36) -#define AS_FAULTEXTRA(as) (MMU_AS(as) + 0x38) - -#define CSF_GPU_LATEST_FLUSH_ID 0x10000 - -#define CSF_DOORBELL(i) (0x80000 + ((i) * 0x10000)) -#define CSF_GLB_DOORBELL_ID 0 - -/* PWR Control registers */ - -#define PWR_CONTROL_BASE 0x800 -#define PWR_CTRL_REG(x) (PWR_CONTROL_BASE + (x)) - -#define PWR_INT_RAWSTAT PWR_CTRL_REG(0x0) -#define PWR_INT_CLEAR PWR_CTRL_REG(0x4) -#define PWR_INT_MASK PWR_CTRL_REG(0x8) -#define PWR_INT_STAT PWR_CTRL_REG(0xc) -#define PWR_IRQ_POWER_CHANGED_SINGLE BIT(0) -#define PWR_IRQ_POWER_CHANGED_ALL BIT(1) -#define PWR_IRQ_DELEGATION_CHANGED BIT(2) -#define PWR_IRQ_RESET_COMPLETED BIT(3) -#define PWR_IRQ_RETRACT_COMPLETED BIT(4) -#define PWR_IRQ_INSPECT_COMPLETED BIT(5) -#define PWR_IRQ_COMMAND_NOT_ALLOWED BIT(30) -#define PWR_IRQ_COMMAND_INVALID BIT(31) - -#define PWR_STATUS PWR_CTRL_REG(0x20) -#define PWR_STATUS_ALLOW_L2 BIT_U64(0) -#define PWR_STATUS_ALLOW_TILER BIT_U64(1) -#define PWR_STATUS_ALLOW_SHADER BIT_U64(8) -#define PWR_STATUS_ALLOW_BASE BIT_U64(14) -#define PWR_STATUS_ALLOW_STACK BIT_U64(15) -#define PWR_STATUS_DOMAIN_ALLOWED(x) BIT_U64(x) -#define PWR_STATUS_DELEGATED_L2 BIT_U64(16) -#define PWR_STATUS_DELEGATED_TILER BIT_U64(17) -#define PWR_STATUS_DELEGATED_SHADER BIT_U64(24) -#define PWR_STATUS_DELEGATED_BASE BIT_U64(30) -#define PWR_STATUS_DELEGATED_STACK BIT_U64(31) -#define PWR_STATUS_DELEGATED_SHIFT 16 -#define PWR_STATUS_DOMAIN_DELEGATED(x) BIT_U64((x) + PWR_STATUS_DELEGATED_SHIFT) -#define PWR_STATUS_ALLOW_SOFT_RESET BIT_U64(33) -#define PWR_STATUS_ALLOW_FAST_RESET BIT_U64(34) -#define PWR_STATUS_POWER_PENDING BIT_U64(41) -#define PWR_STATUS_RESET_PENDING BIT_U64(42) -#define PWR_STATUS_RETRACT_PENDING BIT_U64(43) -#define PWR_STATUS_INSPECT_PENDING BIT_U64(44) - -#define PWR_COMMAND PWR_CTRL_REG(0x28) -#define PWR_COMMAND_POWER_UP 0x10 -#define PWR_COMMAND_POWER_DOWN 0x11 -#define PWR_COMMAND_DELEGATE 0x20 -#define PWR_COMMAND_RETRACT 0x21 -#define PWR_COMMAND_RESET_SOFT 0x31 -#define PWR_COMMAND_RESET_FAST 0x32 -#define PWR_COMMAND_INSPECT 0xF0 -#define PWR_COMMAND_DOMAIN_L2 0 -#define PWR_COMMAND_DOMAIN_TILER 1 -#define PWR_COMMAND_DOMAIN_SHADER 8 -#define PWR_COMMAND_DOMAIN_BASE 14 -#define PWR_COMMAND_DOMAIN_STACK 15 -#define PWR_COMMAND_SUBDOMAIN_RTU BIT(0) -#define PWR_COMMAND_DEF(cmd, domain, subdomain) \ - (((subdomain) << 16) | ((domain) << 8) | (cmd)) - -#define PWR_CMDARG PWR_CTRL_REG(0x30) - -#define PWR_L2_PRESENT PWR_CTRL_REG(0x100) -#define PWR_L2_READY PWR_CTRL_REG(0x108) -#define PWR_L2_PWRTRANS PWR_CTRL_REG(0x110) -#define PWR_L2_PWRACTIVE PWR_CTRL_REG(0x118) -#define PWR_TILER_PRESENT PWR_CTRL_REG(0x140) -#define PWR_TILER_READY PWR_CTRL_REG(0x148) -#define PWR_TILER_PWRTRANS PWR_CTRL_REG(0x150) -#define PWR_TILER_PWRACTIVE PWR_CTRL_REG(0x158) -#define PWR_SHADER_PRESENT PWR_CTRL_REG(0x200) -#define PWR_SHADER_READY PWR_CTRL_REG(0x208) -#define PWR_SHADER_PWRTRANS PWR_CTRL_REG(0x210) -#define PWR_SHADER_PWRACTIVE PWR_CTRL_REG(0x218) -#define PWR_BASE_PRESENT PWR_CTRL_REG(0x380) -#define PWR_BASE_READY PWR_CTRL_REG(0x388) -#define PWR_BASE_PWRTRANS PWR_CTRL_REG(0x390) -#define PWR_BASE_PWRACTIVE PWR_CTRL_REG(0x398) -#define PWR_STACK_PRESENT PWR_CTRL_REG(0x3c0) -#define PWR_STACK_READY PWR_CTRL_REG(0x3c8) -#define PWR_STACK_PWRTRANS PWR_CTRL_REG(0x3d0) - -#endif diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c index 2fe04d0f0e3a8..b09953ac0bd63 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.c +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -28,11 +28,12 @@ #include "panthor_devfreq.h" #include "panthor_device.h" #include "panthor_fw.h" +#include "panthor_fw_regs.h" #include "panthor_gem.h" #include "panthor_gpu.h" +#include "panthor_gpu_regs.h" #include "panthor_heap.h" #include "panthor_mmu.h" -#include "panthor_regs.h" #include "panthor_sched.h" /** @@ -1057,7 +1058,8 @@ group_unbind_locked(struct panthor_group *group) /* Tiler OOM events will be re-issued next time the group is scheduled. */ atomic_set(&group->tiler_oom, 0); - cancel_work(&group->tiler_oom_work); + if (cancel_work(&group->tiler_oom_work)) + group_put(group); for (u32 i = 0; i < group->queue_count; i++) group->queues[i]->doorbell_id = -1; @@ -1151,15 +1153,14 @@ queue_suspend_timeout_locked(struct panthor_queue *queue) static void queue_suspend_timeout(struct panthor_queue *queue) { - spin_lock(&queue->fence_ctx.lock); + guard(spinlock_irqsave)(&queue->fence_ctx.lock); queue_suspend_timeout_locked(queue); - spin_unlock(&queue->fence_ctx.lock); } static void queue_resume_timeout(struct panthor_queue *queue) { - spin_lock(&queue->fence_ctx.lock); + guard(spinlock_irqsave)(&queue->fence_ctx.lock); if (queue_timeout_is_suspended(queue)) { mod_delayed_work(queue->scheduler.timeout_wq, @@ -1168,8 +1169,6 @@ queue_resume_timeout(struct panthor_queue *queue) queue->timeout.remaining = MAX_SCHEDULE_TIMEOUT; } - - spin_unlock(&queue->fence_ctx.lock); } /** @@ -1542,7 +1541,7 @@ cs_slot_process_fault_event_locked(struct panthor_device *ptdev, u64 cs_extract = queue->iface.output->extract; struct panthor_job *job; - spin_lock(&queue->fence_ctx.lock); + guard(spinlock_irqsave)(&queue->fence_ctx.lock); list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) { if (cs_extract >= job->ringbuf.end) continue; @@ -1552,7 +1551,6 @@ cs_slot_process_fault_event_locked(struct panthor_device *ptdev, dma_fence_set_error(job->done_fence, -EINVAL); } - spin_unlock(&queue->fence_ctx.lock); } if (group) { @@ -1604,7 +1602,10 @@ static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) if (unlikely(csg_id < 0)) return 0; - if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) { + if (IS_ERR(heaps)) { + ret = -EINVAL; + heaps = NULL; + } else if (frag_end > vt_end || vt_end >= vt_start) { ret = -EINVAL; } else { /* We do the allocation without holding the scheduler lock to avoid @@ -2181,13 +2182,13 @@ group_term_post_processing(struct panthor_group *group) if (!queue) continue; - spin_lock(&queue->fence_ctx.lock); - list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) { - list_move_tail(&job->node, &faulty_jobs); - dma_fence_set_error(job->done_fence, err); - dma_fence_signal_locked(job->done_fence); + scoped_guard(spinlock_irqsave, &queue->fence_ctx.lock) { + list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) { + list_move_tail(&job->node, &faulty_jobs); + dma_fence_set_error(job->done_fence, err); + dma_fence_signal_locked(job->done_fence); + } } - spin_unlock(&queue->fence_ctx.lock); /* Manually update the syncobj seqno to unblock waiters. */ syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj)); @@ -2366,7 +2367,13 @@ tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *c csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); csg_slot = &sched->csg_slots[csg_id]; - group_bind_locked(group, csg_id); + ret = group_bind_locked(group, csg_id); + if (ret) { + panthor_device_schedule_reset(ptdev); + ctx->csg_upd_failed_mask |= BIT(csg_id); + return; + } + csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--); csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, group->state == PANTHOR_CS_GROUP_SUSPENDED ? @@ -2666,7 +2673,14 @@ static void sched_resume_tick(struct panthor_device *ptdev) else delay_jiffies = 0; - sched_queue_delayed_work(sched, tick, delay_jiffies); + /* We schedule immediate ticks when we need to process events on CSGs, + * but those don't change the resched_target because we want the other + * groups to stay scheduled for the remaining of the GPU timeslot they + * were given. Make sure those immediate ticks don't get overruled by + * a sched_queue_delayed_work() that would delay the tick execution. + */ + if (!delayed_work_pending(&sched->tick_work)) + sched_queue_delayed_work(sched, tick, delay_jiffies); } static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) @@ -3046,39 +3060,39 @@ static bool queue_check_job_completion(struct panthor_queue *queue) LIST_HEAD(done_jobs); cookie = dma_fence_begin_signalling(); - spin_lock(&queue->fence_ctx.lock); - list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { - if (!syncobj) { - struct panthor_group *group = job->group; + scoped_guard(spinlock_irqsave, &queue->fence_ctx.lock) { + list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { + if (!syncobj) { + struct panthor_group *group = job->group; - syncobj = group->syncobjs->kmap + - (job->queue_idx * sizeof(*syncobj)); - } + syncobj = group->syncobjs->kmap + + (job->queue_idx * sizeof(*syncobj)); + } - if (syncobj->seqno < job->done_fence->seqno) - break; + if (syncobj->seqno < job->done_fence->seqno) + break; - list_move_tail(&job->node, &done_jobs); - dma_fence_signal_locked(job->done_fence); - } + list_move_tail(&job->node, &done_jobs); + dma_fence_signal_locked(job->done_fence); + } - if (list_empty(&queue->fence_ctx.in_flight_jobs)) { - /* If we have no job left, we cancel the timer, and reset remaining - * time to its default so it can be restarted next time - * queue_resume_timeout() is called. - */ - queue_suspend_timeout_locked(queue); + if (list_empty(&queue->fence_ctx.in_flight_jobs)) { + /* If we have no job left, we cancel the timer, and reset remaining + * time to its default so it can be restarted next time + * queue_resume_timeout() is called. + */ + queue_suspend_timeout_locked(queue); - /* If there's no job pending, we consider it progress to avoid a - * spurious timeout if the timeout handler and the sync update - * handler raced. - */ - progress = true; - } else if (!list_empty(&done_jobs)) { - queue_reset_timeout_locked(queue); - progress = true; + /* If there's no job pending, we consider it progress to avoid a + * spurious timeout if the timeout handler and the sync update + * handler raced. + */ + progress = true; + } else if (!list_empty(&done_jobs)) { + queue_reset_timeout_locked(queue); + progress = true; + } } - spin_unlock(&queue->fence_ctx.lock); dma_fence_end_signalling(cookie); list_for_each_entry_safe(job, job_tmp, &done_jobs, node) { @@ -3343,9 +3357,8 @@ queue_run_job(struct drm_sched_job *sched_job) job->ringbuf.end = job->ringbuf.start + (instrs.count * sizeof(u64)); panthor_job_get(&job->base); - spin_lock(&queue->fence_ctx.lock); - list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs); - spin_unlock(&queue->fence_ctx.lock); + scoped_guard(spinlock_irqsave, &queue->fence_ctx.lock) + list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs); /* Make sure the ring buffer is updated before the INSERT * register. @@ -3370,7 +3383,7 @@ queue_run_job(struct drm_sched_job *sched_job) if (resume_tick) sched_resume_tick(ptdev); - gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); + gpu_write(ptdev->iomem, CSF_DOORBELL(queue->doorbell_id), 1); if (!sched->pm.has_ref && !(group->blocked_queues & BIT(job->queue_idx))) { pm_runtime_get(ptdev->base.dev); diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c index 20fc87409f2e4..8ce180e22d1d1 100644 --- a/drivers/gpu/drm/radeon/radeon_gem.c +++ b/drivers/gpu/drm/radeon/radeon_gem.c @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -812,6 +813,7 @@ int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tile int aligned = width; int align_large = (ASIC_IS_AVIVO(rdev)) || tiled; int pitch_mask = 0; + int pitch; switch (cpp) { case 1: @@ -826,9 +828,12 @@ int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tile break; } - aligned += pitch_mask; + if (check_add_overflow(aligned, pitch_mask, &aligned)) + return 0; aligned &= ~pitch_mask; - return aligned * cpp; + if (check_mul_overflow(aligned, cpp, &pitch)) + return 0; + return pitch; } int radeon_mode_dumb_create(struct drm_file *file_priv, @@ -842,8 +847,12 @@ int radeon_mode_dumb_create(struct drm_file *file_priv, args->pitch = radeon_align_pitch(rdev, args->width, DIV_ROUND_UP(args->bpp, 8), 0); + if (!args->pitch) + return -EINVAL; args->size = (u64)args->pitch * args->height; args->size = ALIGN(args->size, PAGE_SIZE); + if (!args->size) + return -EINVAL; r = radeon_gem_object_create(rdev, args->size, 0, RADEON_GEM_DOMAIN_VRAM, 0, diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index 581ae20c46e4b..a5dff072c1ac0 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -356,8 +356,10 @@ int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring, /* restore the saved ring content */ r = radeon_ring_lock(rdev, ring, size); - if (r) + if (r) { + kvfree(data); return r; + } for (i = 0; i < size; ++i) { radeon_ring_write(ring, data[i]); diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c index a87a301326c7a..0d0cf10225bb5 100644 --- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c +++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c @@ -1441,6 +1441,11 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev) return dev_err_probe(dsi->dev, PTR_ERR(dsi->prstc), "failed to get prst\n"); + dsi->dcs_buf_virt = dmam_alloc_coherent(dsi->dev, RZG2L_DCS_BUF_SIZE, + &dsi->dcs_buf_phys, GFP_KERNEL); + if (!dsi->dcs_buf_virt) + return -ENOMEM; + platform_set_drvdata(pdev, dsi); pm_runtime_enable(dsi->dev); @@ -1473,11 +1478,6 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev) if (ret < 0) goto err_pm_disable; - dsi->dcs_buf_virt = dma_alloc_coherent(dsi->host.dev, RZG2L_DCS_BUF_SIZE, - &dsi->dcs_buf_phys, GFP_KERNEL); - if (!dsi->dcs_buf_virt) - return -ENOMEM; - return 0; err_phy: @@ -1492,8 +1492,6 @@ static void rzg2l_mipi_dsi_remove(struct platform_device *pdev) { struct rzg2l_mipi_dsi *dsi = platform_get_drvdata(pdev); - dma_free_coherent(dsi->host.dev, RZG2L_DCS_BUF_SIZE, dsi->dcs_buf_virt, - dsi->dcs_buf_phys); mipi_dsi_host_unregister(&dsi->host); pm_runtime_disable(&pdev->dev); } diff --git a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c index dac3d202971ed..8945a245398ca 100644 --- a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -82,7 +83,7 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void * struct drm_connector *connector; int ret; - dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL); + dp = drmm_kzalloc(drm_dev, sizeof(*dp), GFP_KERNEL); if (!dp) return -ENOMEM; @@ -132,9 +133,7 @@ static int dw_dp_probe(struct platform_device *pdev) static void dw_dp_remove(struct platform_device *pdev) { - struct rockchip_dw_dp *dp = platform_get_drvdata(pdev); - - component_del(dp->dev, &dw_dp_rockchip_component_ops); + component_del(&pdev->dev, &dw_dp_rockchip_component_ops); } static const struct dw_dp_plat_data rk3588_dp_plat_data = { diff --git a/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c index 97c20500f7901..28e6fb09aae73 100644 --- a/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c +++ b/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c @@ -14,6 +14,7 @@ #include #include +#include #include #include "rockchip_drm_drv.h" @@ -90,7 +91,7 @@ static int inno_hdmi_rockchip_bind(struct device *dev, struct device *master, vo const struct inno_hdmi_plat_data *plat_data; int ret; - hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL); + hdmi = drmm_kzalloc(drm, sizeof(*hdmi), GFP_KERNEL); if (!hdmi) return -ENOMEM; diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c index 09d14a072d274..b188539dca0b2 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c @@ -334,7 +334,7 @@ void rockchip_gem_free_object(struct drm_gem_object *obj) struct rockchip_drm_private *private = drm->dev_private; struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); - if (obj->import_attach) { + if (drm_gem_is_imported(obj)) { if (private->domain) { rockchip_gem_iommu_unmap(rk_obj); } else { diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c index 06370b7e0e567..c6734a87ef0af 100644 --- a/drivers/gpu/drm/tegra/dc.c +++ b/drivers/gpu/drm/tegra/dc.c @@ -101,8 +101,10 @@ bool tegra_dc_has_output(struct tegra_dc *dc, struct device *dev) int err; of_for_each_phandle(&it, err, np, "nvidia,outputs", NULL, 0) - if (it.node == dev->of_node) + if (it.node == dev->of_node) { + of_node_put(it.node); return true; + } return false; } diff --git a/drivers/gpu/drm/tegra/fbdev.c b/drivers/gpu/drm/tegra/fbdev.c index 8f40882aa76ec..19e39fa54bfae 100644 --- a/drivers/gpu/drm/tegra/fbdev.c +++ b/drivers/gpu/drm/tegra/fbdev.c @@ -110,7 +110,6 @@ int tegra_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, helper->funcs = &tegra_fbdev_helper_funcs; helper->fb = fb; - helper->info = info; info->fbops = &tegra_fb_ops; diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index d2bae88ad545f..436394e048125 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -69,7 +69,7 @@ static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_ return ERR_PTR(-ENOMEM); kref_init(&map->ref); - map->bo = host1x_bo_get(bo); + map->bo = bo; map->direction = direction; map->dev = dev; @@ -170,7 +170,6 @@ static void tegra_bo_unpin(struct host1x_bo_mapping *map) kfree(map->sgt); } - host1x_bo_put(map->bo); kfree(map); } @@ -235,6 +234,7 @@ static const struct host1x_bo_ops tegra_bo_ops = { static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo) { int prot = IOMMU_READ | IOMMU_WRITE; + ssize_t size; int err; if (bo->mm) @@ -256,13 +256,15 @@ static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo) bo->iova = bo->mm->start; - bo->size = iommu_map_sgtable(tegra->domain, bo->iova, bo->sgt, prot); - if (!bo->size) { + size = iommu_map_sgtable(tegra->domain, bo->iova, bo->sgt, prot); + if (size < 0) { dev_err(tegra->drm->dev, "failed to map buffer\n"); - err = -ENOMEM; + err = size; goto remove; } + bo->size = size; + mutex_unlock(&tegra->mm_lock); return 0; @@ -509,17 +511,9 @@ static struct tegra_bo *tegra_bo_import(struct drm_device *drm, void tegra_bo_free_object(struct drm_gem_object *gem) { struct tegra_drm *tegra = gem->dev->dev_private; - struct host1x_bo_mapping *mapping, *tmp; struct tegra_bo *bo = to_tegra_bo(gem); - /* remove all mappings of this buffer object from any caches */ - list_for_each_entry_safe(mapping, tmp, &bo->base.mappings, list) { - if (mapping->cache) - host1x_bo_unpin(mapping); - else - dev_err(gem->dev->dev, "mapping %p stale for device %s\n", mapping, - dev_name(mapping->dev)); - } + host1x_bo_clear_cached_mappings(&bo->base); if (tegra->domain) { tegra_bo_iommu_unmap(tegra, bo); diff --git a/drivers/gpu/drm/tegra/gr2d.c b/drivers/gpu/drm/tegra/gr2d.c index 21f4dd0fa6aff..892e3450b281e 100644 --- a/drivers/gpu/drm/tegra/gr2d.c +++ b/drivers/gpu/drm/tegra/gr2d.c @@ -100,9 +100,6 @@ static int gr2d_exit(struct host1x_client *client) if (err < 0) return err; - pm_runtime_dont_use_autosuspend(client->dev); - pm_runtime_force_suspend(client->dev); - host1x_client_iommu_detach(client); host1x_syncpt_put(client->syncpts[0]); host1x_channel_put(gr2d->channel); @@ -276,15 +273,21 @@ static int gr2d_probe(struct platform_device *pdev) if (err) return err; + /* initialize address register map */ + for (i = 0; i < ARRAY_SIZE(gr2d_addr_regs); i++) + set_bit(gr2d_addr_regs[i], gr2d->addr_regs); + + pm_runtime_enable(dev); + err = host1x_client_register(&gr2d->client.base); if (err < 0) { + pm_runtime_disable(dev); dev_err(dev, "failed to register host1x client: %d\n", err); return err; } - /* initialize address register map */ - for (i = 0; i < ARRAY_SIZE(gr2d_addr_regs); i++) - set_bit(gr2d_addr_regs[i], gr2d->addr_regs); + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, 500); return 0; } @@ -367,10 +370,6 @@ static int __maybe_unused gr2d_runtime_resume(struct device *dev) goto disable_clk; } - pm_runtime_enable(dev); - pm_runtime_use_autosuspend(dev); - pm_runtime_set_autosuspend_delay(dev, 500); - return 0; disable_clk: diff --git a/drivers/gpu/drm/tegra/gr3d.c b/drivers/gpu/drm/tegra/gr3d.c index 42e9656ab80c9..388e47943d5ec 100644 --- a/drivers/gpu/drm/tegra/gr3d.c +++ b/drivers/gpu/drm/tegra/gr3d.c @@ -109,9 +109,6 @@ static int gr3d_exit(struct host1x_client *client) if (err < 0) return err; - pm_runtime_dont_use_autosuspend(client->dev); - pm_runtime_force_suspend(client->dev); - host1x_client_iommu_detach(client); host1x_syncpt_put(client->syncpts[0]); host1x_channel_put(gr3d->channel); @@ -506,16 +503,22 @@ static int gr3d_probe(struct platform_device *pdev) if (err) return err; + /* initialize address register map */ + for (i = 0; i < ARRAY_SIZE(gr3d_addr_regs); i++) + set_bit(gr3d_addr_regs[i], gr3d->addr_regs); + + pm_runtime_enable(&pdev->dev); + err = host1x_client_register(&gr3d->client.base); if (err < 0) { + pm_runtime_disable(&pdev->dev); dev_err(&pdev->dev, "failed to register host1x client: %d\n", err); return err; } - /* initialize address register map */ - for (i = 0; i < ARRAY_SIZE(gr3d_addr_regs); i++) - set_bit(gr3d_addr_regs[i], gr3d->addr_regs); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_set_autosuspend_delay(&pdev->dev, 500); return 0; } @@ -578,10 +581,6 @@ static int __maybe_unused gr3d_runtime_resume(struct device *dev) goto disable_clk; } - pm_runtime_enable(dev); - pm_runtime_use_autosuspend(dev); - pm_runtime_set_autosuspend_delay(dev, 500); - return 0; disable_clk: diff --git a/drivers/gpu/drm/tegra/submit.c b/drivers/gpu/drm/tegra/submit.c index 3009b8b9e6197..e5841857c9378 100644 --- a/drivers/gpu/drm/tegra/submit.c +++ b/drivers/gpu/drm/tegra/submit.c @@ -76,7 +76,7 @@ gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction return ERR_PTR(-ENOMEM); kref_init(&map->ref); - map->bo = host1x_bo_get(bo); + map->bo = bo; map->direction = direction; map->dev = dev; @@ -117,7 +117,6 @@ static void gather_bo_unpin(struct host1x_bo_mapping *map) dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); sg_free_table(map->sgt); kfree(map->sgt); - host1x_bo_put(map->bo); kfree(map); } diff --git a/drivers/gpu/drm/tidss/tidss_kms.c b/drivers/gpu/drm/tidss/tidss_kms.c index 8bb93194e5ac6..b4779c09a1bfa 100644 --- a/drivers/gpu/drm/tidss/tidss_kms.c +++ b/drivers/gpu/drm/tidss/tidss_kms.c @@ -287,8 +287,6 @@ int tidss_modeset_init(struct tidss_device *tidss) if (ret) return ret; - drm_mode_config_reset(ddev); - dev_dbg(tidss->dev, "%s done\n", __func__); return 0; diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 3ddd53b6f4376..f6a9bfa3548ce 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -484,6 +484,8 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, sizeof(indirect_csd.wg_uniform_offsets)); info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); + if (!info->indirect) + return -ENOENT; return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, &info->job, &info->clean_job, diff --git a/drivers/gpu/drm/verisilicon/vs_primary_plane.c b/drivers/gpu/drm/verisilicon/vs_primary_plane.c index e8fcb5958615c..a383b70f1ea0f 100644 --- a/drivers/gpu/drm/verisilicon/vs_primary_plane.c +++ b/drivers/gpu/drm/verisilicon/vs_primary_plane.c @@ -26,14 +26,10 @@ static int vs_primary_plane_atomic_check(struct drm_plane *plane, struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane); struct drm_crtc *crtc = new_plane_state->crtc; - struct drm_crtc_state *crtc_state; + struct drm_crtc_state *crtc_state = NULL; - if (!crtc) - return 0; - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - if (WARN_ON(!crtc_state)) - return -EINVAL; + if (crtc) + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); return drm_atomic_helper_check_plane_state(new_plane_state, crtc_state, diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 03242e8b3d876..de36c8b640baf 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -16,14 +16,14 @@ subdir-ccflags-y += -I$(obj) -I$(src) hostprogs := xe_gen_wa_oob generated_oob := $(obj)/generated/xe_wa_oob.c $(obj)/generated/xe_wa_oob.h quiet_cmd_wa_oob = GEN $(notdir $(generated_oob)) - cmd_wa_oob = mkdir -p $(@D); $^ $(generated_oob) + cmd_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_wa_oob.rules $(generated_oob) $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \ $(src)/xe_wa_oob.rules $(call cmd,wa_oob) generated_device_oob := $(obj)/generated/xe_device_wa_oob.c $(obj)/generated/xe_device_wa_oob.h quiet_cmd_device_wa_oob = GEN $(notdir $(generated_device_oob)) - cmd_device_wa_oob = mkdir -p $(@D); $^ $(generated_device_oob) + cmd_device_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_device_wa_oob.rules $(generated_device_oob) $(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe_gen_wa_oob \ $(src)/xe_device_wa_oob.rules $(call cmd,device_wa_oob) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index f3179b31f13e3..8a32fb469147d 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -9,7 +9,6 @@ #include #include -#include #include #define PLATFORM_CASE(platform__, graphics_step__) \ diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c index 84b66147bf494..81020b4b344e0 100644 --- a/drivers/gpu/drm/xe/xe_drm_client.c +++ b/drivers/gpu/drm/xe/xe_drm_client.c @@ -168,10 +168,20 @@ static void bo_meminfo(struct xe_bo *bo, struct drm_memory_stats stats[TTM_NUM_MEM_TYPES]) { u64 sz = xe_bo_size(bo); - u32 mem_type = bo->ttm.resource->mem_type; + u32 mem_type; xe_bo_assert_held(bo); + /* + * The resource can be NULL if the BO has been purged, plus maybe some + * other cases. Either way there shouldn't be any memory to account for, + * or a current resource to account this against, so skip for now. + */ + if (!bo->ttm.resource) + return; + + mem_type = bo->ttm.resource->mem_type; + if (drm_gem_object_is_shared_for_memory_stats(&bo->ttm.base)) stats[mem_type].shared += sz; else diff --git a/drivers/gpu/drm/xe/xe_guc_relay.c b/drivers/gpu/drm/xe/xe_guc_relay.c index 577a315854afc..eed0a750d2eb2 100644 --- a/drivers/gpu/drm/xe/xe_guc_relay.c +++ b/drivers/gpu/drm/xe/xe_guc_relay.c @@ -689,12 +689,17 @@ static int relay_action_handler(struct xe_guc_relay *relay, u32 origin, return relay_testloop_action_handler(relay, origin, msg, len, response, size); type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); + relay_assert(relay, guc_hxg_type_is_action(type)); - if (IS_SRIOV_PF(relay_to_xe(relay))) - ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, response, size); - else + if (IS_SRIOV_PF(relay_to_xe(relay))) { + if (type == GUC_HXG_TYPE_REQUEST) + ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, + response, size); + else + ret = -EOPNOTSUPP; + } else { ret = -EOPNOTSUPP; - + } if (type == GUC_HXG_TYPE_EVENT) relay_assert(relay, ret <= 0); diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 6dd05fac65956..a7441a66e94c2 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -629,7 +629,7 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, hwe->exl_port = xe_execlist_port_create(xe, hwe); if (IS_ERR(hwe->exl_port)) { err = PTR_ERR(hwe->exl_port); - goto err_hwsp; + goto err_name; } } else { /* GSCCS has a special interrupt for reset */ @@ -649,8 +649,6 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, return devm_add_action_or_reset(xe->drm.dev, hw_engine_fini, hwe); -err_hwsp: - xe_bo_unpin_map_no_vm(hwe->hwsp); err_name: hwe->name = NULL; diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 8e5f4f0dea3f9..c3a69103b9bbf 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -432,6 +432,7 @@ xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, struct xe_pt_stage_bind_walk *xe_walk) { + struct xe_bo *bo = xe_vma_bo(xe_walk->vma); u64 size, dma; if (level > MAX_HUGEPTE_LEVEL) @@ -445,8 +446,8 @@ static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, if (next - xe_walk->va_curs_start > xe_walk->curs->size) return false; - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) + /* null VMA's and purged BO's do not have dma addresses */ + if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo))) return true; /* if we are clearing page table, no dma addresses*/ @@ -467,6 +468,7 @@ static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, static bool xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) { + struct xe_bo *bo = xe_vma_bo(xe_walk->vma); struct xe_res_cursor curs = *xe_walk->curs; if (!IS_ALIGNED(addr, SZ_64K)) @@ -475,8 +477,8 @@ xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) if (next > xe_walk->l0_end_addr) return false; - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) + /* null VMA's and purged BO's do not have dma addresses */ + if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo))) return true; xe_res_next(&curs, addr - xe_walk->va_curs_start); @@ -707,7 +709,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, { struct xe_device *xe = tile_to_xe(tile); struct xe_bo *bo = xe_vma_bo(vma); - struct xe_res_cursor curs; + struct xe_res_cursor curs = {}; struct xe_vm *vm = xe_vma_vm(vma); struct xe_pt_stage_bind_walk xe_walk = { .base = { @@ -884,12 +886,20 @@ static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, { struct xe_pt_zap_ptes_walk *xe_walk = container_of(walk, typeof(*xe_walk), base); - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + struct xe_pt *xe_child; pgoff_t end_offset; - XE_WARN_ON(!*child); XE_WARN_ON(!level); + /* + * Below would be unexpected behavior that needs to be root caused + * but better warn and bail than crash the driver. + */ + if (XE_WARN_ON(!*child)) + return 0; + + xe_child = container_of(*child, typeof(*xe_child), base); + /* * Note that we're called from an entry callback, and we're dealing * with the child of that entry rather than the parent, so need to @@ -1077,7 +1087,7 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma) xe_pt_commit_prepare_locks_assert(vma); if (xe_vma_is_userptr(vma)) - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); } static void xe_pt_commit(struct xe_vma *vma, @@ -1397,6 +1407,38 @@ static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) pt_update_ops, rftree); } +#if IS_ENABLED(CONFIG_DRM_GPUSVM) +/* + * Acquire/release the svm notifier_lock around xe_pt_svm_userptr_pre_commit() + * and the matching late release in xe_pt_update_ops_run(). Read mode by + * default; write mode when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, + * because a userptr op in this critical section may invoke the injected + * xe_vma_userptr_force_invalidate() path that calls + * drm_gpusvm_unmap_pages() with ctx->in_notifier=true, which requires the + * lock held for write. + */ +static void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + down_write(&vm->svm.gpusvm.notifier_lock); +#else + xe_svm_notifier_lock(vm); +#endif +} + +static void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + up_write(&vm->svm.gpusvm.notifier_lock); +#else + xe_svm_notifier_unlock(vm); +#endif +} +#else +static inline void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) { } +static inline void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) { } +#endif + #if IS_ENABLED(CONFIG_DRM_GPUSVM) #ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT @@ -1428,7 +1470,7 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma, struct xe_userptr_vma *uvma; unsigned long notifier_seq; - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); if (!xe_vma_is_userptr(vma)) return 0; @@ -1458,7 +1500,7 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, { int err = 0; - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); switch (op->base.op) { case DRM_GPUVA_OP_MAP: @@ -1530,12 +1572,12 @@ static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) if (err) return err; - xe_svm_notifier_lock(vm); + xe_pt_svm_userptr_notifier_lock(vm); list_for_each_entry(op, &vops->list, link) { err = op_check_svm_userptr(vm, op, pt_update_ops); if (err) { - xe_svm_notifier_unlock(vm); + xe_pt_svm_userptr_notifier_unlock(vm); break; } } @@ -2371,7 +2413,7 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, vma->tile_invalidated & ~BIT(tile->id)); vma->tile_staged &= ~BIT(tile->id); if (xe_vma_is_userptr(vma)) { - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); to_userptr_vma(vma)->userptr.initial_bind = true; } @@ -2407,7 +2449,7 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, if (!vma->tile_present) { list_del_init(&vma->combined_links.rebind); if (xe_vma_is_userptr(vma)) { - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); spin_lock(&vm->userptr.invalidated_lock); list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); @@ -2683,7 +2725,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) } if (pt_update_ops->needs_svm_lock) - xe_svm_notifier_unlock(vm); + xe_pt_svm_userptr_notifier_unlock(vm); /* * The last fence is only used for zero bind queue idling; migrate diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index b7b8eeacf196b..3ca46a6f98c73 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -394,8 +394,19 @@ static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_inst #define xe_svm_assert_in_notifier(vm__) \ lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) -#define xe_svm_assert_held_read(vm__) \ +/* + * Assert the svm notifier_lock is held. Read mode by default; write mode + * when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, because that path forces + * a userptr invalidation that ends in drm_gpusvm_unmap_pages() with + * ctx->in_notifier=true, which requires the lock held for write. + */ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) +#define xe_svm_assert_held_read_or_inject_write(vm__) \ + lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) +#else +#define xe_svm_assert_held_read_or_inject_write(vm__) \ lockdep_assert_held_read(&(vm__)->svm.gpusvm.notifier_lock) +#endif #define xe_svm_notifier_lock(vm__) \ drm_gpusvm_notifier_lock(&(vm__)->svm.gpusvm) @@ -409,7 +420,7 @@ static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_inst #else #define xe_svm_assert_in_notifier(...) do {} while (0) -static inline void xe_svm_assert_held_read(struct xe_vm *vm) +static inline void xe_svm_assert_held_read_or_inject_write(struct xe_vm *vm) { } diff --git a/drivers/gpu/drm/xe/xe_userptr.c b/drivers/gpu/drm/xe/xe_userptr.c index 6761005c0b900..6f71bc66b14eb 100644 --- a/drivers/gpu/drm/xe/xe_userptr.c +++ b/drivers/gpu/drm/xe/xe_userptr.c @@ -269,7 +269,7 @@ static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { */ void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma) { - static struct mmu_interval_notifier_finish *finish; + struct mmu_interval_notifier_finish *finish; struct xe_vm *vm = xe_vma_vm(&uvma->vma); /* Protect against concurrent userptr pinning */ diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index c4fb290041956..246fe18431428 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -643,7 +643,7 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil xe_device_is_l2_flush_optimized(xe) && (pat_index != 19 && coh_mode != XE_COH_2WAY))) { err = -EINVAL; - goto madv_fini; + goto free_vmas; } } diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c index f814eb4941c02..c273a4476f237 100644 --- a/drivers/gpu/host1x/bus.c +++ b/drivers/gpu/host1x/bus.c @@ -485,7 +485,7 @@ static int host1x_device_add(struct host1x *host1x, err = host1x_device_parse_dt(device, driver); if (err < 0) { - kfree(device); + put_device(&device->dev); return err; } @@ -887,6 +887,20 @@ int host1x_client_resume(struct host1x_client *client) } EXPORT_SYMBOL(host1x_client_resume); +/** + * host1x_bo_pin() - Create a DMA mapping for the buffer object + * @dev: Device onto which DMA map to + * @bo: Buffer object to map + * @dir: DMA direction + * @cache: Cache in which to store mapping, or NULL + * + * Creates a DMA mapping pointing to @bo for @dev. The refcount of @bo is incremented + * until host1x_bo_unpin is called. + * + * If @cache is specified, the mapping is also stored in the cache and not released + * until @bo is freed (refcount drops to zero). This improves performance when a buffer + * is pinned and unpinned frequently as in the case of display use. + */ struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction dir, struct host1x_bo_cache *cache) @@ -899,6 +913,7 @@ struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo list_for_each_entry(mapping, &cache->mappings, entry) { if (mapping->bo == bo && mapping->direction == dir) { kref_get(&mapping->ref); + host1x_bo_get(bo); goto unlock; } } @@ -908,6 +923,8 @@ struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo if (IS_ERR(mapping)) goto unlock; + host1x_bo_get(bo); + spin_lock(&mapping->bo->lock); list_add_tail(&mapping->list, &bo->mappings); spin_unlock(&mapping->bo->lock); @@ -918,7 +935,12 @@ struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo list_add_tail(&mapping->entry, &cache->mappings); - /* bump reference count to track the copy in the cache */ + /* + * Bump the mapping reference count to track the mapping in the cache, + * but do not bump the BO's refcount. This allows the BO to still get freed, + * triggering the release of the cache mapping through + * host1x_bo_clear_cached_mappings. + */ kref_get(&mapping->ref); } @@ -948,9 +970,17 @@ static void __host1x_bo_unpin(struct kref *ref) mapping->bo->ops->unpin(mapping); } +/** + * host1x_bo_unpin() - Release an established DMA mapping of a buffer object + * @mapping: Mapping to release + * + * Unmaps the given @mapping, unless it is cached. Decreases the refcount on + * the underlying buffer object. + */ void host1x_bo_unpin(struct host1x_bo_mapping *mapping) { struct host1x_bo_cache *cache = mapping->cache; + struct host1x_bo *bo = mapping->bo; if (cache) mutex_lock(&cache->lock); @@ -959,5 +989,33 @@ void host1x_bo_unpin(struct host1x_bo_mapping *mapping) if (cache) mutex_unlock(&cache->lock); + + host1x_bo_put(bo); } EXPORT_SYMBOL(host1x_bo_unpin); + +/** + * host1x_bo_clear_cached_mappings() - Remove all cached mappings pointing at a bo + * @bo: Buffer object to release mappings of + * + * Drops references to any mappings pointing to @bo left in any caches. This must + * be called by any host1x_bo implementers that may be pinned with caching enabled + * before freeing the bo. + */ +void host1x_bo_clear_cached_mappings(struct host1x_bo *bo) +{ + struct host1x_bo_mapping *mapping, *tmp; + struct host1x_bo_cache *cache; + + list_for_each_entry_safe(mapping, tmp, &bo->mappings, list) { + cache = mapping->cache; + if (WARN_ON(!cache)) + continue; + + mutex_lock(&mapping->cache->lock); + WARN_ON(kref_read(&mapping->ref) != 1); + __host1x_bo_unpin(&mapping->ref); + mutex_unlock(&mapping->cache->lock); + } +} +EXPORT_SYMBOL(host1x_bo_clear_cached_mappings); diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c index 3ed49e1fd9332..70bda32f1ff48 100644 --- a/drivers/gpu/host1x/job.c +++ b/drivers/gpu/host1x/job.c @@ -235,6 +235,8 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) } if (host->domain) { + ssize_t map_err; + for_each_sgtable_sg(map->sgt, sg, j) gather_size += sg->length; @@ -248,11 +250,11 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto put; } - err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc), - map->sgt, IOMMU_READ); - if (err == 0) { + map_err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc), + map->sgt, IOMMU_READ); + if (map_err < 0) { __free_iova(&host->iova, alloc); - err = -EINVAL; + err = map_err; goto put; } diff --git a/drivers/gpu/host1x/mipi.c b/drivers/gpu/host1x/mipi.c index 01513b775d899..9886814239817 100644 --- a/drivers/gpu/host1x/mipi.c +++ b/drivers/gpu/host1x/mipi.c @@ -114,8 +114,10 @@ struct tegra_mipi_device *tegra_mipi_request(struct device *device, if (err < 0) return ERR_PTR(err); - if (provider.np != args.np) - return ERR_PTR(-ENODEV); + if (provider.np != args.np) { + err = -ENODEV; + goto out; + } mipidev = kzalloc_obj(*mipidev); if (!mipidev) { diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c index d0130658091b0..536f6d01fd14c 100644 --- a/drivers/hid/bpf/hid_bpf_dispatch.c +++ b/drivers/hid/bpf/hid_bpf_dispatch.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "hid_bpf_dispatch.h" const struct hid_ops *hid_ops; @@ -296,10 +297,12 @@ __bpf_kfunc __u8 * hid_bpf_get_data(struct hid_bpf_ctx *ctx, unsigned int offset, const size_t rdwr_buf_size) { struct hid_bpf_ctx_kern *ctx_kern; + size_t end; ctx_kern = container_of(ctx, struct hid_bpf_ctx_kern, ctx); - if (rdwr_buf_size + offset > ctx->allocated_size) + if (check_add_overflow(rdwr_buf_size, offset, &end) || + end > ctx->allocated_size) return NULL; return ctx_kern->data + offset; diff --git a/drivers/hid/hid-appleir.c b/drivers/hid/hid-appleir.c index 5e8ced7bc05a3..adaa44a858edd 100644 --- a/drivers/hid/hid-appleir.c +++ b/drivers/hid/hid-appleir.c @@ -109,9 +109,10 @@ struct appleir { struct hid_device *hid; unsigned short keymap[ARRAY_SIZE(appleir_key_table)]; struct timer_list key_up_timer; /* timer for key up */ - spinlock_t lock; /* protects .current_key */ + spinlock_t lock; /* protects .current_key, .removing */ int current_key; /* the currently pressed key */ int prev_key_idx; /* key index in a 2 packets message */ + bool removing; /* set during teardown; gates input_dev access */ }; static int get_key(int data) @@ -172,7 +173,7 @@ static void key_up_tick(struct timer_list *t) unsigned long flags; spin_lock_irqsave(&appleir->lock, flags); - if (appleir->current_key) { + if (!appleir->removing && appleir->current_key) { key_up(hid, appleir, appleir->current_key); appleir->current_key = 0; } @@ -195,6 +196,10 @@ static int appleir_raw_event(struct hid_device *hid, struct hid_report *report, int index; spin_lock_irqsave(&appleir->lock, flags); + if (appleir->removing) { + spin_unlock_irqrestore(&appleir->lock, flags); + goto out; + } /* * If we already have a key down, take it up before marking * this one down @@ -229,17 +234,25 @@ static int appleir_raw_event(struct hid_device *hid, struct hid_report *report, appleir->prev_key_idx = 0; if (!memcmp(data, keyrepeat, sizeof(keyrepeat))) { - key_down(hid, appleir, appleir->current_key); - /* - * Remote doesn't do key up, either pull them up, in the test - * above, or here set a timer which pulls them up after 1/8 s - */ - mod_timer(&appleir->key_up_timer, jiffies + HZ / 8); + spin_lock_irqsave(&appleir->lock, flags); + if (!appleir->removing) { + key_down(hid, appleir, appleir->current_key); + /* + * Remote doesn't do key up, either pull them up, in + * the test above, or here set a timer which pulls them + * up after 1/8 s + */ + mod_timer(&appleir->key_up_timer, jiffies + HZ / 8); + } + spin_unlock_irqrestore(&appleir->lock, flags); goto out; } if (!memcmp(data, flatbattery, sizeof(flatbattery))) { - battery_flat(appleir); + spin_lock_irqsave(&appleir->lock, flags); + if (!appleir->removing) + battery_flat(appleir); + spin_unlock_irqrestore(&appleir->lock, flags); /* Fall through */ } @@ -318,8 +331,20 @@ static int appleir_probe(struct hid_device *hid, const struct hid_device_id *id) static void appleir_remove(struct hid_device *hid) { struct appleir *appleir = hid_get_drvdata(hid); + unsigned long flags; + + /* + * Mark the driver as tearing down so that any concurrent raw_event + * (e.g. from a USB URB completion that hid_hw_stop() has not yet + * killed) and the key_up_timer softirq stop touching input_dev + * before hid_hw_stop() frees it via hidinput_disconnect(). + */ + spin_lock_irqsave(&appleir->lock, flags); + appleir->removing = true; + spin_unlock_irqrestore(&appleir->lock, flags); + + timer_shutdown_sync(&appleir->key_up_timer); hid_hw_stop(hid); - timer_delete_sync(&appleir->key_up_timer); } static const struct hid_device_id appleir_devices[] = { diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 41a79e43c82b3..cf123347a2af7 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -2045,6 +2045,13 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 * u8 *cdata = data; int ret = 0; + if (report_enum->numbered && (size < 1 || bufsize < 1)) { + hid_warn_ratelimited(hid, + "Event data for numbered report is too short (%d vs %zu)\n", + size, bufsize); + return -EINVAL; + } + report = hid_get_report(report_enum, data); if (!report) return 0; diff --git a/drivers/hid/hid-goodix-spi.c b/drivers/hid/hid-goodix-spi.c index 80c0288a3a38b..288cb827e9d6d 100644 --- a/drivers/hid/hid-goodix-spi.c +++ b/drivers/hid/hid-goodix-spi.c @@ -520,6 +520,9 @@ static int goodix_hid_set_raw_report(struct hid_device *hid, memcpy(tmp_buf + tx_len, args, args_len); tx_len += args_len; + if (tx_len + len > sizeof(tmp_buf)) + return -EINVAL; + memcpy(tmp_buf + tx_len, buf, len); tx_len += len; diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index d73cfa2e73d33..c7b8c4ff7a338 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -519,6 +519,13 @@ static struct hid_battery *hidinput_find_battery(struct hid_device *dev, return NULL; } +static void hidinput_cleanup_battery(void *res) +{ + struct hid_battery *bat = res; + + list_del(&bat->list); +} + static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type, struct hid_field *field, bool is_percentage) { @@ -610,6 +617,12 @@ static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type, power_supply_powers(bat->ps, &dev->dev); list_add_tail(&bat->list, &dev->batteries); + + error = devm_add_action_or_reset(&dev->dev, + hidinput_cleanup_battery, bat); + if (error) + return error; + return 0; err_free_name: diff --git a/drivers/hid/hid-lenovo-go.c b/drivers/hid/hid-lenovo-go.c index e0c9d5ec9451b..318b1152ff8ba 100644 --- a/drivers/hid/hid-lenovo-go.c +++ b/drivers/hid/hid-lenovo-go.c @@ -2405,6 +2405,15 @@ static int hid_go_cfg_probe(struct hid_device *hdev, static void hid_go_cfg_remove(struct hid_device *hdev) { + /* + * cfg_setup is scheduled from hid_go_cfg_probe() with a 2 ms delay + * and dereferences drvdata.hdev. Drain it here before tearing + * down so the workqueue cannot run after hid_destroy_device()'s + * put_device() has released the underlying hdev and dereference + * a stale drvdata.hdev pointer. + */ + cancel_delayed_work_sync(&drvdata.go_cfg_setup); + guard(mutex)(&drvdata.cfg_mutex); sysfs_remove_groups(&hdev->dev.kobj, top_level_attr_groups); hid_hw_close(hdev); diff --git a/drivers/hid/hid-letsketch.c b/drivers/hid/hid-letsketch.c index 11e21f9887232..b52e93a91ae54 100644 --- a/drivers/hid/hid-letsketch.c +++ b/drivers/hid/hid-letsketch.c @@ -296,13 +296,42 @@ static int letsketch_probe(struct hid_device *hdev, const struct hid_device_id * ret = letsketch_setup_input_tablet(data); if (ret) - return ret; + goto err_shutdown_timer; ret = letsketch_setup_input_tablet_pad(data); if (ret) - return ret; + goto err_shutdown_timer; + + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + if (ret) + goto err_shutdown_timer; - return hid_hw_start(hdev, HID_CONNECT_HIDRAW); + return 0; + +err_shutdown_timer: + /* + * Drain any pending callback and permanently disable the timer + * before devm releases data: if hid_hw_start() enabled I/O on an + * always-poll-quirk device and then failed, raw_event may have + * armed the timer already. + */ + timer_shutdown_sync(&data->inrange_timer); + return ret; +} + +static void letsketch_remove(struct hid_device *hdev) +{ + struct letsketch_data *data = hid_get_drvdata(hdev); + + /* + * hid_hw_stop() synchronously kills the URBs that deliver + * raw_event(), so once it returns no path can re-arm + * inrange_timer. timer_shutdown_sync() then drains any + * in-flight callback and permanently disables further + * mod_timer() calls before devm releases data. + */ + hid_hw_stop(hdev); + timer_shutdown_sync(&data->inrange_timer); } static const struct hid_device_id letsketch_devices[] = { @@ -315,6 +344,7 @@ static struct hid_driver letsketch_driver = { .name = "letsketch", .id_table = letsketch_devices, .probe = letsketch_probe, + .remove = letsketch_remove, .raw_event = letsketch_raw_event, }; module_hid_driver(letsketch_driver); diff --git a/drivers/hid/hid-lg-g15.c b/drivers/hid/hid-lg-g15.c index 1a88bc44ada44..02ef3e2094b4c 100644 --- a/drivers/hid/hid-lg-g15.c +++ b/drivers/hid/hid-lg-g15.c @@ -1374,11 +1374,27 @@ static const struct hid_device_id lg_g15_devices[] = { }; MODULE_DEVICE_TABLE(hid, lg_g15_devices); +static void lg_g15_remove(struct hid_device *hdev) +{ + struct lg_g15_data *g15 = hid_get_drvdata(hdev); + + /* + * g15->work is only initialized for the models that schedule it + * (G15, G15 v2, G510). The G13 and Z-10 leave it zeroed, so only + * cancel it when it was set up. + */ + if (g15 && g15->work.func) + cancel_work_sync(&g15->work); + + hid_hw_stop(hdev); +} + static struct hid_driver lg_g15_driver = { .name = "lg-g15", .id_table = lg_g15_devices, .raw_event = lg_g15_raw_event, .probe = lg_g15_probe, + .remove = lg_g15_remove, }; module_hid_driver(lg_g15_driver); diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c index 381e4dc5aba7a..9c574ab8b60b3 100644 --- a/drivers/hid/hid-logitech-dj.c +++ b/drivers/hid/hid-logitech-dj.c @@ -1907,8 +1907,13 @@ static int logi_dj_probe(struct hid_device *hdev, output_report_enum = &hdev->report_enum[HID_OUTPUT_REPORT]; rep = output_report_enum->report_id_hash[REPORT_ID_DJ_SHORT]; - if (rep && (rep->maxfield < 1 || - rep->field[0]->report_count != DJREPORT_SHORT_LENGTH - 1)) { + if (rep && rep->maxfield < 1) { + hid_err(hdev, "Expected size of DJ short report is %d, but got 0", + DJREPORT_SHORT_LENGTH - 1); + return -EINVAL; + } + + if (rep && rep->field[0]->report_count != DJREPORT_SHORT_LENGTH - 1) { hid_err(hdev, "Expected size of DJ short report is %d, but got %d", DJREPORT_SHORT_LENGTH - 1, rep->field[0]->report_count); return -EINVAL; diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index ccbf28869a968..1990ba5b26ea6 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -164,7 +164,6 @@ struct hidpp_battery { /** * struct hidpp_scroll_counter - Utility class for processing high-resolution * scroll events. - * @dev: the input device for which events should be reported. * @wheel_multiplier: the scalar multiplier to be applied to each wheel event * @remainder: counts the number of high-resolution units moved since the last * low-resolution event (REL_WHEEL or REL_HWHEEL) was sent. Should diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index eeab0b6e32ccc..b085bed874bf9 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -31,6 +31,7 @@ * [1] https://gitlab.freedesktop.org/libevdev/hid-tools */ +#include #include #include #include @@ -97,8 +98,7 @@ enum report_mode { TOUCHPAD_REPORT_ALL = TOUCHPAD_REPORT_BUTTONS | TOUCHPAD_REPORT_CONTACTS, }; -#define MT_IO_SLOTS_MASK GENMASK(7, 0) /* reserve first 8 bits for slot tracking */ -#define MT_IO_FLAGS_RUNNING 32 +#define MT_IO_FLAGS_RUNNING 0 static const bool mtrue = true; /* default for true */ static const bool mfalse; /* default for false */ @@ -174,10 +174,9 @@ struct mt_device { struct timer_list release_timer; /* to release sticky fingers */ struct hid_haptic_device *haptic; /* haptic related configuration */ struct hid_device *hdev; /* hid_device we're attached to */ - unsigned long mt_io_flags; /* mt flags (MT_IO_FLAGS_RUNNING) - * first 8 bits are reserved for keeping the slot - * states, this is fine because we only support up - * to 250 slots (MT_MAX_MAXCONTACT) + unsigned long mt_io_flags; /* mt flags (MT_IO_FLAGS_RUNNING) */ + unsigned long *active_slots; /* bitmap of slots with an active + * contact, sized for maxcontacts */ __u8 inputmode_value; /* InputMode HID feature value */ __u8 maxcontacts; @@ -1033,7 +1032,7 @@ static void mt_release_pending_palms(struct mt_device *td, for_each_set_bit(slotnum, app->pending_palm_slots, td->maxcontacts) { clear_bit(slotnum, app->pending_palm_slots); - clear_bit(slotnum, &td->mt_io_flags); + clear_bit(slotnum, td->active_slots); input_mt_slot(input, slotnum); input_mt_report_slot_inactive(input); @@ -1244,9 +1243,9 @@ static int mt_process_slot(struct mt_device *td, struct input_dev *input, input_event(input, EV_ABS, ABS_MT_TOUCH_MAJOR, major); input_event(input, EV_ABS, ABS_MT_TOUCH_MINOR, minor); - set_bit(slotnum, &td->mt_io_flags); + set_bit(slotnum, td->active_slots); } else { - clear_bit(slotnum, &td->mt_io_flags); + clear_bit(slotnum, td->active_slots); } return 0; @@ -1381,7 +1380,7 @@ static void mt_touch_report(struct hid_device *hid, * defect. */ if (app->quirks & MT_QUIRK_STICKY_FINGERS) { - if (td->mt_io_flags & MT_IO_SLOTS_MASK) + if (!bitmap_empty(td->active_slots, td->maxcontacts)) mod_timer(&td->release_timer, jiffies + msecs_to_jiffies(100)); else @@ -1440,6 +1439,15 @@ static int mt_touch_input_configured(struct hid_device *hdev, if (td->is_pressurepad) __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); + if (!td->active_slots) { + td->active_slots = devm_kcalloc(&td->hdev->dev, + BITS_TO_LONGS(td->maxcontacts), + sizeof(long), + GFP_KERNEL); + if (!td->active_slots) + return -ENOMEM; + } + app->pending_palm_slots = devm_kcalloc(&hi->input->dev, BITS_TO_LONGS(td->maxcontacts), sizeof(long), @@ -1917,7 +1925,7 @@ static void mt_release_contacts(struct hid_device *hid) for (i = 0; i < mt->num_slots; i++) { input_mt_slot(input_dev, i); input_mt_report_slot_inactive(input_dev); - clear_bit(i, &td->mt_io_flags); + clear_bit(i, td->active_slots); } input_mt_sync_frame(input_dev); input_sync(input_dev); @@ -1940,7 +1948,7 @@ static void mt_expired_timeout(struct timer_list *t) */ if (test_and_set_bit_lock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags)) return; - if (td->mt_io_flags & MT_IO_SLOTS_MASK) + if (!bitmap_empty(td->active_slots, td->maxcontacts)) mt_release_contacts(hdev); clear_bit_unlock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags); } diff --git a/drivers/hid/hid-picolcd_core.c b/drivers/hid/hid-picolcd_core.c index 2cc01e1bc1a84..d73e97c8b853e 100644 --- a/drivers/hid/hid-picolcd_core.c +++ b/drivers/hid/hid-picolcd_core.c @@ -72,7 +72,8 @@ struct picolcd_pending *picolcd_send_and_wait(struct hid_device *hdev, struct picolcd_pending *work; struct hid_report *report = picolcd_out_report(report_id, hdev); unsigned long flags; - int i, j, k; + int i, j; + unsigned int k; if (!report || !data) return NULL; diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c index 90666ff629def..34f710c465b80 100644 --- a/drivers/hid/hid-sensor-hub.c +++ b/drivers/hid/hid-sensor-hub.c @@ -286,6 +286,54 @@ int sensor_hub_get_feature(struct hid_sensor_hub_device *hsdev, u32 report_id, } EXPORT_SYMBOL_GPL(sensor_hub_get_feature); +int sensor_hub_input_attr_read_values(struct hid_sensor_hub_device *hsdev, + u32 usage_id, u32 attr_usage_id, + u32 report_id, + enum sensor_hub_read_flags flag, + u32 buffer_size, u8 *buffer) +{ + struct sensor_hub_data *data = hid_get_drvdata(hsdev->hdev); + struct hid_report *report; + unsigned long flags; + long cycles; + int ret; + + report = sensor_hub_report(report_id, hsdev->hdev, HID_INPUT_REPORT); + if (!report) + return -EINVAL; + + mutex_lock(hsdev->mutex_ptr); + if (flag == SENSOR_HUB_SYNC) { + memset(&hsdev->pending, 0, sizeof(hsdev->pending)); + init_completion(&hsdev->pending.ready); + hsdev->pending.usage_id = usage_id; + hsdev->pending.attr_usage_id = attr_usage_id; + hsdev->pending.max_raw_size = buffer_size; + hsdev->pending.raw_data = buffer; + + spin_lock_irqsave(&data->lock, flags); + hsdev->pending.status = true; + spin_unlock_irqrestore(&data->lock, flags); + } + mutex_lock(&data->mutex); + hid_hw_request(hsdev->hdev, report, HID_REQ_GET_REPORT); + mutex_unlock(&data->mutex); + ret = 0; + if (flag == SENSOR_HUB_SYNC) { + cycles = wait_for_completion_interruptible_timeout(&hsdev->pending.ready, + HZ * 5); + if (cycles == 0) + ret = -ETIMEDOUT; + else if (cycles < 0) + ret = cycles; + + hsdev->pending.status = false; + } + mutex_unlock(hsdev->mutex_ptr); + + return ret; +} +EXPORT_SYMBOL_GPL(sensor_hub_input_attr_read_values); int sensor_hub_input_attr_get_raw_value(struct hid_sensor_hub_device *hsdev, u32 usage_id, @@ -478,6 +526,8 @@ static int sensor_hub_raw_event(struct hid_device *hdev, struct hid_collection *collection = NULL; void *priv = NULL; struct hid_sensor_hub_device *hsdev = NULL; + u32 copy_size; + u32 avail; hid_dbg(hdev, "sensor_hub_raw_event report id:0x%x size:%d type:%d\n", report->id, size, report->type); @@ -518,12 +568,27 @@ static int sensor_hub_raw_event(struct hid_device *hdev, hsdev->pending.attr_usage_id == report->field[i]->logical)) { hid_dbg(hdev, "data was pending ...\n"); - hsdev->pending.raw_data = kmemdup(ptr, sz, GFP_ATOMIC); - if (hsdev->pending.raw_data) - hsdev->pending.raw_size = sz; - else - hsdev->pending.raw_size = 0; - complete(&hsdev->pending.ready); + if (hsdev->pending.max_raw_size) { + if (hsdev->pending.index < hsdev->pending.max_raw_size) { + avail = hsdev->pending.max_raw_size - hsdev->pending.index; + copy_size = clamp(sz, 0U, avail); + + memcpy(hsdev->pending.raw_data + hsdev->pending.index, + ptr, copy_size); + hsdev->pending.index += copy_size; + if (hsdev->pending.index >= hsdev->pending.max_raw_size) { + hsdev->pending.raw_size = hsdev->pending.index; + complete(&hsdev->pending.ready); + } + } + } else { + hsdev->pending.raw_data = kmemdup(ptr, sz, GFP_ATOMIC); + if (hsdev->pending.raw_data) + hsdev->pending.raw_size = sz; + else + hsdev->pending.raw_size = 0; + complete(&hsdev->pending.ready); + } } if (callback->capture_sample) { if (report->field[i]->logical) diff --git a/drivers/hid/hid-wiimote-modules.c b/drivers/hid/hid-wiimote-modules.c index dbccdfa639167..dccb78bb3afd6 100644 --- a/drivers/hid/hid-wiimote-modules.c +++ b/drivers/hid/hid-wiimote-modules.c @@ -2403,7 +2403,7 @@ static const struct wiimod_ops wiimod_guitar = { .in_ext = wiimod_guitar_in_ext, }; -/* +/* * Turntable * DJ Hero came with a Turntable Controller that was plugged in * as an extension. @@ -2439,15 +2439,15 @@ static const __u16 wiimod_turntable_map[] = { static void wiimod_turntable_in_ext(struct wiimote_data *wdata, const __u8 *ext) { __u8 be, cs, sx, sy, ed, rtt, rbg, rbr, rbb, ltt, lbg, lbr, lbb, bp, bm; - /* + /* * Byte | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | *------+------+-----+-----+-----+-----+------+------+--------+ - * 0 | RTT<4:3> | SX <5:0> | - * 1 | RTT<2:1> | SY <5:0> | + * 0 | RTT<4:3> | SX <5:0> | + * 1 | RTT<2:1> | SY <5:0> | *------+------+-----+-----+-----+-----+------+------+--------+ * 2 |RTT<0>| ED<4:3> | CS<3:0> | RTT<5> | *------+------+-----+-----+-----+-----+------+------+--------+ - * 3 | ED<2:0> | LTT<4:0> | + * 3 | ED<2:0> | LTT<4:0> | *------+------+-----+-----+-----+-----+------+------+--------+ * 4 | 0 | 0 | LBR | B- | 0 | B+ | RBR | LTT<5> | *------+------+-----+-----+-----+-----+------+------+--------+ @@ -2458,20 +2458,20 @@ static void wiimod_turntable_in_ext(struct wiimote_data *wdata, const __u8 *ext) * With Motion+ enabled, it will look like this: * Byte | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | *------+------+-----+-----+-----+-----+------+------+--------+ - * 1 | RTT<4:3> | SX <5:1> | 0 | - * 2 | RTT<2:1> | SY <5:1> | 0 | + * 1 | RTT<4:3> | SX <5:1> | 0 | + * 2 | RTT<2:1> | SY <5:1> | 0 | *------+------+-----+-----+-----+-----+------+------+--------+ * 3 |RTT<0>| ED<4:3> | CS<3:0> | RTT<5> | *------+------+-----+-----+-----+-----+------+------+--------+ - * 4 | ED<2:0> | LTT<4:0> | + * 4 | ED<2:0> | LTT<4:0> | *------+------+-----+-----+-----+-----+------+------+--------+ * 5 | 0 | 0 | LBR | B- | 0 | B+ | RBR | XXXX | *------+------+-----+-----+-----+-----+------+------+--------+ * 6 | LBB | 0 | RBG | BE | LBG | RBB | XXXX | XXXX | *------+------+-----+-----+-----+-----+------+------+--------+ */ - - be = !(ext[5] & 0x10); + + be = !(ext[5] & 0x10); cs = ((ext[2] & 0x1e)); sx = ext[0] & 0x3f; sy = ext[1] & 0x3f; @@ -2499,32 +2499,32 @@ static void wiimod_turntable_in_ext(struct wiimote_data *wdata, const __u8 *ext) input_report_abs(wdata->extension.input, ABS_HAT1X, ltt); input_report_abs(wdata->extension.input, ABS_HAT2X, cs); input_report_abs(wdata->extension.input, ABS_HAT3X, ed); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_G_RIGHT], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_G_RIGHT], rbg); input_report_key(wdata->extension.input, wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_R_RIGHT], rbr); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_B_RIGHT], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_B_RIGHT], rbb); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_G_LEFT], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_G_LEFT], lbg); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_R_LEFT], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_R_LEFT], lbr); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_B_LEFT], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_B_LEFT], lbb); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_EUPHORIA], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_EUPHORIA], be); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_PLUS], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_PLUS], bp); - input_report_key(wdata->extension.input, - wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_MINUS], + input_report_key(wdata->extension.input, + wiimod_turntable_map[WIIMOD_TURNTABLE_KEY_MINUS], bm); input_sync(wdata->extension.input); @@ -2557,7 +2557,7 @@ static void wiimod_turntable_close(struct input_dev *dev) static int wiimod_turntable_probe(const struct wiimod_ops *ops, struct wiimote_data *wdata) { - int ret, i; + int ret, i; wdata->extension.input = input_allocate_device(); if (!wdata->extension.input) @@ -2594,9 +2594,9 @@ static int wiimod_turntable_probe(const struct wiimod_ops *ops, input_set_abs_params(wdata->extension.input, ABS_HAT1X, -8, 8, 0, 0); input_set_abs_params(wdata->extension.input, - ABS_HAT2X, 0, 31, 1, 1); + ABS_HAT2X, 0, 31, 1, 1); input_set_abs_params(wdata->extension.input, - ABS_HAT3X, 0, 7, 0, 0); + ABS_HAT3X, 0, 7, 0, 0); ret = input_register_device(wdata->extension.input); if (ret) goto err_free; diff --git a/drivers/hid/uhid.c b/drivers/hid/uhid.c index 524b53a3c87b3..37b60c3aaf66a 100644 --- a/drivers/hid/uhid.c +++ b/drivers/hid/uhid.c @@ -595,8 +595,8 @@ static int uhid_dev_input(struct uhid_device *uhid, struct uhid_event *ev) if (!READ_ONCE(uhid->running)) return -EINVAL; - hid_input_report(uhid->hid, HID_INPUT_REPORT, ev->u.input.data, - min_t(size_t, ev->u.input.size, UHID_DATA_MAX), 0); + hid_safe_input_report(uhid->hid, HID_INPUT_REPORT, ev->u.input.data, UHID_DATA_MAX, + min_t(size_t, ev->u.input.size, UHID_DATA_MAX), 0); return 0; } @@ -606,8 +606,8 @@ static int uhid_dev_input2(struct uhid_device *uhid, struct uhid_event *ev) if (!READ_ONCE(uhid->running)) return -EINVAL; - hid_input_report(uhid->hid, HID_INPUT_REPORT, ev->u.input2.data, - min_t(size_t, ev->u.input2.size, UHID_DATA_MAX), 0); + hid_safe_input_report(uhid->hid, HID_INPUT_REPORT, ev->u.input2.data, UHID_DATA_MAX, + min_t(size_t, ev->u.input2.size, UHID_DATA_MAX), 0); return 0; } diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c index c45f182d04480..5f4395f7c6458 100644 --- a/drivers/hid/usbhid/hid-pidff.c +++ b/drivers/hid/usbhid/hid-pidff.c @@ -522,7 +522,7 @@ static void pidff_set_effect_report(struct pidff_device *pidff, pidff->set_effect[PID_EFFECT_BLOCK_INDEX].value[0] = pidff->block_load[PID_EFFECT_BLOCK_INDEX].value[0]; pidff->set_effect_type->value[0] = - pidff->create_new_effect_type->value[0]; + pidff_get_effect_type_id(pidff, effect); pidff_set_duration(&pidff->set_effect[PID_DURATION], effect->replay.length); diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c index 2220168bf1164..24b8b33a07f60 100644 --- a/drivers/hid/wacom_sys.c +++ b/drivers/hid/wacom_sys.c @@ -54,7 +54,7 @@ static void wacom_wac_queue_insert(struct hid_device *hdev, { bool warned = false; - while (kfifo_avail(fifo) < size) { + while (kfifo_avail(fifo) < size && !kfifo_is_empty(fifo)) { if (!warned) hid_warn(hdev, "%s: kfifo has filled, starting to drop events\n", __func__); warned = true; @@ -62,7 +62,9 @@ static void wacom_wac_queue_insert(struct hid_device *hdev, kfifo_skip(fifo); } - kfifo_in(fifo, raw_data, size); + if (!kfifo_in(fifo, raw_data, size)) + hid_warn_ratelimited(hdev, "%s: report is too large (%d)\n", + __func__, size); } static void wacom_wac_queue_flush(struct hid_device *hdev, @@ -74,7 +76,7 @@ static void wacom_wac_queue_flush(struct hid_device *hdev, unsigned int count; int err; - buf = kzalloc(size, GFP_KERNEL); + buf = kzalloc(size, GFP_ATOMIC); if (!buf) { kfifo_skip(fifo); continue; @@ -2462,16 +2464,16 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless) error = wacom_register_inputs(wacom); if (error) - goto fail; + goto fail_hw_stop; if (wacom->wacom_wac.features.device_type & WACOM_DEVICETYPE_PAD) { error = wacom_initialize_leds(wacom); if (error) - goto fail; + goto fail_hw_stop; error = wacom_initialize_remotes(wacom); if (error) - goto fail; + goto fail_hw_stop; } if (!wireless) { @@ -2485,14 +2487,14 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless) cancel_delayed_work_sync(&wacom->init_work); _wacom_query_tablet_data(wacom); error = -ENODEV; - goto fail_quirks; + goto fail_hw_stop; } if (features->device_type & WACOM_DEVICETYPE_WL_MONITOR) { error = hid_hw_open(hdev); if (error) { hid_err(hdev, "hw open failed\n"); - goto fail_quirks; + goto fail_hw_stop; } } @@ -2501,7 +2503,7 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless) return 0; -fail_quirks: +fail_hw_stop: hid_hw_stop(hdev); fail: wacom_release_resources(wacom); diff --git a/drivers/hv/mshv_synic.c b/drivers/hv/mshv_synic.c index 88170ce6b83f0..7c168e5a740dd 100644 --- a/drivers/hv/mshv_synic.c +++ b/drivers/hv/mshv_synic.c @@ -384,6 +384,11 @@ mshv_intercept_isr(struct hv_message *msg) */ vp_index = ((struct hv_opaque_intercept_message *)msg->u.payload)->vp_index; + /* This shouldn't happen, but just in case. */ + if (unlikely(vp_index >= MSHV_MAX_VPS)) { + pr_debug("VP index %u out of bounds\n", vp_index); + goto unlock_out; + } vp = partition->pt_vp_array[vp_index]; if (unlikely(!vp)) { pr_debug("failed to find VP %u\n", vp_index); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index b80a35c778ab4..23206640c6139 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -548,34 +548,6 @@ static ssize_t device_show(struct device *dev, } static DEVICE_ATTR_RO(device); -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct hv_device *hv_dev = device_to_hv_device(dev); - int ret; - - ret = driver_set_override(dev, &hv_dev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} - -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct hv_device *hv_dev = device_to_hv_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", hv_dev->driver_override); - device_unlock(dev); - - return len; -} -static DEVICE_ATTR_RW(driver_override); - /* Set up per device attributes in /sys/bus/vmbus/devices/ */ static struct attribute *vmbus_dev_attrs[] = { &dev_attr_id.attr, @@ -606,7 +578,6 @@ static struct attribute *vmbus_dev_attrs[] = { &dev_attr_channel_vp_mapping.attr, &dev_attr_vendor.attr, &dev_attr_device.attr, - &dev_attr_driver_override.attr, NULL, }; @@ -718,9 +689,11 @@ static const struct hv_vmbus_device_id *hv_vmbus_get_id(const struct hv_driver * { const guid_t *guid = &dev->dev_type; const struct hv_vmbus_device_id *id; + int ret; - /* When driver_override is set, only bind to the matching driver */ - if (dev->driver_override && strcmp(dev->driver_override, drv->name)) + /* If a driver override is set, only bind to the matching driver */ + ret = device_match_driver_override(&dev->device, &drv->driver); + if (ret == 0) return NULL; /* Look at the dynamic ids first, before the static ones */ @@ -728,8 +701,11 @@ static const struct hv_vmbus_device_id *hv_vmbus_get_id(const struct hv_driver * if (!id) id = hv_vmbus_dev_match(drv->id_table, guid); - /* driver_override will always match, send a dummy id */ - if (!id && dev->driver_override) + /* + * If there's a matching driver override, this function should succeed, + * thus return a dummy device ID if no matching ID is found. + */ + if (!id && ret > 0) id = &vmbus_device_null; return id; @@ -1031,6 +1007,7 @@ static const struct dev_pm_ops vmbus_pm = { /* The one and only one */ static const struct bus_type hv_bus = { .name = "vmbus", + .driver_override = true, .match = vmbus_match, .shutdown = vmbus_shutdown, .remove = vmbus_remove, diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 14e4cea48acc4..b1f121b84bf68 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1076,6 +1076,7 @@ config SENSORS_LTC2992 tristate "Linear Technology LTC2992" depends on I2C depends on GPIOLIB + select REGMAP_I2C help If you say yes here you get support for Linear Technology LTC2992 I2C System Monitor. The LTC2992 measures current, voltage, and @@ -1214,6 +1215,7 @@ config SENSORS_MAX16065 config SENSORS_MAX1619 tristate "Maxim MAX1619 sensor chip" depends on I2C + select REGMAP help If you say yes here you get support for MAX1619 sensor chip. @@ -1332,6 +1334,7 @@ config SENSORS_MAX6650 config SENSORS_MAX6697 tristate "Maxim MAX6697 and compatibles" depends on I2C + select REGMAP_I2C help If you say yes here you get support for MAX6581, MAX6602, MAX6622, MAX6636, MAX6689, MAX6693, MAX6694, MAX6697, MAX6698, and MAX6699 diff --git a/drivers/hwmon/aspeed-g6-pwm-tach.c b/drivers/hwmon/aspeed-g6-pwm-tach.c index 4f6e6d440dd40..5d611a8e52690 100644 --- a/drivers/hwmon/aspeed-g6-pwm-tach.c +++ b/drivers/hwmon/aspeed-g6-pwm-tach.c @@ -293,7 +293,10 @@ static int aspeed_tach_val_to_rpm(struct aspeed_pwm_tach_data *priv, u32 tach_va priv->clk_rate, tach_val, tach_div); rpm = (u64)priv->clk_rate * 60; - do_div(rpm, tach_div); + if (tach_div) + do_div(rpm, tach_div); + else + rpm = 0; return (int)rpm; } diff --git a/drivers/hwmon/asus_atk0110.c b/drivers/hwmon/asus_atk0110.c index 109318b0434d9..92afb64c09df8 100644 --- a/drivers/hwmon/asus_atk0110.c +++ b/drivers/hwmon/asus_atk0110.c @@ -1037,6 +1037,9 @@ static int atk_ec_present(struct atk_data *data) if (obj->type != ACPI_TYPE_PACKAGE) continue; + if (!obj->package.count) + continue; + id = &obj->package.elements[0]; if (id->type != ACPI_TYPE_INTEGER) continue; diff --git a/drivers/hwmon/gpd-fan.c b/drivers/hwmon/gpd-fan.c index 80de5f20781eb..4615f98771ddf 100644 --- a/drivers/hwmon/gpd-fan.c +++ b/drivers/hwmon/gpd-fan.c @@ -40,12 +40,11 @@ enum FAN_PWM_ENABLE { AUTOMATIC = 2, }; -static struct { +struct gpd_fan_data { enum FAN_PWM_ENABLE pwm_enable; u8 pwm_value; - const struct gpd_fan_drvdata *drvdata; -} gpd_driver_priv; +}; struct gpd_fan_drvdata { const char *board_name; // Board name for module param comparison @@ -249,10 +248,10 @@ static const struct gpd_fan_drvdata *gpd_module_drvdata[] = { }; // Helper functions to handle EC read/write -static void gpd_ecram_read(u16 offset, u8 *val) +static void gpd_ecram_read(const struct gpd_fan_drvdata *drvdata, u16 offset, u8 *val) { - u16 addr_port = gpd_driver_priv.drvdata->addr_port; - u16 data_port = gpd_driver_priv.drvdata->data_port; + u16 addr_port = drvdata->addr_port; + u16 data_port = drvdata->data_port; outb(0x2E, addr_port); outb(0x11, data_port); @@ -270,10 +269,10 @@ static void gpd_ecram_read(u16 offset, u8 *val) *val = inb(data_port); } -static void gpd_ecram_write(u16 offset, u8 value) +static void gpd_ecram_write(const struct gpd_fan_drvdata *drvdata, u16 offset, u8 value) { - u16 addr_port = gpd_driver_priv.drvdata->addr_port; - u16 data_port = gpd_driver_priv.drvdata->data_port; + u16 addr_port = drvdata->addr_port; + u16 data_port = drvdata->data_port; outb(0x2E, addr_port); outb(0x11, data_port); @@ -291,198 +290,198 @@ static void gpd_ecram_write(u16 offset, u8 value) outb(value, data_port); } -static int gpd_generic_read_rpm(void) +static int gpd_generic_read_rpm(struct gpd_fan_data *data) { - const struct gpd_fan_drvdata *const drvdata = gpd_driver_priv.drvdata; + const struct gpd_fan_drvdata *drvdata = data->drvdata; u8 high, low; - gpd_ecram_read(drvdata->rpm_read, &high); - gpd_ecram_read(drvdata->rpm_read + 1, &low); + gpd_ecram_read(drvdata, drvdata->rpm_read, &high); + gpd_ecram_read(drvdata, drvdata->rpm_read + 1, &low); return (u16)high << 8 | low; } -static int gpd_wm2_read_rpm(void) +static int gpd_wm2_read_rpm(struct gpd_fan_data *data) { + const struct gpd_fan_drvdata *drvdata = data->drvdata; + for (u16 pwm_ctr_offset = GPD_PWM_CTR_OFFSET; pwm_ctr_offset <= GPD_PWM_CTR_OFFSET + 2; pwm_ctr_offset++) { u8 PWMCTR; - gpd_ecram_read(pwm_ctr_offset, &PWMCTR); + gpd_ecram_read(drvdata, pwm_ctr_offset, &PWMCTR); if (PWMCTR != 0xB8) - gpd_ecram_write(pwm_ctr_offset, 0xB8); + gpd_ecram_write(drvdata, pwm_ctr_offset, 0xB8); } - return gpd_generic_read_rpm(); + return gpd_generic_read_rpm(data); } // Read value for fan1_input -static int gpd_read_rpm(void) +static int gpd_read_rpm(struct gpd_fan_data *data) { - switch (gpd_driver_priv.drvdata->board) { + switch (data->drvdata->board) { case win4_6800u: case win_mini: case duo: case mpc2: - return gpd_generic_read_rpm(); + return gpd_generic_read_rpm(data); case win_max_2: - return gpd_wm2_read_rpm(); + return gpd_wm2_read_rpm(data); } return 0; } -static int gpd_wm2_read_pwm(void) +static int gpd_wm2_read_pwm(struct gpd_fan_data *data) { - const struct gpd_fan_drvdata *const drvdata = gpd_driver_priv.drvdata; + const struct gpd_fan_drvdata *drvdata = data->drvdata; u8 var; - gpd_ecram_read(drvdata->pwm_write, &var); + gpd_ecram_read(drvdata, drvdata->pwm_write, &var); // Match gpd_generic_write_pwm(u8) below return DIV_ROUND_CLOSEST((var - 1) * 255, (drvdata->pwm_max - 1)); } // Read value for pwm1 -static int gpd_read_pwm(void) +static int gpd_read_pwm(struct gpd_fan_data *data) { - switch (gpd_driver_priv.drvdata->board) { + switch (data->drvdata->board) { case win_mini: case duo: case win4_6800u: case mpc2: - switch (gpd_driver_priv.pwm_enable) { + switch (data->pwm_enable) { case DISABLE: return 255; case MANUAL: - return gpd_driver_priv.pwm_value; + return data->pwm_value; case AUTOMATIC: return -EOPNOTSUPP; } break; case win_max_2: - return gpd_wm2_read_pwm(); + return gpd_wm2_read_pwm(data); } return 0; } // PWM value's range in EC is 1 - pwm_max, cast 0 - 255 to it. -static inline u8 gpd_cast_pwm_range(u8 val) +static inline u8 gpd_cast_pwm_range(const struct gpd_fan_drvdata *drvdata, u8 val) { - const struct gpd_fan_drvdata *const drvdata = gpd_driver_priv.drvdata; - return DIV_ROUND_CLOSEST(val * (drvdata->pwm_max - 1), 255) + 1; } -static void gpd_generic_write_pwm(u8 val) +static void gpd_generic_write_pwm(struct gpd_fan_data *data, u8 val) { - const struct gpd_fan_drvdata *const drvdata = gpd_driver_priv.drvdata; + const struct gpd_fan_drvdata *drvdata = data->drvdata; u8 pwm_reg; - pwm_reg = gpd_cast_pwm_range(val); - gpd_ecram_write(drvdata->pwm_write, pwm_reg); + pwm_reg = gpd_cast_pwm_range(drvdata, val); + gpd_ecram_write(drvdata, drvdata->pwm_write, pwm_reg); } -static void gpd_duo_write_pwm(u8 val) +static void gpd_duo_write_pwm(struct gpd_fan_data *data, u8 val) { - const struct gpd_fan_drvdata *const drvdata = gpd_driver_priv.drvdata; + const struct gpd_fan_drvdata *drvdata = data->drvdata; u8 pwm_reg; - pwm_reg = gpd_cast_pwm_range(val); - gpd_ecram_write(drvdata->pwm_write, pwm_reg); - gpd_ecram_write(drvdata->pwm_write + 1, pwm_reg); + pwm_reg = gpd_cast_pwm_range(drvdata, val); + gpd_ecram_write(drvdata, drvdata->pwm_write, pwm_reg); + gpd_ecram_write(drvdata, drvdata->pwm_write + 1, pwm_reg); } // Write value for pwm1 -static int gpd_write_pwm(u8 val) +static int gpd_write_pwm(struct gpd_fan_data *data, u8 val) { - if (gpd_driver_priv.pwm_enable != MANUAL) + if (data->pwm_enable != MANUAL) return -EPERM; - switch (gpd_driver_priv.drvdata->board) { + switch (data->drvdata->board) { case duo: - gpd_duo_write_pwm(val); + gpd_duo_write_pwm(data, val); break; case win_mini: case win4_6800u: case win_max_2: case mpc2: - gpd_generic_write_pwm(val); + gpd_generic_write_pwm(data, val); break; } return 0; } -static void gpd_win_mini_set_pwm_enable(enum FAN_PWM_ENABLE pwm_enable) +static void gpd_win_mini_set_pwm_enable(struct gpd_fan_data *data, enum FAN_PWM_ENABLE pwm_enable) { switch (pwm_enable) { case DISABLE: - gpd_generic_write_pwm(255); + gpd_generic_write_pwm(data, 255); break; case MANUAL: - gpd_generic_write_pwm(gpd_driver_priv.pwm_value); + gpd_generic_write_pwm(data, data->pwm_value); break; case AUTOMATIC: - gpd_ecram_write(gpd_driver_priv.drvdata->pwm_write, 0); + gpd_ecram_write(data->drvdata, data->drvdata->pwm_write, 0); break; } } -static void gpd_duo_set_pwm_enable(enum FAN_PWM_ENABLE pwm_enable) +static void gpd_duo_set_pwm_enable(struct gpd_fan_data *data, enum FAN_PWM_ENABLE pwm_enable) { switch (pwm_enable) { case DISABLE: - gpd_duo_write_pwm(255); + gpd_duo_write_pwm(data, 255); break; case MANUAL: - gpd_duo_write_pwm(gpd_driver_priv.pwm_value); + gpd_duo_write_pwm(data, data->pwm_value); break; case AUTOMATIC: - gpd_ecram_write(gpd_driver_priv.drvdata->pwm_write, 0); + gpd_ecram_write(data->drvdata, data->drvdata->pwm_write, 0); break; } } -static void gpd_wm2_set_pwm_enable(enum FAN_PWM_ENABLE enable) +static void gpd_wm2_set_pwm_enable(struct gpd_fan_data *data, enum FAN_PWM_ENABLE enable) { - const struct gpd_fan_drvdata *const drvdata = gpd_driver_priv.drvdata; + const struct gpd_fan_drvdata *drvdata = data->drvdata; switch (enable) { case DISABLE: - gpd_generic_write_pwm(255); - gpd_ecram_write(drvdata->manual_control_enable, 1); + gpd_generic_write_pwm(data, 255); + gpd_ecram_write(drvdata, drvdata->manual_control_enable, 1); break; case MANUAL: - gpd_generic_write_pwm(gpd_driver_priv.pwm_value); - gpd_ecram_write(drvdata->manual_control_enable, 1); + gpd_generic_write_pwm(data, data->pwm_value); + gpd_ecram_write(drvdata, drvdata->manual_control_enable, 1); break; case AUTOMATIC: - gpd_ecram_write(drvdata->manual_control_enable, 0); + gpd_ecram_write(drvdata, drvdata->manual_control_enable, 0); break; } } // Write value for pwm1_enable -static void gpd_set_pwm_enable(enum FAN_PWM_ENABLE enable) +static void gpd_set_pwm_enable(struct gpd_fan_data *data, enum FAN_PWM_ENABLE enable) { if (enable == MANUAL) // Set pwm_value to max firstly when switching to manual mode, in // consideration of device safety. - gpd_driver_priv.pwm_value = 255; + data->pwm_value = 255; - switch (gpd_driver_priv.drvdata->board) { + switch (data->drvdata->board) { case win_mini: case win4_6800u: case mpc2: - gpd_win_mini_set_pwm_enable(enable); + gpd_win_mini_set_pwm_enable(data, enable); break; case duo: - gpd_duo_set_pwm_enable(enable); + gpd_duo_set_pwm_enable(data, enable); break; case win_max_2: - gpd_wm2_set_pwm_enable(enable); + gpd_wm2_set_pwm_enable(data, enable); break; } } @@ -505,15 +504,16 @@ static umode_t gpd_fan_hwmon_is_visible(__always_unused const void *drvdata, return 0; } -static int gpd_fan_hwmon_read(__always_unused struct device *dev, +static int gpd_fan_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, __always_unused int channel, long *val) { + struct gpd_fan_data *data = dev_get_drvdata(dev); int ret; if (type == hwmon_fan) { if (attr == hwmon_fan_input) { - ret = gpd_read_rpm(); + ret = gpd_read_rpm(data); if (ret < 0) return ret; @@ -524,10 +524,10 @@ static int gpd_fan_hwmon_read(__always_unused struct device *dev, } else if (type == hwmon_pwm) { switch (attr) { case hwmon_pwm_enable: - *val = gpd_driver_priv.pwm_enable; + *val = data->pwm_enable; return 0; case hwmon_pwm_input: - ret = gpd_read_pwm(); + ret = gpd_read_pwm(data); if (ret < 0) return ret; @@ -540,27 +540,29 @@ static int gpd_fan_hwmon_read(__always_unused struct device *dev, return -EOPNOTSUPP; } -static int gpd_fan_hwmon_write(__always_unused struct device *dev, +static int gpd_fan_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, __always_unused int channel, long val) { + struct gpd_fan_data *data = dev_get_drvdata(dev); + if (type == hwmon_pwm) { switch (attr) { case hwmon_pwm_enable: if (!in_range(val, 0, 3)) return -EINVAL; - gpd_driver_priv.pwm_enable = val; + data->pwm_enable = val; - gpd_set_pwm_enable(gpd_driver_priv.pwm_enable); + gpd_set_pwm_enable(data, data->pwm_enable); return 0; case hwmon_pwm_input: if (!in_range(val, 0, 256)) return -EINVAL; - gpd_driver_priv.pwm_value = val; + data->pwm_value = val; - return gpd_write_pwm(val); + return gpd_write_pwm(data, val); } } @@ -584,26 +586,37 @@ static struct hwmon_chip_info gpd_fan_chip_info = { .info = gpd_fan_hwmon_channel_info }; -static void gpd_win4_init_ec(void) +static void gpd_win4_init_ec(struct gpd_fan_data *data) { + const struct gpd_fan_drvdata *drvdata = data->drvdata; u8 chip_id, chip_ver; - gpd_ecram_read(0x2000, &chip_id); + gpd_ecram_read(drvdata, 0x2000, &chip_id); if (chip_id == 0x55) { - gpd_ecram_read(0x1060, &chip_ver); - gpd_ecram_write(0x1060, chip_ver | 0x80); + gpd_ecram_read(drvdata, 0x1060, &chip_ver); + gpd_ecram_write(drvdata, 0x1060, chip_ver | 0x80); } } -static void gpd_init_ec(void) +static void gpd_init_ec(struct gpd_fan_data *data) { // The buggy firmware won't initialize EC properly on boot. // Before its initialization, reading RPM will always return 0, // and writing PWM will have no effect. // Initialize it manually on driver load. - if (gpd_driver_priv.drvdata->board == win4_6800u) - gpd_win4_init_ec(); + if (data->drvdata->board == win4_6800u) + gpd_win4_init_ec(data); +} + +static void gpd_fan_reset_hardware(void *pdata) +{ + struct gpd_fan_data *data = pdata; + + if (data) { + data->pwm_enable = AUTOMATIC; + gpd_set_pwm_enable(data, AUTOMATIC); + } } static int gpd_fan_probe(struct platform_device *pdev) @@ -611,7 +624,10 @@ static int gpd_fan_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; const struct resource *region; const struct resource *res; - const struct device *hwdev; + struct device *hwdev; + struct gpd_fan_data *data; + const struct gpd_fan_drvdata *match; + int ret; res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (!res) @@ -624,29 +640,39 @@ static int gpd_fan_probe(struct platform_device *pdev) return dev_err_probe(dev, -EBUSY, "Failed to request region\n"); + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + match = dev_get_platdata(dev); + if (!match) + return -EINVAL; + + data->drvdata = match; + data->pwm_enable = AUTOMATIC; + data->pwm_value = 255; + + dev_set_drvdata(dev, data); + + gpd_init_ec(data); + + ret = devm_add_action_or_reset(dev, gpd_fan_reset_hardware, data); + if (ret) + return ret; + hwdev = devm_hwmon_device_register_with_info(dev, DRIVER_NAME, - NULL, + data, &gpd_fan_chip_info, NULL); if (IS_ERR(hwdev)) return dev_err_probe(dev, PTR_ERR(hwdev), "Failed to register hwmon device\n"); - - gpd_init_ec(); - return 0; } -static void gpd_fan_remove(__always_unused struct platform_device *pdev) -{ - gpd_driver_priv.pwm_enable = AUTOMATIC; - gpd_set_pwm_enable(AUTOMATIC); -} - static struct platform_driver gpd_fan_driver = { .probe = gpd_fan_probe, - .remove = gpd_fan_remove, .driver = { .name = KBUILD_MODNAME, }, @@ -668,6 +694,7 @@ static int __init gpd_fan_init(void) if (!match) { const struct dmi_system_id *dmi_match = dmi_first_match(dmi_table); + if (dmi_match) match = dmi_match->driver_data; } @@ -675,10 +702,6 @@ static int __init gpd_fan_init(void) if (!match) return -ENODEV; - gpd_driver_priv.pwm_enable = AUTOMATIC; - gpd_driver_priv.pwm_value = 255; - gpd_driver_priv.drvdata = match; - struct resource gpd_fan_resources[] = { { .start = match->addr_port, @@ -690,7 +713,8 @@ static int __init gpd_fan_init(void) gpd_fan_platform_device = platform_create_bundle(&gpd_fan_driver, gpd_fan_probe, gpd_fan_resources, - 1, NULL, 0); + 1, + match, sizeof(*match)); if (IS_ERR(gpd_fan_platform_device)) { pr_warn("Failed to create platform device\n"); diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c index 5fd310662ee43..87edb1b6048bb 100644 --- a/drivers/hwmon/it87.c +++ b/drivers/hwmon/it87.c @@ -1412,6 +1412,9 @@ static ssize_t set_fan(struct device *dev, struct device_attribute *attr, if (kstrtol(buf, 10, &val) < 0) return -EINVAL; + if (val < 0) + val = 0; + err = it87_lock(data); if (err) return err; diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index 42cc6068bb08b..e18e80e832fd3 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -214,6 +214,11 @@ int occ_update_response(struct occ *occ) if (rc) return rc; + if (!occ->active) { + rc = -ENODEV; + goto unlock; + } + /* limit the maximum rate of polling the OCC */ if (time_after(jiffies, occ->next_update)) { rc = occ_poll(occ); @@ -222,6 +227,7 @@ int occ_update_response(struct occ *occ) rc = occ->last_error; } +unlock: mutex_unlock(&occ->lock); return rc; } @@ -1105,11 +1111,16 @@ static void occ_parse_poll_response(struct occ *occ) int occ_active(struct occ *occ, bool active) { - int rc = mutex_lock_interruptible(&occ->lock); + struct device *hwmon = NULL; + int rc = mutex_lock_interruptible(&occ->hwmon_lock); if (rc) return rc; + rc = mutex_lock_interruptible(&occ->lock); + if (rc) + goto unlock_hwmon; + if (active) { if (occ->active) { rc = -EALREADY; @@ -1154,14 +1165,17 @@ int occ_active(struct occ *occ, bool active) goto unlock; } - if (occ->hwmon) - hwmon_device_unregister(occ->hwmon); + hwmon = occ->hwmon; occ->active = false; occ->hwmon = NULL; } unlock: mutex_unlock(&occ->lock); + if (hwmon) + hwmon_device_unregister(hwmon); +unlock_hwmon: + mutex_unlock(&occ->hwmon_lock); return rc; } @@ -1170,6 +1184,7 @@ int occ_setup(struct occ *occ) int rc; mutex_init(&occ->lock); + mutex_init(&occ->hwmon_lock); occ->groups[0] = &occ->group; rc = occ_setup_sysfs(occ); @@ -1190,15 +1205,22 @@ EXPORT_SYMBOL_GPL(occ_setup); void occ_shutdown(struct occ *occ) { - mutex_lock(&occ->lock); + struct device *hwmon; occ_shutdown_sysfs(occ); - if (occ->hwmon) - hwmon_device_unregister(occ->hwmon); + mutex_lock(&occ->hwmon_lock); + mutex_lock(&occ->lock); + + hwmon = occ->hwmon; + occ->active = false; occ->hwmon = NULL; mutex_unlock(&occ->lock); + + if (hwmon) + hwmon_device_unregister(hwmon); + mutex_unlock(&occ->hwmon_lock); } EXPORT_SYMBOL_GPL(occ_shutdown); diff --git a/drivers/hwmon/occ/common.h b/drivers/hwmon/occ/common.h index 7ac4b2febce65..82f600093c7fa 100644 --- a/drivers/hwmon/occ/common.h +++ b/drivers/hwmon/occ/common.h @@ -101,6 +101,7 @@ struct occ { unsigned long next_update; struct mutex lock; /* lock OCC access */ + struct mutex hwmon_lock; /* serialize hwmon registration/removal */ struct device *hwmon; struct occ_attribute *attrs; diff --git a/drivers/hwmon/pmbus/adm1275.c b/drivers/hwmon/pmbus/adm1275.c index bc2a6a07dc3e2..43baa5ded35e5 100644 --- a/drivers/hwmon/pmbus/adm1275.c +++ b/drivers/hwmon/pmbus/adm1275.c @@ -512,7 +512,7 @@ static int adm1275_enable_vout_temp(struct adm1275_data *data, static int adm1275_probe(struct i2c_client *client) { s32 (*config_read_fn)(const struct i2c_client *client, u8 reg); - u8 block_buffer[I2C_SMBUS_BLOCK_MAX + 1]; + u8 block_buffer[I2C_SMBUS_BLOCK_MAX + 1] = {0}; int config, device_config; int ret; struct pmbus_driver_info *info; diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index e8fdd799c71c0..3143b9e0316c4 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -1095,9 +1095,27 @@ static u16 pmbus_data2reg_direct(struct pmbus_data *data, static u16 pmbus_data2reg_vid(struct pmbus_data *data, struct pmbus_sensor *sensor, s64 val) { - val = clamp_val(val, 500, 1600); - - return 2 + DIV_ROUND_CLOSEST_ULL((1600LL - val) * 100LL, 625); + switch (data->info->vrm_version[sensor->page]) { + case vr12: + val = clamp_val(val, 250, 1520); + return 1 + DIV_ROUND_CLOSEST_ULL(val - 250, 5); + case vr13: + val = clamp_val(val, 500, 3040); + return 1 + DIV_ROUND_CLOSEST_ULL(val - 500, 10); + case imvp9: + val = clamp_val(val, 200, 2740); + return 1 + DIV_ROUND_CLOSEST_ULL(val - 200, 10); + case amd625mv: + val = clamp_val(val, 200, 1550); + return DIV_ROUND_CLOSEST_ULL((1550LL - val) * 100LL, 625); + case nvidia195mv: + val = clamp_val(val, 195, 1465); + return 1 + DIV_ROUND_CLOSEST_ULL(val - 195, 5); + case vr11: + default: + val = clamp_val(val, 500, 1600); + return 2 + DIV_ROUND_CLOSEST_ULL((1600LL - val) * 100LL, 625); + } } static u16 pmbus_data2reg(struct pmbus_data *data, @@ -3329,18 +3347,23 @@ static void pmbus_regulator_notify_worker(struct work_struct *work) int i, j; for (i = 0; i < data->info->pages; i++) { - int event; + unsigned int event; event = atomic_xchg(&data->regulator_events[i], 0); if (!event) continue; for (j = 0; j < data->info->num_regulators; j++) { - if (i == rdev_get_id(data->rdevs[j])) { + if (i != rdev_get_id(data->rdevs[j])) + continue; + while (event) { + unsigned int _event = BIT(__ffs(event)); + regulator_notifier_call_chain(data->rdevs[j], - event, NULL); - break; + _event, NULL); + event &= ~_event; } + break; } } } diff --git a/drivers/hwmon/w83627hf.c b/drivers/hwmon/w83627hf.c index 95115d7b863e3..bb993bb09f401 100644 --- a/drivers/hwmon/w83627hf.c +++ b/drivers/hwmon/w83627hf.c @@ -1823,6 +1823,8 @@ static int w83627hf_probe(struct platform_device *pdev) return 0; error: + device_remove_file(dev, &dev_attr_vrm); + device_remove_file(dev, &dev_attr_cpu0_vid); sysfs_remove_group(&dev->kobj, &w83627hf_group); sysfs_remove_group(&dev->kobj, &w83627hf_group_opt); return err; @@ -1834,6 +1836,8 @@ static void w83627hf_remove(struct platform_device *pdev) hwmon_device_unregister(data->hwmon_dev); + device_remove_file(&pdev->dev, &dev_attr_vrm); + device_remove_file(&pdev->dev, &dev_attr_cpu0_vid); sysfs_remove_group(&pdev->dev.kobj, &w83627hf_group); sysfs_remove_group(&pdev->dev.kobj, &w83627hf_group_opt); } diff --git a/drivers/hwmon/w83793.c b/drivers/hwmon/w83793.c index 24772cfbecb32..8726c5fe8a952 100644 --- a/drivers/hwmon/w83793.c +++ b/drivers/hwmon/w83793.c @@ -1917,6 +1917,7 @@ static int w83793_probe(struct i2c_client *client) for (i = 0; i < ARRAY_SIZE(w83793_vid); i++) device_remove_file(dev, &w83793_vid[i].dev_attr); + device_remove_file(dev, &dev_attr_vrm); for (i = 0; i < ARRAY_SIZE(w83793_left_fan); i++) device_remove_file(dev, &w83793_left_fan[i].dev_attr); diff --git a/drivers/hwspinlock/qcom_hwspinlock.c b/drivers/hwspinlock/qcom_hwspinlock.c index 0390979fd765d..712003a4640cc 100644 --- a/drivers/hwspinlock/qcom_hwspinlock.c +++ b/drivers/hwspinlock/qcom_hwspinlock.c @@ -202,7 +202,6 @@ static struct regmap *qcom_hwspinlock_probe_mmio(struct platform_device *pdev, static int qcom_hwspinlock_probe(struct platform_device *pdev) { struct hwspinlock_device *bank; - struct reg_field field; struct regmap *regmap; size_t array_size; u32 stride; @@ -224,9 +223,7 @@ static int qcom_hwspinlock_probe(struct platform_device *pdev) platform_set_drvdata(pdev, bank); for (i = 0; i < QCOM_MUTEX_NUM_LOCKS; i++) { - field.reg = base + i * stride; - field.lsb = 0; - field.msb = 31; + struct reg_field field = REG_FIELD(base + i * stride, 0, 31); bank->lock[i].priv = devm_regmap_field_alloc(&pdev->dev, regmap, field); diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 46f247f73cf64..256f6a32621b8 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -499,10 +499,19 @@ static int coresight_enable_helpers(struct coresight_device *csdev, ret = coresight_enable_helper(helper, mode, path); if (ret) - return ret; + goto err; } return 0; + +err: + while (i--) { + helper = csdev->pdata->out_conns[i]->dest_dev; + if (helper && coresight_is_helper(helper)) + coresight_disable_helper(helper, path); + } + + return ret; } int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) @@ -739,8 +748,8 @@ static int coresight_get_trace_id(struct coresight_device *csdev, * Call this after creating the path and before enabling it. This leaves * the trace ID set on the path, or it remains 0 if it couldn't be assigned. */ -void coresight_path_assign_trace_id(struct coresight_path *path, - enum cs_mode mode) +int coresight_path_assign_trace_id(struct coresight_path *path, + enum cs_mode mode) { struct coresight_device *sink = coresight_get_sink(path); struct coresight_node *nd; @@ -750,15 +759,18 @@ void coresight_path_assign_trace_id(struct coresight_path *path, /* Assign a trace ID to the path for the first device that wants to do it */ trace_id = coresight_get_trace_id(nd->csdev, mode, sink); - /* - * 0 in this context is that it didn't want to assign so keep searching. - * Non 0 is either success or fail. - */ - if (trace_id != 0) { - path->trace_id = trace_id; - return; - } + /* 0 means the device has no ID assignment, so keep searching */ + if (trace_id == 0) + continue; + + if (!IS_VALID_CS_TRACE_ID(trace_id)) + return -EINVAL; + + path->trace_id = trace_id; + return 0; } + + return -EINVAL; } /** diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c index 4eff96f48594e..d6d5388705c3e 100644 --- a/drivers/hwtracing/coresight/coresight-cti-platform.c +++ b/drivers/hwtracing/coresight/coresight-cti-platform.c @@ -329,6 +329,7 @@ static int cti_plat_process_filter_sigs(struct cti_drvdata *drvdata, if (!tg) return -ENOMEM; + tg->nr_sigs = nr_filter_sigs; err = cti_plat_read_trig_group(tg, fwnode, CTI_DT_FILTER_OUT_SIGS); if (!err) drvdata->config.trig_out_filter |= tg->used_mask; diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index b952a1d47f12f..a827f76b8144a 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -83,7 +83,7 @@ struct etb_drvdata { struct coresight_device *csdev; struct miscdevice miscdev; raw_spinlock_t spinlock; - local_t reading; + atomic_t reading; pid_t pid; u8 *buf; u32 buffer_depth; @@ -601,7 +601,7 @@ static int etb_open(struct inode *inode, struct file *file) struct etb_drvdata *drvdata = container_of(file->private_data, struct etb_drvdata, miscdev); - if (local_cmpxchg(&drvdata->reading, 0, 1)) + if (atomic_cmpxchg(&drvdata->reading, 0, 1)) return -EBUSY; dev_dbg(&drvdata->csdev->dev, "%s: successfully opened\n", __func__); @@ -639,7 +639,7 @@ static int etb_release(struct inode *inode, struct file *file) { struct etb_drvdata *drvdata = container_of(file->private_data, struct etb_drvdata, miscdev); - local_set(&drvdata->reading, 0); + atomic_set(&drvdata->reading, 0); dev_dbg(&drvdata->csdev->dev, "%s: released\n", __func__); return 0; diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index f85dedf89a3f9..89ba7c9a66137 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -324,6 +324,7 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, struct coresight_device *sink = NULL; struct coresight_device *user_sink = NULL, *last_sink = NULL; struct etm_event_data *event_data = NULL; + int ret; event_data = alloc_event_data(cpu); if (!event_data) @@ -420,8 +421,8 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, } /* ensure we can allocate a trace ID for this CPU */ - coresight_path_assign_trace_id(path, CS_MODE_PERF); - if (!IS_VALID_CS_TRACE_ID(path->trace_id)) { + ret = coresight_path_assign_trace_id(path, CS_MODE_PERF); + if (ret) { cpumask_clear_cpu(cpu, mask); coresight_release_path(path); continue; diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index d565a73f0042e..a251375db24b1 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -56,10 +56,14 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot"); #define PARAM_PM_SAVE_NEVER 1 /* never save any state */ #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */ +/* + * Save option for ETM4. ETE, sysreg ETM4s and ACPI boots ignore this option and + * will always save. + */ static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE; module_param(pm_save_enable, int, 0444); MODULE_PARM_DESC(pm_save_enable, - "Save/restore state on power down: 1 = never, 2 = self-hosted"); + "Save/restore state on power down: 1 = never, 2 = self-hosted. MMIO and DT only."); static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; static void etm4_set_default_config(struct etmv4_config *config); @@ -1979,7 +1983,7 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); + state->trcvmidcctlr1 = etm4x_read32(csa, TRCVMIDCCTLR1); state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); @@ -2012,7 +2016,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) { int ret = 0; - if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED) + if (!drvdata->save_state) return 0; /* @@ -2102,7 +2106,7 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, state->trcvmidcctlr1, TRCVMIDCCTLR1); etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); @@ -2127,7 +2131,7 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) { - if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED) + if (!drvdata->save_state) return; if (coresight_get_mode(drvdata->csdev)) @@ -2212,6 +2216,17 @@ static void etm4_pm_clear(void) } } +static bool etm4x_always_pm_save(struct device *dev, struct csdev_access *csa) +{ + /* + * Only IO mem ETM devices will benefit from skipping PM save and only + * DT has the option to control it, not ACPI. Otherwise system register + * based ETMs and ETEs will always lose context on CPU power down, so + * always save. + */ + return !csa->io_mem || is_acpi_device_node(dev_fwnode(dev)); +} + static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) { int ret; @@ -2221,6 +2236,7 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) struct coresight_desc desc = { 0 }; u8 major, minor; char *type_name; + bool pm_save; if (!drvdata) return -EINVAL; @@ -2248,6 +2264,21 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) etm4_set_default(&drvdata->config); + if (etm4x_always_pm_save(dev, init_arg->csa)) + pm_save = true; + else if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) + pm_save = coresight_loses_context_with_cpu(dev); + else + pm_save = pm_save_enable != PARAM_PM_SAVE_NEVER; + + if (pm_save) { + drvdata->save_state = devm_kmalloc(dev, + sizeof(struct etmv4_save_state), + GFP_KERNEL); + if (!drvdata->save_state) + return -ENOMEM; + } + pdata = coresight_get_platform_data(dev); if (IS_ERR(pdata)) return PTR_ERR(pdata); @@ -2305,17 +2336,6 @@ static int etm4_probe(struct device *dev) if (ret) return ret; - if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) - pm_save_enable = coresight_loses_context_with_cpu(dev) ? - PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER; - - if (pm_save_enable != PARAM_PM_SAVE_NEVER) { - drvdata->save_state = devm_kmalloc(dev, - sizeof(struct etmv4_save_state), GFP_KERNEL); - if (!drvdata->save_state) - return -ENOMEM; - } - raw_spin_lock_init(&drvdata->spinlock); drvdata->cpu = coresight_get_cpu(dev); diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c index e337b6e2bf327..93c2d075cad66 100644 --- a/drivers/hwtracing/coresight/coresight-platform.c +++ b/drivers/hwtracing/coresight/coresight-platform.c @@ -45,9 +45,8 @@ coresight_add_out_conn(struct device *dev, } } - pdata->nr_outconns++; pdata->out_conns = - devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns, + devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns + 1, sizeof(*pdata->out_conns), GFP_KERNEL); if (!pdata->out_conns) return ERR_PTR(-ENOMEM); @@ -63,7 +62,8 @@ coresight_add_out_conn(struct device *dev, * used right away. */ *conn = *new_conn; - pdata->out_conns[pdata->nr_outconns - 1] = conn; + pdata->out_conns[pdata->nr_outconns] = conn; + pdata->nr_outconns++; return conn; } EXPORT_SYMBOL_GPL(coresight_add_out_conn); @@ -86,13 +86,13 @@ int coresight_add_in_conn(struct coresight_connection *out_conn) return 0; } - pdata->nr_inconns++; pdata->in_conns = - devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns, + devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns + 1, sizeof(*pdata->in_conns), GFP_KERNEL); if (!pdata->in_conns) return -ENOMEM; - pdata->in_conns[pdata->nr_inconns - 1] = out_conn; + pdata->in_conns[pdata->nr_inconns] = out_conn; + pdata->nr_inconns++; return 0; } EXPORT_SYMBOL_GPL(coresight_add_in_conn); diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index 1ea882dffd703..34c7e792adbd9 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -153,7 +153,7 @@ int coresight_make_links(struct coresight_device *orig, void coresight_remove_links(struct coresight_device *orig, struct coresight_connection *conn); u32 coresight_get_sink_id(struct coresight_device *csdev); -void coresight_path_assign_trace_id(struct coresight_path *path, +int coresight_path_assign_trace_id(struct coresight_path *path, enum cs_mode mode); #if IS_ENABLED(CONFIG_CORESIGHT_SOURCE_ETM3X) diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c index d2a6ed8bcc74d..da6f22b512c92 100644 --- a/drivers/hwtracing/coresight/coresight-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-sysfs.c @@ -211,8 +211,8 @@ int coresight_enable_sysfs(struct coresight_device *csdev) goto out; } - coresight_path_assign_trace_id(path, CS_MODE_SYSFS); - if (!IS_VALID_CS_TRACE_ID(path->trace_id)) + ret = coresight_path_assign_trace_id(path, CS_MODE_SYSFS); + if (ret) goto err_path; ret = coresight_enable_path(path, CS_MODE_SYSFS); @@ -244,8 +244,10 @@ int coresight_enable_sysfs(struct coresight_device *csdev) */ hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev))); ret = idr_alloc_u32(&path_idr, path, &hash, hash, GFP_KERNEL); - if (ret) + if (ret) { + coresight_disable_source_sysfs(csdev, NULL); goto err_source; + } break; default: /* We can't be here */ diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 4dc1defe27a5f..361a433e6f0c5 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -154,7 +154,7 @@ tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr) for (i = 0; i < tmc_pages->nr_pages; i++) { page_start = tmc_pages->daddrs[i]; if (addr >= page_start && addr < (page_start + PAGE_SIZE)) - return i * PAGE_SIZE + (addr - page_start); + return (long)i * PAGE_SIZE + (addr - page_start); } return -EINVAL; @@ -1379,7 +1379,7 @@ alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); /* Use the minimum limit if the required size is smaller */ - size = nr_pages << PAGE_SHIFT; + size = (ssize_t)nr_pages << PAGE_SHIFT; size = max_t(ssize_t, size, TMC_ETR_PERF_MIN_BUF_SIZE); /* diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.c b/drivers/hwtracing/coresight/ultrasoc-smb.c index 5776f63468fa0..20a950b9dd4ff 100644 --- a/drivers/hwtracing/coresight/ultrasoc-smb.c +++ b/drivers/hwtracing/coresight/ultrasoc-smb.c @@ -337,6 +337,7 @@ static void smb_sync_perf_buffer(struct smb_drv_data *drvdata, unsigned long to_copy; long pg_idx, pg_offset; + head %= (unsigned long)buf->nr_pages << PAGE_SHIFT; pg_idx = head >> PAGE_SHIFT; pg_offset = head & (PAGE_SIZE - 1); diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c index 66c23535656b5..0617f416cb0b5 100644 --- a/drivers/i2c/busses/i2c-davinci.c +++ b/drivers/i2c/busses/i2c-davinci.c @@ -818,12 +818,14 @@ static int davinci_i2c_probe(struct platform_device *pdev) adap->nr = pdev->id; r = i2c_add_numbered_adapter(adap); if (r) - goto err_unuse_clocks; + goto err_cpufreq; pm_runtime_put_autosuspend(dev->dev); return 0; +err_cpufreq: + i2c_davinci_cpufreq_deregister(dev); err_unuse_clocks: pm_runtime_dont_use_autosuspend(dev->dev); pm_runtime_put_sync(dev->dev); diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 32a3cef02c7b5..b29c99ed38839 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -931,13 +931,13 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, */ if (hwpec) iowrite8(ioread8(SMBAUXCTL(priv)) & ~SMBAUXCTL_CRC, SMBAUXCTL(priv)); -out: /* * Unlock the SMBus device for use by BIOS/ACPI, * and clear status flags if not done already. */ iowrite8(SMBHSTSTS_INUSE_STS | STATUS_FLAGS, SMBHSTSTS(priv)); +out: pm_runtime_put_autosuspend(&priv->pci_dev->dev); mutex_unlock(&priv->acpi_lock); return ret; diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index cd4da50c4dd96..e6c24a9d934d7 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -1646,7 +1646,18 @@ static int __maybe_unused lpi2c_runtime_resume(struct device *dev) static int __maybe_unused lpi2c_suspend_noirq(struct device *dev) { - return pm_runtime_force_suspend(dev); + struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev); + int ret; + + i2c_mark_adapter_suspended(&lpi2c_imx->adapter); + + ret = pm_runtime_force_suspend(dev); + if (ret) { + i2c_mark_adapter_resumed(&lpi2c_imx->adapter); + return ret; + } + + return 0; } static int __maybe_unused lpi2c_resume_noirq(struct device *dev) @@ -1666,6 +1677,8 @@ static int __maybe_unused lpi2c_resume_noirq(struct device *dev) if (lpi2c_imx->target) lpi2c_imx_target_init(lpi2c_imx); + i2c_mark_adapter_resumed(&lpi2c_imx->adapter); + return 0; } diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index 28313d0fad37e..d5e6e2eca3b34 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -1061,11 +1061,28 @@ static inline enum imx_i2c_state i2c_imx_isr_read_continue(struct imx_i2c_struct static inline void i2c_imx_isr_read_block_data_len(struct imx_i2c_struct *i2c_imx) { u8 len = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); + unsigned int temp; if (len == 0 || len > I2C_SMBUS_BLOCK_MAX) { + /* + * SMBus 3.1 6.5.7: support count byte of 0. + * I2C_SMBUS_BLOCK_MAX case should not hold the SDA either. + * So NACK it (TXAK) to not hold the bus. + */ + temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); + temp |= I2CR_TXAK; + imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); + + if (len == 0) { + i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = 0; + i2c_imx->msg->len = 2; + return; + } + i2c_imx->isr_result = -EPROTO; i2c_imx->state = IMX_I2C_STATE_FAILED; wake_up(&i2c_imx->queue); + return; } i2c_imx->msg->len += len; i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = len; @@ -1415,6 +1432,7 @@ static int i2c_imx_atomic_read(struct imx_i2c_struct *i2c_imx, int i, result; unsigned int temp; int block_data = msgs->flags & I2C_M_RECV_LEN; + int block_err = 0; result = i2c_imx_prepare_read(i2c_imx, msgs, false); if (result) @@ -1436,8 +1454,20 @@ static int i2c_imx_atomic_read(struct imx_i2c_struct *i2c_imx, */ if ((!i) && block_data) { len = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); - if ((len == 0) || (len > I2C_SMBUS_BLOCK_MAX)) - return -EPROTO; + if ((len == 0) || (len > I2C_SMBUS_BLOCK_MAX)) { + /* + * SMBus 3.1 6.5.7: support count byte of 0. + * I2C_SMBUS_BLOCK_MAX case should not hold the SDA either. + */ + if (len > I2C_SMBUS_BLOCK_MAX) + block_err = -EPROTO; + temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); + temp |= I2CR_TXAK; + imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); + msgs->buf[0] = 0; + msgs->len = 2; + continue; + } dev_dbg(&i2c_imx->adapter.dev, "<%s> read length: 0x%X\n", __func__, len); @@ -1485,7 +1515,7 @@ static int i2c_imx_atomic_read(struct imx_i2c_struct *i2c_imx, "<%s> read byte: B%d=0x%X\n", __func__, i, msgs->buf[i]); } - return 0; + return block_err; } static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, diff --git a/drivers/i2c/busses/i2c-k1.c b/drivers/i2c/busses/i2c-k1.c index 9152cf436bea0..51a0c3d80fc92 100644 --- a/drivers/i2c/busses/i2c-k1.c +++ b/drivers/i2c/busses/i2c-k1.c @@ -596,7 +596,7 @@ static irqreturn_t spacemit_i2c_irq_handler(int irq, void *devid) status = readl(i2c->base + SPACEMIT_ISR); if (!status) - return IRQ_HANDLED; + return IRQ_NONE; i2c->status = status; diff --git a/drivers/i2c/busses/i2c-mlxbf.c b/drivers/i2c/busses/i2c-mlxbf.c index 6c1cfe9ec8acc..e33512b253531 100644 --- a/drivers/i2c/busses/i2c-mlxbf.c +++ b/drivers/i2c/busses/i2c-mlxbf.c @@ -1051,8 +1051,10 @@ static int mlxbf_i2c_init_resource(struct platform_device *pdev, tmp_res->io = devm_platform_get_and_ioremap_resource(pdev, type, &tmp_res->params); if (IS_ERR(tmp_res->io)) { + int ret = PTR_ERR(tmp_res->io); + devm_kfree(dev, tmp_res); - return PTR_ERR(tmp_res->io); + return ret; } tmp_res->type = type; diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c index 28c5c5c1fb7a6..a21fa45bd64c4 100644 --- a/drivers/i2c/busses/i2c-mpc.c +++ b/drivers/i2c/busses/i2c-mpc.c @@ -844,7 +844,7 @@ static int fsl_i2c_probe(struct platform_device *op) "fsl,timeout", &mpc_ops.timeout); if (!result) { - mpc_ops.timeout *= HZ / 1000000; + mpc_ops.timeout = mpc_ops.timeout * HZ / 1000000; if (mpc_ops.timeout < 5) mpc_ops.timeout = 5; } else { diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c index 126040ca05f14..307925fb78e36 100644 --- a/drivers/i2c/busses/i2c-mt65xx.c +++ b/drivers/i2c/busses/i2c-mt65xx.c @@ -1258,7 +1258,7 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap, i2c->auto_restart = i2c->dev_comp->auto_restart; /* checking if we can skip restart and optimize using WRRD mode */ - if (i2c->auto_restart && num == 2) { + if (num == 2) { if (!(msgs[0].flags & I2C_M_RD) && (msgs[1].flags & I2C_M_RD) && msgs[0].addr == msgs[1].addr) { i2c->auto_restart = 0; diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index 067af255bd22a..0ddd8a1d0e4d0 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -464,8 +464,13 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, { struct stm32f7_i2c_spec *specs; u32 p_prev = STM32F7_PRESC_MAX; - u32 i2cclk = DIV_ROUND_CLOSEST(NSEC_PER_SEC, - setup->clock_src); + /* + * Truncate instead of rounding to closest: if the clock period is + * overestimated, the computed SCL timings will come out shorter on + * the wire, which can push the bus above the target rate and below + * the spec's tLOW/tHIGH minimums. + */ + u32 i2cclk = NSEC_PER_SEC / setup->clock_src; u32 i2cbus = DIV_ROUND_CLOSEST(NSEC_PER_SEC, setup->speed_freq); u32 clk_error_prev = i2cbus; diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index a2132d70fb360..8bea9fc6888fb 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -63,6 +63,7 @@ static DEFINE_MUTEX(core_lock); static DEFINE_IDR(i2c_adapter_idr); +static void i2c_deregister_clients(struct i2c_adapter *adap); static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver); static DEFINE_STATIC_KEY_FALSE(i2c_trace_msg_key); @@ -1555,11 +1556,18 @@ static int i2c_register_adapter(struct i2c_adapter *adap) goto out_list; } - dev_set_name(&adap->dev, "i2c-%d", adap->nr); + res = dev_set_name(&adap->dev, "i2c-%d", adap->nr); + if (res) + goto err_remove_irq_domain; + adap->dev.bus = &i2c_bus_type; adap->dev.type = &i2c_adapter_type; device_initialize(&adap->dev); + res = i2c_init_recovery(adap); + if (res == -EPROBE_DEFER) + goto err_put_adap; + /* * This adapter can be used as a parent immediately after device_add(), * setup runtime-pm (especially ignore-children) before hand. @@ -1569,23 +1577,22 @@ static int i2c_register_adapter(struct i2c_adapter *adap) pm_suspend_ignore_children(&adap->dev, true); pm_runtime_enable(&adap->dev); + mutex_lock(&core_lock); + idr_replace(&i2c_adapter_idr, adap, adap->nr); + mutex_unlock(&core_lock); + + adap->debugfs = debugfs_create_dir(dev_name(&adap->dev), i2c_debugfs_root); + res = device_add(&adap->dev); if (res) { pr_err("adapter '%s': can't register device (%d)\n", adap->name, res); - put_device(&adap->dev); - goto out_list; + goto err_replace_id; } - adap->debugfs = debugfs_create_dir(dev_name(&adap->dev), i2c_debugfs_root); - res = i2c_setup_smbus_alert(adap); if (res) goto out_reg; - res = i2c_init_recovery(adap); - if (res == -EPROBE_DEFER) - goto out_reg; - dev_dbg(&adap->dev, "adapter [%s] registered\n", adap->name); /* create pre-declared device nodes */ @@ -1604,10 +1611,19 @@ static int i2c_register_adapter(struct i2c_adapter *adap) return 0; out_reg: + i2c_deregister_clients(adap); + device_del(&adap->dev); +err_replace_id: + mutex_lock(&core_lock); + idr_replace(&i2c_adapter_idr, NULL, adap->nr); + mutex_unlock(&core_lock); debugfs_remove_recursive(adap->debugfs); +err_put_adap: init_completion(&adap->dev_released); - device_unregister(&adap->dev); + put_device(&adap->dev); wait_for_completion(&adap->dev_released); +err_remove_irq_domain: + i2c_host_notify_irq_teardown(adap); out_list: mutex_lock(&core_lock); idr_remove(&i2c_adapter_idr, adap->nr); @@ -1627,7 +1643,7 @@ static int __i2c_add_numbered_adapter(struct i2c_adapter *adap) int id; mutex_lock(&core_lock); - id = idr_alloc(&i2c_adapter_idr, adap, adap->nr, adap->nr + 1, GFP_KERNEL); + id = idr_alloc(&i2c_adapter_idr, NULL, adap->nr, adap->nr + 1, GFP_KERNEL); mutex_unlock(&core_lock); if (WARN(id < 0, "couldn't get idr")) return id == -ENOSPC ? -EBUSY : id; @@ -1661,7 +1677,7 @@ int i2c_add_adapter(struct i2c_adapter *adapter) } mutex_lock(&core_lock); - id = idr_alloc(&i2c_adapter_idr, adapter, + id = idr_alloc(&i2c_adapter_idr, NULL, __i2c_first_dynamic_bus_num, 0, GFP_KERNEL); mutex_unlock(&core_lock); if (WARN(id < 0, "couldn't get idr")) @@ -1743,29 +1759,10 @@ static int __process_removed_adapter(struct device_driver *d, void *data) return 0; } -/** - * i2c_del_adapter - unregister I2C adapter - * @adap: the adapter being unregistered - * Context: can sleep - * - * This unregisters an I2C adapter which was previously registered - * by @i2c_add_adapter or @i2c_add_numbered_adapter. - */ -void i2c_del_adapter(struct i2c_adapter *adap) +static void i2c_deregister_clients(struct i2c_adapter *adap) { - struct i2c_adapter *found; struct i2c_client *client, *next; - /* First make sure that this adapter was ever added */ - mutex_lock(&core_lock); - found = idr_find(&i2c_adapter_idr, adap->nr); - mutex_unlock(&core_lock); - if (found != adap) { - pr_debug("attempting to delete unregistered adapter [%s]\n", adap->name); - return; - } - - i2c_acpi_remove_space_handler(adap); /* Tell drivers about this removal */ mutex_lock(&core_lock); bus_for_each_drv(&i2c_bus_type, NULL, adap, @@ -1791,6 +1788,34 @@ void i2c_del_adapter(struct i2c_adapter *adap) * them up properly, so we give them a chance to do that first. */ device_for_each_child(&adap->dev, NULL, __unregister_client); device_for_each_child(&adap->dev, NULL, __unregister_dummy); +} + +/** + * i2c_del_adapter - unregister I2C adapter + * @adap: the adapter being unregistered + * Context: can sleep + * + * This unregisters an I2C adapter which was previously registered + * by @i2c_add_adapter or @i2c_add_numbered_adapter. + */ +void i2c_del_adapter(struct i2c_adapter *adap) +{ + struct i2c_adapter *found; + + /* First make sure that this adapter was ever added */ + mutex_lock(&core_lock); + found = idr_find(&i2c_adapter_idr, adap->nr); + if (found == adap) + idr_replace(&i2c_adapter_idr, NULL, adap->nr); + mutex_unlock(&core_lock); + if (found != adap) { + pr_debug("attempting to delete unregistered adapter [%s]\n", adap->name); + return; + } + + i2c_acpi_remove_space_handler(adap); + + i2c_deregister_clients(adap); /* device name is gone after device_unregister */ dev_dbg(&adap->dev, "adapter [%s] unregistered\n", adap->name); diff --git a/drivers/i2c/i2c-core-of.c b/drivers/i2c/i2c-core-of.c index 354a88d0599e3..30b48a428c0be 100644 --- a/drivers/i2c/i2c-core-of.c +++ b/drivers/i2c/i2c-core-of.c @@ -176,11 +176,6 @@ static int of_i2c_notify(struct notifier_block *nb, unsigned long action, return NOTIFY_OK; } - /* - * Clear the flag before adding the device so that fw_devlink - * doesn't skip adding consumers to this device. - */ - fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); client = of_i2c_register_device(adap, rd->dn); if (IS_ERR(client)) { dev_err(&adap->dev, "failed to create client for '%pOF'\n", diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 5cd4e5da22333..74c04e19f86c5 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -368,14 +368,6 @@ static void i3c_device_remove(struct device *dev) driver->remove(i3cdev); } -const struct bus_type i3c_bus_type = { - .name = "i3c", - .match = i3c_device_match, - .probe = i3c_device_probe, - .remove = i3c_device_remove, -}; -EXPORT_SYMBOL_GPL(i3c_bus_type); - static enum i3c_addr_slot_status i3c_bus_get_addr_slot_status_mask(struct i3c_bus *bus, u16 addr, u32 mask) { @@ -633,6 +625,14 @@ static ssize_t i2c_scl_frequency_show(struct device *dev, } static DEVICE_ATTR_RO(i2c_scl_frequency); +static void i3c_master_hj_work_fn(struct work_struct *work) +{ + struct i3c_master_controller *master = container_of(work, typeof(*master), hj_work); + + if (!master->shutting_down) + i3c_master_do_daa(master); +} + static int i3c_set_hotjoin(struct i3c_master_controller *master, bool enable) { int ret; @@ -649,9 +649,11 @@ static int i3c_set_hotjoin(struct i3c_master_controller *master, bool enable) return ret; } - i3c_bus_normaluse_lock(&master->bus); + i3c_bus_maintenance_lock(&master->bus); - if (enable) + if (master->shutting_down) + ret = -ENODEV; + else if (enable) ret = master->ops->enable_hotjoin(master); else ret = master->ops->disable_hotjoin(master); @@ -659,7 +661,7 @@ static int i3c_set_hotjoin(struct i3c_master_controller *master, bool enable) if (!ret) master->hotjoin = enable; - i3c_bus_normaluse_unlock(&master->bus); + i3c_bus_maintenance_unlock(&master->bus); if ((enable && ret) || (!enable && !ret) || master->rpm_ibi_allowed) i3c_master_rpm_put(master); @@ -711,6 +713,18 @@ int i3c_master_disable_hotjoin(struct i3c_master_controller *master) } EXPORT_SYMBOL_GPL(i3c_master_disable_hotjoin); +/** + * i3c_master_queue_hotjoin - Queue DAA processing after a Hot-Join event + * @master: I3C master object + * + * Queue the hot-join worker on the master's workqueue. + */ +void i3c_master_queue_hotjoin(struct i3c_master_controller *master) +{ + queue_work(master->wq, &master->hj_work); +} +EXPORT_SYMBOL_GPL(i3c_master_queue_hotjoin); + static ssize_t hotjoin_show(struct device *dev, struct device_attribute *da, char *buf) { struct i3c_bus *i3cbus = dev_to_i3cbus(dev); @@ -728,7 +742,14 @@ static DEVICE_ATTR_RW(hotjoin); static ssize_t dev_nack_retry_count_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sysfs_emit(buf, "%u\n", dev_to_i3cmaster(dev)->dev_nack_retry_count); + struct i3c_bus *i3cbus = dev_to_i3cbus(dev); + ssize_t ret; + + i3c_bus_normaluse_lock(i3cbus); + ret = sysfs_emit(buf, "%u\n", dev_to_i3cmaster(dev)->dev_nack_retry_count); + i3c_bus_normaluse_unlock(i3cbus); + + return ret; } static ssize_t dev_nack_retry_count_store(struct device *dev, @@ -744,16 +765,19 @@ static ssize_t dev_nack_retry_count_store(struct device *dev, if (ret) return ret; + ret = i3c_master_rpm_get(master); + if (ret) + return ret; + i3c_bus_maintenance_lock(i3cbus); ret = master->ops->set_dev_nack_retry(master, val); + if (!ret) + master->dev_nack_retry_count = val; i3c_bus_maintenance_unlock(i3cbus); - if (ret) - return ret; - - master->dev_nack_retry_count = val; + i3c_master_rpm_put(master); - return count; + return ret ?: count; } static DEVICE_ATTR_RW(dev_nack_retry_count); @@ -818,6 +842,31 @@ static const struct device_type i3c_masterdev_type = { .groups = i3c_masterdev_groups, }; +static void i3c_master_shutdown(struct i3c_master_controller *master) +{ + i3c_bus_maintenance_lock(&master->bus); + master->shutting_down = true; + i3c_bus_maintenance_unlock(&master->bus); + + cancel_work_sync(&master->hj_work); + cancel_work_sync(&master->reg_work); +} + +static void i3c_device_shutdown(struct device *dev) +{ + if (dev->type == &i3c_masterdev_type) + i3c_master_shutdown(dev_to_i3cmaster(dev)); +} + +const struct bus_type i3c_bus_type = { + .name = "i3c", + .match = i3c_device_match, + .probe = i3c_device_probe, + .remove = i3c_device_remove, + .shutdown = i3c_device_shutdown, +}; +EXPORT_SYMBOL_GPL(i3c_bus_type); + static int i3c_bus_set_mode(struct i3c_bus *i3cbus, enum i3c_bus_mode mode, unsigned long max_i2c_scl_rate) { @@ -1800,6 +1849,16 @@ i3c_master_register_new_i3c_devs(struct i3c_master_controller *master) } } +static void i3c_master_reg_work_fn(struct work_struct *work) +{ + struct i3c_master_controller *master = container_of(work, typeof(*master), reg_work); + + i3c_bus_normaluse_lock(&master->bus); + if (!master->shutting_down) + i3c_master_register_new_i3c_devs(master); + i3c_bus_normaluse_unlock(&master->bus); +} + /** * i3c_master_do_daa_ext() - Dynamic Address Assignment (extended version) * @master: controller @@ -1827,19 +1886,20 @@ int i3c_master_do_daa_ext(struct i3c_master_controller *master, bool rstdaa) i3c_bus_maintenance_lock(&master->bus); - if (rstdaa) - rstret = i3c_master_rstdaa_locked(master, I3C_BROADCAST_ADDR); - - ret = master->ops->do_daa(master); + if (master->shutting_down) { + ret = -ENODEV; + } else { + if (rstdaa) + rstret = i3c_master_rstdaa_locked(master, I3C_BROADCAST_ADDR); + ret = master->ops->do_daa(master); + } i3c_bus_maintenance_unlock(&master->bus); if (ret) goto out; - i3c_bus_normaluse_lock(&master->bus); - i3c_master_register_new_i3c_devs(master); - i3c_bus_normaluse_unlock(&master->bus); + queue_work(master->wq, &master->reg_work); out: i3c_master_rpm_put(master); @@ -2274,12 +2334,11 @@ int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, bool enable_ibi = false; int ret; - if (!master) - return -EINVAL; - newdev = i3c_master_alloc_i3c_dev(master, &info); - if (IS_ERR(newdev)) - return PTR_ERR(newdev); + if (IS_ERR(newdev)) { + ret = PTR_ERR(newdev); + goto err_prevent_addr_reuse; + } ret = i3c_master_attach_i3c_dev(master, newdev); if (ret) @@ -2401,6 +2460,16 @@ int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, err_free_dev: i3c_master_free_i3c_dev(newdev); +err_prevent_addr_reuse: + /* + * Although the device has not been added, the address has been + * assigned. Prevent the address from being used again. + */ + if (i3c_bus_get_addr_slot_status(&master->bus, addr) == I3C_ADDR_SLOT_FREE) + i3c_bus_set_addr_slot_status(&master->bus, addr, I3C_ADDR_SLOT_I3C_DEV); + + dev_err(&master->dev, "Failed to add I3C device at address %u, error %d\n", addr, ret); + return ret; } EXPORT_SYMBOL_GPL(i3c_master_add_i3c_dev_locked); @@ -3079,11 +3148,13 @@ int i3c_master_register(struct i3c_master_controller *master, if (ret) goto err_put_dev; - master->wq = alloc_workqueue("%s", WQ_PERCPU, 0, dev_name(parent)); + master->wq = alloc_workqueue("%s", WQ_PERCPU | WQ_FREEZABLE, 0, dev_name(parent)); if (!master->wq) { ret = -ENOMEM; goto err_put_dev; } + INIT_WORK(&master->hj_work, i3c_master_hj_work_fn); + INIT_WORK(&master->reg_work, i3c_master_reg_work_fn); ret = i3c_master_bus_init(master); if (ret) @@ -3109,12 +3180,15 @@ int i3c_master_register(struct i3c_master_controller *master, /* * We're done initializing the bus and the controller, we can now - * register I3C devices discovered during the initial DAA. + * register I3C devices discovered during the initial DAA. Device + * registration is done via reg_work because that keeps a single + * registration code path and ensures the worker is the only writer + * of desc->dev. Flush the work to preserve synchronous probe-time + * behavior. */ master->init_done = true; - i3c_bus_normaluse_lock(&master->bus); - i3c_master_register_new_i3c_devs(master); - i3c_bus_normaluse_unlock(&master->bus); + queue_work(master->wq, &master->reg_work); + flush_work(&master->reg_work); if (master->ops->set_dev_nack_retry) device_create_file(&master->dev, &dev_attr_dev_nack_retry_count); @@ -3146,6 +3220,7 @@ EXPORT_SYMBOL_GPL(i3c_master_register); void i3c_master_unregister(struct i3c_master_controller *master) { i3c_bus_notify(&master->bus, I3C_NOTIFY_BUS_REMOVE); + i3c_master_shutdown(master); if (master->ops->set_dev_nack_retry) device_remove_file(&master->dev, &dev_attr_dev_nack_retry_count); diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 655693a2187e5..eb9a13a736848 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -1445,7 +1445,7 @@ static void dw_i3c_master_irq_handle_ibis(struct dw_i3c_master *master) if (IBI_TYPE_SIRQ(reg)) { dw_i3c_master_handle_ibi_sir(master, reg); } else if (IBI_TYPE_HJ(reg)) { - queue_work(master->base.wq, &master->hj_work); + i3c_master_queue_hotjoin(&master->base); } else { len = IBI_QUEUE_STATUS_DATA_LEN(reg); dev_info(&master->base.dev, @@ -1554,14 +1554,6 @@ static const struct dw_i3c_platform_ops dw_i3c_platform_ops_default = { .set_dat_ibi = dw_i3c_platform_set_dat_ibi_nop, }; -static void dw_i3c_hj_work(struct work_struct *work) -{ - struct dw_i3c_master *master = - container_of(work, typeof(*master), hj_work); - - i3c_master_do_daa(&master->base); -} - int dw_i3c_common_probe(struct dw_i3c_master *master, struct platform_device *pdev) { @@ -1636,8 +1628,6 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, if (master->quirks & DW_I3C_DISABLE_RUNTIME_PM_QUIRK) pm_runtime_get_noresume(&pdev->dev); - INIT_WORK(&master->hj_work, dw_i3c_hj_work); - device_set_of_node_from_dev(&master->base.i2c.dev, &pdev->dev); ret = i3c_master_register(&master->base, &pdev->dev, &dw_mipi_i3c_ops, false); @@ -1659,7 +1649,6 @@ EXPORT_SYMBOL_GPL(dw_i3c_common_probe); void dw_i3c_common_remove(struct dw_i3c_master *master) { - cancel_work_sync(&master->hj_work); i3c_master_unregister(&master->base); /* Balance pm_runtime_get_noresume() from probe() */ @@ -1804,7 +1793,7 @@ static void dw_i3c_shutdown(struct platform_device *pdev) return; } - cancel_work_sync(&master->hj_work); + cancel_work_sync(&master->base.hj_work); /* Disable interrupts */ writel((u32)~INTR_ALL, master->regs + INTR_STATUS_EN); diff --git a/drivers/i3c/master/dw-i3c-master.h b/drivers/i3c/master/dw-i3c-master.h index c5cb695c16ab8..2f029bd362326 100644 --- a/drivers/i3c/master/dw-i3c-master.h +++ b/drivers/i3c/master/dw-i3c-master.h @@ -68,8 +68,6 @@ struct dw_i3c_master { /* platform-specific data */ const struct dw_i3c_platform_ops *platform_ops; - - struct work_struct hj_work; }; struct dw_i3c_platform_ops { diff --git a/drivers/i3c/master/i3c-master-cdns.c b/drivers/i3c/master/i3c-master-cdns.c index 5cfec6761494d..6d221596ea357 100644 --- a/drivers/i3c/master/i3c-master-cdns.c +++ b/drivers/i3c/master/i3c-master-cdns.c @@ -398,7 +398,6 @@ struct cdns_i3c_data { }; struct cdns_i3c_master { - struct work_struct hj_work; struct i3c_master_controller base; u32 free_rr_slots; unsigned int maxdevs; @@ -1357,7 +1356,7 @@ static void cnds_i3c_master_demux_ibis(struct cdns_i3c_master *master) case IBIR_TYPE_HJ: WARN_ON(IBIR_XFER_BYTES(ibir) || (ibir & IBIR_ERROR)); - queue_work(master->base.wq, &master->hj_work); + i3c_master_queue_hotjoin(&master->base); break; case IBIR_TYPE_MR: @@ -1528,15 +1527,6 @@ static const struct i3c_master_controller_ops cdns_i3c_master_ops = { .recycle_ibi_slot = cdns_i3c_master_recycle_ibi_slot, }; -static void cdns_i3c_master_hj(struct work_struct *work) -{ - struct cdns_i3c_master *master = container_of(work, - struct cdns_i3c_master, - hj_work); - - i3c_master_do_daa(&master->base); -} - static struct cdns_i3c_data cdns_i3c_devdata = { .thd_delay_ns = 10, }; @@ -1584,7 +1574,6 @@ static int cdns_i3c_master_probe(struct platform_device *pdev) spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); - INIT_WORK(&master->hj_work, cdns_i3c_master_hj); writel(0xffffffff, master->regs + MST_IDR); writel(0xffffffff, master->regs + SLV_IDR); ret = devm_request_irq(&pdev->dev, irq, cdns_i3c_master_interrupt, 0, @@ -1627,7 +1616,6 @@ static void cdns_i3c_master_remove(struct platform_device *pdev) { struct cdns_i3c_master *master = platform_get_drvdata(pdev); - cancel_work_sync(&master->hj_work); i3c_master_unregister(&master->base); } diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c index b781dbed21650..388a42f4d2ec7 100644 --- a/drivers/i3c/master/mipi-i3c-hci/core.c +++ b/drivers/i3c/master/mipi-i3c-hci/core.c @@ -22,6 +22,7 @@ #include "ext_caps.h" #include "cmd.h" #include "dat.h" +#include "ibi.h" /* * Host Controller Capabilities and Operation Registers @@ -124,6 +125,7 @@ static void i3c_hci_set_master_dyn_addr(struct i3c_hci *hci) static int i3c_hci_bus_init(struct i3c_master_controller *m) { struct i3c_hci *hci = to_i3c_hci(m); + struct device *dev = hci->master.dev.parent; struct i3c_device_info info; int ret; @@ -144,6 +146,10 @@ static int i3c_hci_bus_init(struct i3c_master_controller *m) if (ret) return ret; + hci->ibi_devs = devm_kcalloc(dev, hci->DAT_entries, sizeof(*hci->ibi_devs), GFP_KERNEL); + if (!hci->ibi_devs) + return -ENOMEM; + ret = hci->io->init(hci); if (ret) return ret; @@ -556,14 +562,40 @@ static int i3c_hci_request_ibi(struct i3c_dev_desc *dev, return hci->io->request_ibi(hci, dev, req); } +static void __i3c_hci_disable_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + + mipi_i3c_hci_dat_v1.set_flags(hci, dev_data->dat_idx, DAT_0_SIR_REJECT, 0); + scoped_guard(spinlock_irqsave, &hci->lock) + hci->ibi_devs[dev_data->dat_idx] = NULL; +} + static void i3c_hci_free_ibi(struct i3c_dev_desc *dev) { struct i3c_master_controller *m = i3c_dev_get_master(dev); struct i3c_hci *hci = to_i3c_hci(m); + /* Must ensure the IBI has been disabled */ + __i3c_hci_disable_ibi(hci, dev); hci->io->free_ibi(hci, dev); } +struct i3c_dev_desc *i3c_hci_addr_to_dev(struct i3c_hci *hci, unsigned int addr) +{ + int dat_idx; + + lockdep_assert_held(&hci->lock); + + for (dat_idx = 0; dat_idx < hci->DAT_entries; dat_idx++) { + struct i3c_dev_desc *dev = hci->ibi_devs[dat_idx]; + + if (dev && dev->info.dyn_addr == addr) + return dev; + } + return NULL; +} + static int i3c_hci_enable_ibi(struct i3c_dev_desc *dev) { struct i3c_master_controller *m = i3c_dev_get_master(dev); @@ -571,6 +603,8 @@ static int i3c_hci_enable_ibi(struct i3c_dev_desc *dev) struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); mipi_i3c_hci_dat_v1.clear_flags(hci, dev_data->dat_idx, DAT_0_SIR_REJECT, 0); + scoped_guard(spinlock_irqsave, &hci->lock) + hci->ibi_devs[dev_data->dat_idx] = dev; return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); } @@ -578,9 +612,8 @@ static int i3c_hci_disable_ibi(struct i3c_dev_desc *dev) { struct i3c_master_controller *m = i3c_dev_get_master(dev); struct i3c_hci *hci = to_i3c_hci(m); - struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); - mipi_i3c_hci_dat_v1.set_flags(hci, dev_data->dat_idx, DAT_0_SIR_REJECT, 0); + __i3c_hci_disable_ibi(hci, dev); return i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); } @@ -762,15 +795,10 @@ static int i3c_hci_reset_and_init(struct i3c_hci *hci) int i3c_hci_rpm_suspend(struct device *dev) { struct i3c_hci *hci = dev_get_drvdata(dev); - int ret; - ret = i3c_hci_bus_disable(hci); - if (ret) { - /* Fall back to software reset to disable the bus */ - ret = i3c_hci_software_reset(hci); - i3c_hci_sync_irq_inactive(hci); - return ret; - } + /* Fall back to software reset to disable the bus */ + if (i3c_hci_bus_disable(hci)) + i3c_hci_software_reset(hci); hci->io->suspend(hci); diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c index e4daaa6120550..8517d2c14e8bf 100644 --- a/drivers/i3c/master/mipi-i3c-hci/dma.c +++ b/drivers/i3c/master/mipi-i3c-hci/dma.c @@ -554,7 +554,7 @@ static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, if (ring_status & RING_STATUS_RUNNING) { /* stop the ring */ reinit_completion(&rh->op_done); - rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_ABORT); + rh_reg_write(RING_CONTROL, rh_reg_read(RING_CONTROL) | RING_CTRL_ABORT); wait_for_completion_timeout(&rh->op_done, HZ); ring_status = rh_reg_read(RING_STATUS); if (ring_status & RING_STATUS_RUNNING) { @@ -783,8 +783,11 @@ static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) /* determine who this is for */ dev = i3c_hci_addr_to_dev(hci, ibi_addr); if (!dev) { - dev_err(&hci->master.dev, - "IBI for unknown device %#x\n", ibi_addr); + /* + * Either an IBI received just before IBI's were disabled, or + * the controller is broken. Assume the former. + */ + dev_dbg(&hci->master.dev, "IBI when not enabled at address %#x\n", ibi_addr); goto done; } diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h index f17f43494c1b5..b623e4c22c6c9 100644 --- a/drivers/i3c/master/mipi-i3c-hci/hci.h +++ b/drivers/i3c/master/mipi-i3c-hci/hci.h @@ -60,6 +60,7 @@ struct i3c_hci { unsigned int DAT_entry_size; void *DAT_data; struct dat_words *DAT; + struct i3c_dev_desc **ibi_devs; unsigned int DCT_entries; unsigned int DCT_entry_size; u8 version_major; diff --git a/drivers/i3c/master/mipi-i3c-hci/ibi.h b/drivers/i3c/master/mipi-i3c-hci/ibi.h index e1f98e264da0e..073ca67b7d04d 100644 --- a/drivers/i3c/master/mipi-i3c-hci/ibi.h +++ b/drivers/i3c/master/mipi-i3c-hci/ibi.h @@ -26,17 +26,6 @@ #define IBI_DATA_LENGTH GENMASK(7, 0) /* handy helpers */ -static inline struct i3c_dev_desc * -i3c_hci_addr_to_dev(struct i3c_hci *hci, unsigned int addr) -{ - struct i3c_bus *bus = i3c_master_get_bus(&hci->master); - struct i3c_dev_desc *dev; - - i3c_bus_for_each_i3cdev(bus, dev) { - if (dev->info.dyn_addr == addr) - return dev; - } - return NULL; -} +struct i3c_dev_desc *i3c_hci_addr_to_dev(struct i3c_hci *hci, unsigned int addr); #endif diff --git a/drivers/i3c/master/mipi-i3c-hci/pio.c b/drivers/i3c/master/mipi-i3c-hci/pio.c index 8f48a81e65abe..c99c85693214c 100644 --- a/drivers/i3c/master/mipi-i3c-hci/pio.c +++ b/drivers/i3c/master/mipi-i3c-hci/pio.c @@ -863,8 +863,11 @@ static bool hci_pio_prep_new_ibi(struct i3c_hci *hci, struct hci_pio_data *pio) dev = i3c_hci_addr_to_dev(hci, ibi->addr); if (!dev) { - dev_err(&hci->master.dev, - "IBI for unknown device %#x\n", ibi->addr); + /* + * Either an IBI received just before IBI's were disabled, or + * the controller is broken. Assume the former. + */ + dev_dbg(&hci->master.dev, "IBI when not enabled at address %#x\n", ibi->addr); return true; } diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index e2d99a3ac07da..893710d05a2e8 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -208,7 +208,6 @@ struct svc_i3c_drvdata { * @free_slots: Bit array of available slots * @addrs: Array containing the dynamic addresses of each attached device * @descs: Array of descriptors, one per attached device - * @hj_work: Hot-join work * @irq: Main interrupt * @num_clks: I3C clock number * @fclk: Fast clock (bus) @@ -235,7 +234,6 @@ struct svc_i3c_master { u32 free_slots; u8 addrs[SVC_I3C_MAX_DEVS]; struct i3c_dev_desc *descs[SVC_I3C_MAX_DEVS]; - struct work_struct hj_work; int irq; int num_clks; struct clk *fclk; @@ -366,14 +364,6 @@ to_svc_i3c_master(struct i3c_master_controller *master) return container_of(master, struct svc_i3c_master, base); } -static void svc_i3c_master_hj_work(struct work_struct *work) -{ - struct svc_i3c_master *master; - - master = container_of(work, struct svc_i3c_master, hj_work); - i3c_master_do_daa(&master->base); -} - static struct i3c_dev_desc * svc_i3c_master_dev_from_addr(struct svc_i3c_master *master, unsigned int ibiaddr) @@ -651,7 +641,7 @@ static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN: svc_i3c_master_emit_stop(master); if (is_events_enabled(master, SVC_I3C_EVENT_HOTJOIN)) - queue_work(master->base.wq, &master->hj_work); + i3c_master_queue_hotjoin(&master->base); break; case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST: svc_i3c_master_emit_stop(master); @@ -672,10 +662,18 @@ static irqreturn_t svc_i3c_master_irq_handler(int irq, void *dev_id) /* Clear the interrupt status */ writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS); - /* Ignore the false event */ - if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART) && - !SVC_I3C_MSTATUS_STATE_SLVREQ(active)) - return IRQ_HANDLED; + if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART)) { + /* + * Re-read MSTATUS to obtain the latest state and avoid + * missing an IBI that arrives after MSTATUS is latched + * but before SLVSTART is cleared. + */ + active = readl(master->regs + SVC_I3C_MSTATUS); + + /* Ignore the false event */ + if (!SVC_I3C_MSTATUS_STATE_SLVREQ(active)) + return IRQ_HANDLED; + } /* * The SDA line remains low until the request is processed. @@ -2022,7 +2020,6 @@ static int svc_i3c_master_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "can't enable I3C clocks\n"); - INIT_WORK(&master->hj_work, svc_i3c_master_hj_work); mutex_init(&master->lock); ret = devm_request_irq(dev, master->irq, svc_i3c_master_irq_handler, @@ -2081,7 +2078,6 @@ static void svc_i3c_master_remove(struct platform_device *pdev) { struct svc_i3c_master *master = platform_get_drvdata(pdev); - cancel_work_sync(&master->hj_work); i3c_master_unregister(&master->base); pm_runtime_dont_use_autosuspend(&pdev->dev); diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c index 2398eb7e12cdd..dc8a6285cf3df 100644 --- a/drivers/iio/accel/bmc150-accel-core.c +++ b/drivers/iio/accel/bmc150-accel-core.c @@ -991,6 +991,8 @@ static int __bmc150_accel_fifo_flush(struct iio_dev *indio_dev, if (samples && count > samples) count = samples; + count = min_t(u8, count, BMC150_ACCEL_FIFO_LENGTH); + ret = bmc150_accel_fifo_transfer(data, (u8 *)buffer, count); if (ret) return ret; diff --git a/drivers/iio/accel/kxsd9.c b/drivers/iio/accel/kxsd9.c index 4717d80fc24af..7ac885d94d7f4 100644 --- a/drivers/iio/accel/kxsd9.c +++ b/drivers/iio/accel/kxsd9.c @@ -147,8 +147,9 @@ static int kxsd9_write_raw(struct iio_dev *indio_dev, if (mask == IIO_CHAN_INFO_SCALE) { /* Check no integer component */ if (val) - return -EINVAL; - ret = kxsd9_write_scale(indio_dev, val2); + ret = -EINVAL; + else + ret = kxsd9_write_scale(indio_dev, val2); } pm_runtime_put_autosuspend(st->dev); diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c index 15172ba2972c5..cefc7cf4bd835 100644 --- a/drivers/iio/accel/mma8452.c +++ b/drivers/iio/accel/mma8452.c @@ -252,6 +252,8 @@ static int mma8452_read(struct mma8452_data *data, __be16 buf[3]) ret = i2c_smbus_read_i2c_block_data(data->client, MMA8452_OUT_X, 3 * sizeof(__be16), (u8 *)buf); + if (ret < 0) + return ret; ret = mma8452_set_runtime_pm_state(data->client, false); diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index a9dedbb8eb461..3a3dcba3db3b0 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -78,6 +78,7 @@ config AD4030 config AD4062 tristate "Analog Devices AD4062 Driver" depends on I3C + depends on GPIOLIB select REGMAP_I3C select IIO_BUFFER select IIO_TRIGGERED_BUFFER @@ -311,6 +312,7 @@ config AD7298 config AD7380 tristate "Analog Devices AD7380 ADC driver" depends on SPI_MASTER + select REGMAP select SPI_OFFLOAD select IIO_BUFFER select IIO_BUFFER_DMAENGINE @@ -414,6 +416,7 @@ config AD7766 config AD7768_1 tristate "Analog Devices AD7768-1 ADC driver" depends on SPI + select GPIOLIB select REGULATOR select REGMAP_SPI select RATIONAL @@ -434,6 +437,7 @@ config AD7779 depends on SPI select CRC8 select IIO_BUFFER + select IIO_TRIGGERED_BUFFER select IIO_BACKEND help Say yes here to build support for Analog Devices AD777X family diff --git a/drivers/iio/adc/ad_sigma_delta.c b/drivers/iio/adc/ad_sigma_delta.c index a955556f9ec88..1b410291da537 100644 --- a/drivers/iio/adc/ad_sigma_delta.c +++ b/drivers/iio/adc/ad_sigma_delta.c @@ -262,11 +262,25 @@ static int ad_sigma_delta_clear_pending_event(struct ad_sigma_delta *sigma_delta /* * Read RÌ…DÌ…YÌ… pin (if possible) or status register to check if there is an - * old event. + * old event. For devices with neither an RDY GPIO nor registers, + * ad_sd_read_reg() transmits no address byte and clocks raw MISO bytes, + * which is indistinguishable from reading conversion data and would + * partially consume a pending result. Skip the check for such devices. + * + * This is safe for all current registerless devices: ad7191 and ad7780 + * (with powerdown GPIO) are reset between conversions by CS deassertion, + * so there is no stale result to drain; ad7780 (without powerdown GPIO) + * and max11205 are continuously-converting and cycle ~DRDY at the output + * data rate regardless of whether the previous result was read, so the + * next falling edge fires naturally. + * + * A future registerless device that holds ~DRDY asserted until data is + * read would be broken by this early return and would need either + * num_resetclks set or a rdy-gpio. */ if (sigma_delta->rdy_gpiod) { pending_event = gpiod_get_value(sigma_delta->rdy_gpiod); - } else { + } else if (sigma_delta->info->has_registers) { unsigned int status_reg; ret = ad_sd_read_reg(sigma_delta, AD_SD_REG_STATUS, 1, &status_reg); @@ -274,11 +288,24 @@ static int ad_sigma_delta_clear_pending_event(struct ad_sigma_delta *sigma_delta return ret; pending_event = !(status_reg & AD_SD_REG_STATUS_RDY); + } else { + return 0; } if (!pending_event) return 0; + /* + * With num_resetclks = 0, data_read_len is 0 and the drain sequence + * below would compute memset(data + 2, 0xff, 0 - 1), underflowing to + * SIZE_MAX and corrupting the heap. There is no safe way to drain the + * stale result without knowing the data register size; it will be + * consumed by the first ad_sd_read_reg() call in + * ad_sigma_delta_single_conversion(). + */ + if (!data_read_len) + return 0; + /* * In general the size of the data register is unknown. It varies from * device to device, might be one byte longer if CONTROL.DATA_STATUS is @@ -441,11 +468,10 @@ int ad_sigma_delta_single_conversion(struct iio_dev *indio_dev, out: ad_sd_disable_irq(sigma_delta); - ad_sigma_delta_set_mode(sigma_delta, AD_SD_MODE_IDLE); - ad_sigma_delta_disable_one(sigma_delta, chan->address); - out_unlock: sigma_delta->keep_cs_asserted = false; + ad_sigma_delta_set_mode(sigma_delta, AD_SD_MODE_IDLE); + ad_sigma_delta_disable_one(sigma_delta, chan->address); sigma_delta->bus_locked = false; spi_bus_unlock(sigma_delta->spi->controller); out_release: @@ -578,6 +604,9 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) return 0; err_unlock: + sigma_delta->keep_cs_asserted = false; + ad_sigma_delta_set_mode(sigma_delta, AD_SD_MODE_IDLE); + sigma_delta->bus_locked = false; spi_bus_unlock(sigma_delta->spi->controller); spi_unoptimize_message(&sigma_delta->sample_msg); diff --git a/drivers/iio/adc/lpc32xx_adc.c b/drivers/iio/adc/lpc32xx_adc.c index 43a7bc8158b57..db3a602327ffe 100644 --- a/drivers/iio/adc/lpc32xx_adc.c +++ b/drivers/iio/adc/lpc32xx_adc.c @@ -179,6 +179,8 @@ static int lpc32xx_adc_probe(struct platform_device *pdev) if (irq < 0) return irq; + init_completion(&st->completion); + retval = devm_request_irq(&pdev->dev, irq, lpc32xx_adc_isr, 0, LPC32XXAD_NAME, st); if (retval < 0) { @@ -197,8 +199,6 @@ static int lpc32xx_adc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, iodev); - init_completion(&st->completion); - iodev->name = LPC32XXAD_NAME; iodev->info = &lpc32xx_adc_iio_info; iodev->modes = INDIO_DIRECT_MODE; diff --git a/drivers/iio/adc/nxp-sar-adc.c b/drivers/iio/adc/nxp-sar-adc.c index 8f4ed3db94f07..753f6a0a6e7bb 100644 --- a/drivers/iio/adc/nxp-sar-adc.c +++ b/drivers/iio/adc/nxp-sar-adc.c @@ -198,13 +198,13 @@ static void nxp_sar_adc_irq_cfg(struct nxp_sar_adc *info, bool enable) writel(0, NXP_SAR_ADC_IMR(info->regs)); } -static void nxp_sar_adc_wait_for(struct nxp_sar_adc *info, unsigned int cycles) +static void nxp_sar_adc_wait_for(struct nxp_sar_adc *info, u64 cycles) { u64 rate; rate = clk_get_rate(info->clk); if (rate) - ndelay(div64_u64(NSEC_PER_SEC, rate * cycles)); + ndelay(div64_u64(NSEC_PER_SEC * cycles, rate)); } static bool nxp_sar_adc_set_enabled(struct nxp_sar_adc *info, bool enable) diff --git a/drivers/iio/adc/spear_adc.c b/drivers/iio/adc/spear_adc.c index 91995489bb1c6..d47278f1c4fc3 100644 --- a/drivers/iio/adc/spear_adc.c +++ b/drivers/iio/adc/spear_adc.c @@ -280,6 +280,7 @@ static int spear_adc_probe(struct platform_device *pdev) st = iio_priv(indio_dev); st->dev = dev; + init_completion(&st->completion); mutex_init(&st->lock); /* @@ -326,8 +327,6 @@ static int spear_adc_probe(struct platform_device *pdev) spear_adc_configure(st); - init_completion(&st->completion); - indio_dev->name = SPEAR_ADC_MOD_NAME; indio_dev->info = &spear_adc_info; indio_dev->modes = INDIO_DIRECT_MODE; diff --git a/drivers/iio/adc/ti-ads1119.c b/drivers/iio/adc/ti-ads1119.c index 2320be0efb508..82040f9395b07 100644 --- a/drivers/iio/adc/ti-ads1119.c +++ b/drivers/iio/adc/ti-ads1119.c @@ -459,7 +459,11 @@ static int ads1119_triggered_buffer_preenable(struct iio_dev *indio_dev) if (ret) return ret; - return i2c_smbus_write_byte(st->client, ADS1119_CMD_START_SYNC); + ret = i2c_smbus_write_byte(st->client, ADS1119_CMD_START_SYNC); + if (ret) + pm_runtime_put_autosuspend(dev); + + return ret; } static int ads1119_triggered_buffer_postdisable(struct iio_dev *indio_dev) diff --git a/drivers/iio/adc/ti-ads124s08.c b/drivers/iio/adc/ti-ads124s08.c index 8ea1269f74db0..57eed8554bd92 100644 --- a/drivers/iio/adc/ti-ads124s08.c +++ b/drivers/iio/adc/ti-ads124s08.c @@ -321,7 +321,8 @@ static int ads124s_probe(struct spi_device *spi) ads124s_priv->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(ads124s_priv->reset_gpio)) - dev_info(&spi->dev, "Reset GPIO not defined\n"); + return dev_err_probe(&spi->dev, PTR_ERR(ads124s_priv->reset_gpio), + "Failed to get reset GPIO\n"); ads124s_priv->chip_info = &ads124s_chip_info_tbl[spi_id->driver_data]; diff --git a/drivers/iio/adc/ti-ads1298.c b/drivers/iio/adc/ti-ads1298.c index ae30b47e45142..731792f06993a 100644 --- a/drivers/iio/adc/ti-ads1298.c +++ b/drivers/iio/adc/ti-ads1298.c @@ -279,6 +279,7 @@ static const u8 ads1298_pga_settings[] = { 6, 1, 2, 3, 4, 8, 12 }; static int ads1298_get_scale(struct ads1298_private *priv, int channel, int *val, int *val2) { + unsigned int pga_idx; int ret; unsigned int regval; u8 gain; @@ -302,7 +303,11 @@ static int ads1298_get_scale(struct ads1298_private *priv, if (ret) return ret; - gain = ads1298_pga_settings[FIELD_GET(ADS1298_MASK_CH_PGA, regval)]; + pga_idx = FIELD_GET(ADS1298_MASK_CH_PGA, regval); + if (pga_idx >= ARRAY_SIZE(ads1298_pga_settings)) + return -EINVAL; + + gain = ads1298_pga_settings[pga_idx]; *val /= gain; /* Full scale is VREF / gain */ *val2 = ADS1298_BITS_PER_SAMPLE - 1; /* Signed, hence the -1 */ diff --git a/drivers/iio/adc/xilinx-ams.c b/drivers/iio/adc/xilinx-ams.c index 124470c925297..6191cd1b29a51 100644 --- a/drivers/iio/adc/xilinx-ams.c +++ b/drivers/iio/adc/xilinx-ams.c @@ -871,6 +871,9 @@ static const struct iio_chan_spec *ams_event_to_channel(struct iio_dev *dev, if (dev->channels[i].scan_index == scan_index) break; + if (i == dev->num_channels) + return NULL; + return &dev->channels[i]; } @@ -1012,6 +1015,8 @@ static void ams_handle_event(struct iio_dev *indio_dev, u32 event) const struct iio_chan_spec *chan; chan = ams_event_to_channel(indio_dev, event); + if (!chan) + return; if (chan->type == IIO_TEMP) { /* diff --git a/drivers/iio/buffer/industrialio-hw-consumer.c b/drivers/iio/buffer/industrialio-hw-consumer.c index 700528c9a0a46..10e912bbf0c5f 100644 --- a/drivers/iio/buffer/industrialio-hw-consumer.c +++ b/drivers/iio/buffer/industrialio-hw-consumer.c @@ -40,6 +40,8 @@ static void iio_hw_buf_release(struct iio_buffer *buffer) { struct hw_consumer_buffer *hw_buf = iio_buffer_to_hw_consumer_buffer(buffer); + + bitmap_free(buffer->scan_mask); kfree(hw_buf); } diff --git a/drivers/iio/chemical/scd30_core.c b/drivers/iio/chemical/scd30_core.c index 11d6bc1b63e67..d02f2de42327d 100644 --- a/drivers/iio/chemical/scd30_core.c +++ b/drivers/iio/chemical/scd30_core.c @@ -4,6 +4,8 @@ * * Copyright (c) 2020 Tomasz Duszynski */ + +#include #include #include #include @@ -43,6 +45,11 @@ #define SCD30_TEMP_OFFSET_MAX 655360 #define SCD30_EXTRA_TIMEOUT_PER_S 250 +/* Floating point arithmetic macros */ +#define SCD30_FLOAT_MANTISSA_MSK GENMASK(22, 0) +#define SCD30_FLOAT_EXP_MSK GENMASK(30, 23) +#define SCD30_FLOAT_SIGN_MSK BIT(31) + enum { SCD30_CONC, SCD30_TEMP, @@ -89,10 +96,14 @@ static int scd30_reset(struct scd30_state *state) /* simplified float to fixed point conversion with a scaling factor of 0.01 */ static int scd30_float_to_fp(int float32) { - int fraction, shift, - mantissa = float32 & GENMASK(22, 0), - sign = (float32 & BIT(31)) ? -1 : 1, - exp = (float32 & ~BIT(31)) >> 23; + int fraction, shift, sign; + int mantissa = FIELD_GET(SCD30_FLOAT_MANTISSA_MSK, float32); + int exp = FIELD_GET(SCD30_FLOAT_EXP_MSK, float32); + + if (float32 & SCD30_FLOAT_SIGN_MSK) + sign = -1; + else + sign = 1; /* special case 0 */ if (!exp && !mantissa) diff --git a/drivers/iio/common/st_sensors/st_sensors_core.c b/drivers/iio/common/st_sensors/st_sensors_core.c index dbc5e16fbde43..76f91696f66a3 100644 --- a/drivers/iio/common/st_sensors/st_sensors_core.c +++ b/drivers/iio/common/st_sensors/st_sensors_core.c @@ -498,6 +498,7 @@ static int st_sensors_read_axis_data(struct iio_dev *indio_dev, u8 *outdata; struct st_sensor_data *sdata = iio_priv(indio_dev); unsigned int byte_for_channel; + u32 tmp; byte_for_channel = DIV_ROUND_UP(ch->scan_type.realbits + ch->scan_type.shift, 8); @@ -508,12 +509,22 @@ static int st_sensors_read_axis_data(struct iio_dev *indio_dev, if (err < 0) return err; - if (byte_for_channel == 1) - *data = (s8)*outdata; - else if (byte_for_channel == 2) - *data = (s16)get_unaligned_le16(outdata); - else if (byte_for_channel == 3) - *data = (s32)sign_extend32(get_unaligned_le24(outdata), 23); + if (byte_for_channel == 1) { + tmp = *outdata; + } else if (byte_for_channel == 2) { + if (ch->scan_type.endianness == IIO_BE) + tmp = get_unaligned_be16(outdata); + else + tmp = get_unaligned_le16(outdata); + } else if (byte_for_channel == 3) { + if (ch->scan_type.endianness == IIO_BE) + tmp = get_unaligned_be24(outdata); + else + tmp = get_unaligned_le24(outdata); + } else { + return -EINVAL; + } + *data = sign_extend32(tmp, BYTES_TO_BITS(byte_for_channel) - 1); return 0; } diff --git a/drivers/iio/dac/ad3552r-hs.c b/drivers/iio/dac/ad3552r-hs.c index a9578afa7015d..6bc64f53bce91 100644 --- a/drivers/iio/dac/ad3552r-hs.c +++ b/drivers/iio/dac/ad3552r-hs.c @@ -549,7 +549,7 @@ static ssize_t ad3552r_hs_write_data_source(struct file *f, guard(mutex)(&st->lock); - if (count >= sizeof(buf)) + if (*ppos != 0 || count >= sizeof(buf)) return -ENOSPC; ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, userbuf, diff --git a/drivers/iio/dac/mcp47feb02.c b/drivers/iio/dac/mcp47feb02.c index faccb804a5ed5..5f84b46ec6f3d 100644 --- a/drivers/iio/dac/mcp47feb02.c +++ b/drivers/iio/dac/mcp47feb02.c @@ -1136,26 +1136,33 @@ static int mcp47feb02_probe(struct i2c_client *client) vdd_uV = ret; - ret = devm_regulator_get_enable_read_voltage(dev, "vref"); - if (ret > 0) { - vref_uV = ret; + if (device_property_present(dev, "vref-supply")) { + vref_uV = devm_regulator_get_enable_read_voltage(dev, "vref"); + if (vref_uV < 0) + return vref_uV; + + if (vref_uV == 0) + return dev_err_probe(dev, -EINVAL, "Vref is 0 uV.\n"); + data->use_vref = true; } else { vref_uV = 0; - dev_dbg(dev, "using internal band gap as voltage reference.\n"); - dev_dbg(dev, "Vref is unavailable.\n"); + dev_dbg(dev, "Using internal band gap as voltage reference.\n"); } - if (chip_features->have_ext_vref1) { - ret = devm_regulator_get_enable_read_voltage(dev, "vref1"); - if (ret > 0) { - vref1_uV = ret; - data->use_vref1 = true; - } else { - vref1_uV = 0; - dev_dbg(dev, "using internal band gap as voltage reference 1.\n"); - dev_dbg(dev, "Vref1 is unavailable.\n"); - } + if (chip_features->have_ext_vref1 && + device_property_present(dev, "vref1-supply")) { + vref1_uV = devm_regulator_get_enable_read_voltage(dev, "vref1"); + if (vref1_uV < 0) + return vref1_uV; + + if (vref1_uV == 0) + return dev_err_probe(dev, -EINVAL, "Vref1 is 0 uV.\n"); + + data->use_vref1 = true; + } else { + vref1_uV = 0; + dev_dbg(dev, "Using internal band gap as voltage reference 1.\n"); } ret = mcp47feb02_init_ctrl_regs(data); diff --git a/drivers/iio/gyro/bmg160_core.c b/drivers/iio/gyro/bmg160_core.c index 38394b5f32755..d611341a0e2ad 100644 --- a/drivers/iio/gyro/bmg160_core.c +++ b/drivers/iio/gyro/bmg160_core.c @@ -201,6 +201,9 @@ static int bmg160_get_filter(struct bmg160_data *data, int *val) break; } + if (i == ARRAY_SIZE(bmg160_samp_freq_table)) + return -EINVAL; + *val = bmg160_samp_freq_table[i].filter; return ret ? ret : IIO_VAL_INT; @@ -218,6 +221,9 @@ static int bmg160_set_filter(struct bmg160_data *data, int val) break; } + if (i == ARRAY_SIZE(bmg160_samp_freq_table)) + return -EINVAL; + ret = regmap_write(data->regmap, BMG160_REG_PMU_BW, bmg160_samp_freq_table[i].bw_bits); if (ret < 0) { @@ -258,8 +264,14 @@ static int bmg160_chip_init(struct bmg160_data *data) if (ret < 0) return ret; - /* Wait upto 500 ms to be ready after changing mode */ - usleep_range(500, 1000); + /* + * Wait for the chip to be ready after switching to normal mode. + * The BMG160 datasheet (BST-BMG160-DS000-07 Rev. 1.0, May 2013) + * specifies a start-up / wake-up time (tsu, twusm) of 30 ms; use + * BMG160_MAX_STARTUP_TIME_MS (80 ms) as a safety margin, matching + * what bmg160_runtime_resume() already does. + */ + msleep(BMG160_MAX_STARTUP_TIME_MS); /* Set Bandwidth */ ret = bmg160_set_bw(data, BMG160_DEF_BW); diff --git a/drivers/iio/imu/adis_trigger.c b/drivers/iio/imu/adis_trigger.c index d76e13cbac680..3e6a7af6ab017 100644 --- a/drivers/iio/imu/adis_trigger.c +++ b/drivers/iio/imu/adis_trigger.c @@ -94,7 +94,7 @@ int devm_adis_probe_trigger(struct adis *adis, struct iio_dev *indio_dev) else ret = devm_request_irq(&adis->spi->dev, adis->spi->irq, &iio_trigger_generic_data_rdy_poll, - adis->irq_flag, + adis->irq_flag | IRQF_NO_THREAD, indio_dev->name, adis->trig); if (ret) diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c index 4abb83b75e2e8..86f6ecfd64aa2 100644 --- a/drivers/iio/imu/bmi160/bmi160_core.c +++ b/drivers/iio/imu/bmi160/bmi160_core.c @@ -788,7 +788,8 @@ int bmi160_probe_trigger(struct iio_dev *indio_dev, int irq, u32 irq_type) ret = devm_request_irq(&indio_dev->dev, irq, &iio_trigger_generic_data_rdy_poll, - irq_type, "bmi160", data->trig); + irq_type | IRQF_NO_THREAD, + "bmi160", data->trig); if (ret) return ret; diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c index 532d5fdffaf8d..7df920ef3cf08 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c @@ -1170,10 +1170,10 @@ struct iio_dev *inv_icm42600_accel_init(struct inv_icm42600_state *st) accel_st->filter = INV_ICM42600_FILTER_AVG_16X; /* - * clock period is 32kHz (31250ns) + * clock period is 8kHz (125000ns) * jitter is +/- 2% (20 per mille) */ - ts_chip.clock_period = 31250; + ts_chip.clock_period = 125000; ts_chip.jitter = 20; ts_chip.init_period = inv_icm42600_odr_to_period(st->conf.accel.odr); inv_sensors_timestamp_init(&accel_st->ts, &ts_chip); diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c index 68a3957580318..5c3840acf0857 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c @@ -248,6 +248,7 @@ int inv_icm42600_buffer_update_watermark(struct inv_icm42600_state *st) /* compute watermark value in bytes */ wm_size = watermark * packet_size; + st->fifo.watermark.value = watermark; /* changing FIFO watermark requires to turn off watermark interrupt */ ret = regmap_update_bits_check(st->map, INV_ICM42600_REG_INT_SOURCE0, @@ -454,11 +455,10 @@ int inv_icm42600_buffer_fifo_read(struct inv_icm42600_state *st, st->fifo.nb.accel = 0; st->fifo.nb.total = 0; - /* compute maximum FIFO read size */ + /* compute maximum FIFO read size (watermark for max = 0 interrupt case) */ if (max == 0) - max_count = sizeof(st->fifo.data); - else - max_count = max * inv_icm42600_get_packet_size(st->fifo.en); + max = st->fifo.watermark.value; + max_count = max * inv_icm42600_get_packet_size(st->fifo.en); /* read FIFO count value */ raw_fifo_count = (__be16 *)st->buffer; @@ -574,6 +574,7 @@ int inv_icm42600_buffer_init(struct inv_icm42600_state *st) st->fifo.watermark.eff_gyro = 1; st->fifo.watermark.eff_accel = 1; + st->fifo.watermark.value = 1; /* * Default FIFO configuration (bits 7 to 5) diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h index ffca4da1e2493..88b8b9f780af3 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h @@ -34,6 +34,7 @@ struct inv_icm42600_fifo { unsigned int accel; unsigned int eff_gyro; unsigned int eff_accel; + unsigned int value; } watermark; size_t count; struct { diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c index 11339ddf1da36..a18dcac939298 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c @@ -755,10 +755,10 @@ struct iio_dev *inv_icm42600_gyro_init(struct inv_icm42600_state *st) } /* - * clock period is 32kHz (31250ns) + * clock period is 8kHz (125000ns) * jitter is +/- 2% (20 per mille) */ - ts_chip.clock_period = 31250; + ts_chip.clock_period = 125000; ts_chip.jitter = 20; ts_chip.init_period = inv_icm42600_odr_to_period(st->conf.accel.odr); inv_sensors_timestamp_init(&gyro_st->ts, &ts_chip); diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c index 450cb5b47346e..8261d9cf0d940 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c @@ -1660,6 +1660,26 @@ static int st_lsm6dsx_check_whoami(struct st_lsm6dsx_hw *hw, int id, return -ENODEV; } + hw->settings = &st_lsm6dsx_sensor_settings[i]; + + if (hw->settings->shub_settings.page_mux.addr) { + /* + * If the IMU has the shub page selected on init, for example + * after a CPU watchdog reset while the page is selected, the + * regular register space is shadowed. While the regular + * register space is shadowed, the registers needed for + * initializing the IMU are not available. + * + * Unconditionally clear the shub page selection to ensure + * normal register access. + */ + err = st_lsm6dsx_set_page(hw, false); + if (err < 0) { + dev_err(hw->dev, "failed to clear shub page\n"); + return err; + } + } + err = regmap_read(hw->regmap, ST_LSM6DSX_REG_WHOAMI_ADDR, &data); if (err < 0) { dev_err(hw->dev, "failed to read whoami register\n"); @@ -1672,7 +1692,6 @@ static int st_lsm6dsx_check_whoami(struct st_lsm6dsx_hw *hw, int id, } *name = st_lsm6dsx_sensor_settings[i].id[j].name; - hw->settings = &st_lsm6dsx_sensor_settings[i]; return 0; } diff --git a/drivers/iio/industrialio-backend.c b/drivers/iio/industrialio-backend.c index 10e689f494419..cd697fbeae47b 100644 --- a/drivers/iio/industrialio-backend.c +++ b/drivers/iio/industrialio-backend.c @@ -156,7 +156,7 @@ static ssize_t iio_backend_debugfs_write_reg(struct file *file, ssize_t rc; int ret; - if (count >= sizeof(buf)) + if (*ppos != 0 || count >= sizeof(buf)) return -ENOSPC; rc = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, userbuf, count); diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 22eefd048ba92..15b56f8972fc9 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -418,7 +418,7 @@ static ssize_t iio_debugfs_write_reg(struct file *file, char buf[80]; int ret; - if (count >= sizeof(buf)) + if (*ppos != 0 || count >= sizeof(buf)) return -EINVAL; ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, userbuf, diff --git a/drivers/iio/industrialio-event.c b/drivers/iio/industrialio-event.c index a0d6fcf2a9c9b..e6730f52262a4 100644 --- a/drivers/iio/industrialio-event.c +++ b/drivers/iio/industrialio-event.c @@ -207,6 +207,8 @@ static int iio_event_getfd(struct iio_dev *indio_dev) goto unlock; } + kfifo_reset_out(&ev_int->det_events); + iio_device_get(indio_dev); fd = anon_inode_getfd("iio:event", &iio_event_chrdev_fileops, @@ -214,10 +216,7 @@ static int iio_event_getfd(struct iio_dev *indio_dev) if (fd < 0) { clear_bit(IIO_BUSY_BIT_POS, &ev_int->flags); iio_device_put(indio_dev); - } else { - kfifo_reset_out(&ev_int->det_events); } - unlock: mutex_unlock(&iio_dev_opaque->mlock); return fd; diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig index eff33e456c70d..a28f308bc14ec 100644 --- a/drivers/iio/light/Kconfig +++ b/drivers/iio/light/Kconfig @@ -45,6 +45,7 @@ config ADUX1020 config AL3000A tristate "AL3000a ambient light sensor" + select REGMAP_I2C depends on I2C help Say Y here if you want to build a driver for the Dyna Image AL3000a @@ -55,6 +56,7 @@ config AL3000A config AL3010 tristate "AL3010 ambient light sensor" + select REGMAP_I2C depends on I2C help Say Y here if you want to build a driver for the Dyna Image AL3010 @@ -65,6 +67,7 @@ config AL3010 config AL3320A tristate "AL3320A ambient light sensor" + select REGMAP_I2C depends on I2C help Say Y here if you want to build a driver for the Dyna Image AL3320A diff --git a/drivers/iio/light/acpi-als.c b/drivers/iio/light/acpi-als.c index ab229318dce94..1983a7f17aa93 100644 --- a/drivers/iio/light/acpi-als.c +++ b/drivers/iio/light/acpi-als.c @@ -179,11 +179,15 @@ static irqreturn_t acpi_als_trigger_handler(int irq, void *p) static int acpi_als_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct acpi_device *device = ACPI_COMPANION(dev); + struct acpi_device *device; struct iio_dev *indio_dev; struct acpi_als *als; int ret; + device = ACPI_COMPANION(dev); + if (!device) + return -ENODEV; + indio_dev = devm_iio_device_alloc(dev, sizeof(*als)); if (!indio_dev) return -ENOMEM; diff --git a/drivers/iio/light/al3010.c b/drivers/iio/light/al3010.c index 0932fa2b49fa7..0e72906c637f0 100644 --- a/drivers/iio/light/al3010.c +++ b/drivers/iio/light/al3010.c @@ -42,7 +42,7 @@ enum al3xxxx_range { }; static const int al3010_scales[][2] = { - {0, 1187200}, {0, 296800}, {0, 74200}, {0, 18600} + { 1, 187200 }, { 0, 296800 }, { 0, 74200 }, { 0, 18600 }, }; static const struct regmap_config al3010_regmap_config = { @@ -111,7 +111,8 @@ static int al3010_read_raw(struct iio_dev *indio_dev, int *val2, long mask) { struct al3010_data *data = iio_priv(indio_dev); - int ret, gain, raw; + int ret, gain; + __le16 raw; switch (mask) { case IIO_CHAN_INFO_RAW: @@ -120,11 +121,12 @@ static int al3010_read_raw(struct iio_dev *indio_dev, * - low byte of output is stored at AL3010_REG_DATA_LOW * - high byte of output is stored at AL3010_REG_DATA_LOW + 1 */ - ret = regmap_read(data->regmap, AL3010_REG_DATA_LOW, &raw); + ret = regmap_bulk_read(data->regmap, AL3010_REG_DATA_LOW, + &raw, sizeof(raw)); if (ret) return ret; - *val = raw; + *val = le16_to_cpu(raw); return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: diff --git a/drivers/iio/light/al3320a.c b/drivers/iio/light/al3320a.c index 63f5a85912fc9..b9076f4344179 100644 --- a/drivers/iio/light/al3320a.c +++ b/drivers/iio/light/al3320a.c @@ -135,7 +135,8 @@ static int al3320a_read_raw(struct iio_dev *indio_dev, int *val2, long mask) { struct al3320a_data *data = iio_priv(indio_dev); - int ret, gain, raw; + int ret, gain; + __le16 raw; switch (mask) { case IIO_CHAN_INFO_RAW: @@ -144,11 +145,12 @@ static int al3320a_read_raw(struct iio_dev *indio_dev, * - low byte of output is stored at AL3320A_REG_DATA_LOW * - high byte of output is stored at AL3320A_REG_DATA_LOW + 1 */ - ret = regmap_read(data->regmap, AL3320A_REG_DATA_LOW, &raw); + ret = regmap_bulk_read(data->regmap, AL3320A_REG_DATA_LOW, + &raw, sizeof(raw)); if (ret) return ret; - *val = raw; + *val = le16_to_cpu(raw); return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c index a0d8a58f27042..886accf5e8591 100644 --- a/drivers/iio/light/gp2ap002.c +++ b/drivers/iio/light/gp2ap002.c @@ -258,7 +258,7 @@ static int gp2ap002_read_raw(struct iio_dev *indio_dev, case IIO_LIGHT: ret = gp2ap002_get_lux(gp2ap002); if (ret < 0) - return ret; + goto out; *val = ret; ret = IIO_VAL_INT; goto out; diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c index 53bc455b7bad1..0d35d23da091e 100644 --- a/drivers/iio/light/opt3001.c +++ b/drivers/iio/light/opt3001.c @@ -366,8 +366,10 @@ static int opt3001_get_processed(struct opt3001 *opt, int *val, int *val2) ret = wait_event_timeout(opt->result_ready_queue, opt->result_ready, msecs_to_jiffies(OPT3001_RESULT_READY_LONG)); - if (ret == 0) - return -ETIMEDOUT; + if (ret == 0) { + ret = -ETIMEDOUT; + goto err; + } } else { /* Sleep for result ready time */ timeout = (opt->int_time == OPT3001_INT_TIME_SHORT) ? diff --git a/drivers/iio/light/si1133.c b/drivers/iio/light/si1133.c index 44fa152dbd24c..bf7bf0f1631d4 100644 --- a/drivers/iio/light/si1133.c +++ b/drivers/iio/light/si1133.c @@ -395,8 +395,14 @@ static int si1133_command(struct si1133_data *data, u8 cmd) expected_seq = (data->rsp_seq + 1) & SI1133_MAX_CMD_CTR; - if (cmd == SI1133_CMD_FORCE) + if (cmd == SI1133_CMD_FORCE) { + /* Flush pending IRQs from a previous timeout. */ + regmap_read(data->regmap, SI1133_REG_IRQ_STATUS, &resp); + regmap_write(data->regmap, SI1133_REG_IRQ_ENABLE, + SI1133_IRQ_CHANNEL_ENABLE); + reinit_completion(&data->completion); + } err = regmap_write(data->regmap, SI1133_REG_COMMAND, cmd); if (err) { @@ -409,6 +415,7 @@ static int si1133_command(struct si1133_data *data, u8 cmd) /* wait for irq */ if (!wait_for_completion_timeout(&data->completion, msecs_to_jiffies(SI1133_COMPLETION_TIMEOUT_MS))) { + regmap_write(data->regmap, SI1133_REG_IRQ_ENABLE, 0); err = -ETIMEDOUT; goto out; } @@ -427,6 +434,11 @@ static int si1133_command(struct si1133_data *data, u8 cmd) dev_warn(dev, "Failed to read command 0x%02x, ret=%d\n", cmd, err); + /* + * Reset counter on err to prevent software and hardware + * counters being out of sync. + */ + si1133_cmd_reset_counter(data); goto out; } } diff --git a/drivers/iio/light/tcs3472.c b/drivers/iio/light/tcs3472.c index 12429a3261b38..849ca7885d719 100644 --- a/drivers/iio/light/tcs3472.c +++ b/drivers/iio/light/tcs3472.c @@ -440,6 +440,23 @@ static const struct iio_info tcs3472_info = { .attrs = &tcs3472_attribute_group, }; +static int tcs3472_powerdown(struct tcs3472_data *data) +{ + int ret; + u8 enable_mask = TCS3472_ENABLE_AEN | TCS3472_ENABLE_PON; + + mutex_lock(&data->lock); + + ret = i2c_smbus_write_byte_data(data->client, TCS3472_ENABLE, + data->enable & ~enable_mask); + if (!ret) + data->enable &= ~enable_mask; + + mutex_unlock(&data->lock); + + return ret; +} + static int tcs3472_probe(struct i2c_client *client) { struct tcs3472_data *data; @@ -513,7 +530,7 @@ static int tcs3472_probe(struct i2c_client *client) ret = iio_triggered_buffer_setup(indio_dev, NULL, tcs3472_trigger_handler, NULL); if (ret < 0) - return ret; + goto error_powerdown; if (client->irq) { ret = request_threaded_irq(client->irq, NULL, @@ -536,23 +553,8 @@ static int tcs3472_probe(struct i2c_client *client) free_irq(client->irq, indio_dev); buffer_cleanup: iio_triggered_buffer_cleanup(indio_dev); - return ret; -} - -static int tcs3472_powerdown(struct tcs3472_data *data) -{ - int ret; - u8 enable_mask = TCS3472_ENABLE_AEN | TCS3472_ENABLE_PON; - - mutex_lock(&data->lock); - - ret = i2c_smbus_write_byte_data(data->client, TCS3472_ENABLE, - data->enable & ~enable_mask); - if (!ret) - data->enable &= ~enable_mask; - - mutex_unlock(&data->lock); - +error_powerdown: + tcs3472_powerdown(data); return ret; } diff --git a/drivers/iio/light/tsl2591.c b/drivers/iio/light/tsl2591.c index c5557867ea43d..c5ccd833dde6f 100644 --- a/drivers/iio/light/tsl2591.c +++ b/drivers/iio/light/tsl2591.c @@ -1137,10 +1137,8 @@ static int tsl2591_probe(struct i2c_client *client) NULL, tsl2591_event_handler, IRQF_TRIGGER_FALLING | IRQF_ONESHOT, "tsl2591_irq", indio_dev); - if (ret) { - dev_err_probe(&client->dev, ret, "IRQ request error\n"); - return -EINVAL; - } + if (ret) + return ret; indio_dev->info = &tsl2591_info; } else { indio_dev->info = &tsl2591_info_no_irq; diff --git a/drivers/iio/light/veml6030.c b/drivers/iio/light/veml6030.c index 6bcacae3863c0..da8c32cabfd68 100644 --- a/drivers/iio/light/veml6030.c +++ b/drivers/iio/light/veml6030.c @@ -875,9 +875,11 @@ static irqreturn_t veml6030_event_handler(int irq, void *private) else evtdir = IIO_EV_DIR_FALLING; - iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE(IIO_INTENSITY, - 0, IIO_EV_TYPE_THRESH, evtdir), - iio_get_time_ns(indio_dev)); + iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE(IIO_LIGHT, + 0, + IIO_EV_TYPE_THRESH, + evtdir), + iio_get_time_ns(indio_dev)); return IRQ_HANDLED; } diff --git a/drivers/iio/light/veml6075.c b/drivers/iio/light/veml6075.c index edbb434070543..f7eb159e5cb40 100644 --- a/drivers/iio/light/veml6075.c +++ b/drivers/iio/light/veml6075.c @@ -100,7 +100,7 @@ static const struct iio_chan_spec veml6075_channels[] = { static int veml6075_request_measurement(struct veml6075_data *data) { - int ret, conf, int_time; + int ret, conf, int_time, int_index; ret = regmap_read(data->regmap, VEML6075_CMD_CONF, &conf); if (ret < 0) @@ -117,7 +117,11 @@ static int veml6075_request_measurement(struct veml6075_data *data) * time for all possible configurations. Using a 1.50 factor simplifies * operations and ensures reliability under all circumstances. */ - int_time = veml6075_it_ms[FIELD_GET(VEML6075_CONF_IT, conf)]; + int_index = FIELD_GET(VEML6075_CONF_IT, conf); + if (int_index >= ARRAY_SIZE(veml6075_it_ms)) + return -EINVAL; + + int_time = veml6075_it_ms[int_index]; msleep(int_time + (int_time / 2)); /* shutdown again, data registers are still accessible */ diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c index b648b0afa5733..ddf6dcc6fbdda 100644 --- a/drivers/iio/magnetometer/ak8975.c +++ b/drivers/iio/magnetometer/ak8975.c @@ -495,6 +495,10 @@ static int ak8975_who_i_am(struct i2c_client *client, dev_err(&client->dev, "Error reading WIA\n"); return ret; } + if (ret != sizeof(wia_val)) { + dev_err(&client->dev, "Error reading WIA\n"); + return -EIO; + } if (wia_val[0] != AK8975_DEVICE_ID) return -ENODEV; @@ -619,6 +623,10 @@ static int ak8975_setup(struct i2c_client *client) dev_err(&client->dev, "Not able to read asa data\n"); return ret; } + if (ret != sizeof(data->asa)) { + dev_err(&client->dev, "Error reading asa data\n"); + return -EIO; + } /* After reading fuse ROM data set power-down mode */ ret = ak8975_set_mode(data, POWER_DOWN); @@ -758,6 +766,10 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val) sizeof(rval), (u8*)&rval); if (ret < 0) goto exit; + if (ret != sizeof(rval)) { + ret = -EIO; + goto exit; + } /* Read out ST2 for release lock on measurement data. */ ret = i2c_smbus_read_byte_data(client, data->def->ctrl_regs[ST2]); @@ -784,6 +796,7 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val) exit: mutex_unlock(&data->lock); + pm_runtime_put_autosuspend(&data->client->dev); dev_err(&client->dev, "Error in reading axis\n"); return ret; } @@ -873,6 +886,8 @@ static void ak8975_fill_buffer(struct iio_dev *indio_dev) (u8 *)fval); if (ret < 0) goto unlock; + if (ret != sizeof(fval)) + goto unlock; mutex_unlock(&data->lock); diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c index 5a5e6e4fbe34b..83a0b0283605b 100644 --- a/drivers/iio/orientation/hid-sensor-rotation.c +++ b/drivers/iio/orientation/hid-sensor-rotation.c @@ -74,6 +74,13 @@ static int dev_rot_read_raw(struct iio_dev *indio_dev, long mask) { struct dev_rot_state *rot_state = iio_priv(indio_dev); + struct hid_sensor_hub_device *hsdev = rot_state->common_attributes.hsdev; + struct hid_sensor_hub_attribute_info *info = &rot_state->quaternion; + u32 usage_id = HID_USAGE_SENSOR_ORIENT_QUATERNION; + union { + s16 val16[4]; + s32 val32[4]; + } raw_buf; int ret_type; int i; @@ -83,8 +90,37 @@ static int dev_rot_read_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: if (size >= 4) { - for (i = 0; i < 4; ++i) - vals[i] = rot_state->scan.sampled_vals[i]; + if (info->size <= 0 || info->size > sizeof(raw_buf)) + return -EINVAL; + + hid_sensor_power_state(&rot_state->common_attributes, true); + + ret_type = sensor_hub_input_attr_read_values(hsdev, + hsdev->usage, + usage_id, + info->report_id, + SENSOR_HUB_SYNC, + info->size, + (u8 *)&raw_buf); + + hid_sensor_power_state(&rot_state->common_attributes, false); + + if (ret_type < 0) + return ret_type; + + switch (info->size) { + case sizeof(raw_buf.val16): + for (i = 0; i < ARRAY_SIZE(raw_buf.val16); i++) + vals[i] = raw_buf.val16[i]; + break; + case sizeof(raw_buf.val32): + for (i = 0; i < ARRAY_SIZE(raw_buf.val32); i++) + vals[i] = raw_buf.val32[i]; + break; + default: + return -EINVAL; + } + ret_type = IIO_VAL_INT_MULTIPLE; *val_len = 4; } else diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c index 9b489766e457a..0ebe0b682caae 100644 --- a/drivers/iio/pressure/bmp280-core.c +++ b/drivers/iio/pressure/bmp280-core.c @@ -1910,7 +1910,7 @@ static irqreturn_t bmp380_trigger_handler(int irq, void *p) u32 comp_press; s32 comp_temp; aligned_s64 timestamp; - } buffer; + } buffer = { }; int ret; guard(mutex)(&data->lock); diff --git a/drivers/iio/pressure/mpl115.c b/drivers/iio/pressure/mpl115.c index 830a5065c008f..16e112b796ba9 100644 --- a/drivers/iio/pressure/mpl115.c +++ b/drivers/iio/pressure/mpl115.c @@ -106,18 +106,18 @@ static int mpl115_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_PROCESSED: pm_runtime_get_sync(data->dev); ret = mpl115_comp_pressure(data, val, val2); + pm_runtime_put_autosuspend(data->dev); if (ret < 0) return ret; - pm_runtime_put_autosuspend(data->dev); return IIO_VAL_INT_PLUS_MICRO; case IIO_CHAN_INFO_RAW: pm_runtime_get_sync(data->dev); /* temperature -5.35 C / LSB, 472 LSB is 25 C */ ret = mpl115_read_temp(data); + pm_runtime_put_autosuspend(data->dev); if (ret < 0) return ret; - pm_runtime_put_autosuspend(data->dev); *val = ret >> 6; return IIO_VAL_INT; diff --git a/drivers/iio/proximity/vl53l0x-i2c.c b/drivers/iio/proximity/vl53l0x-i2c.c index ad3e46d47fa8d..6c6e6dab045f7 100644 --- a/drivers/iio/proximity/vl53l0x-i2c.c +++ b/drivers/iio/proximity/vl53l0x-i2c.c @@ -87,15 +87,14 @@ static irqreturn_t vl53l0x_trigger_handler(int irq, void *priv) ret = i2c_smbus_read_i2c_block_data(data->client, VL_REG_RESULT_RANGE_STATUS, sizeof(buffer), buffer); - if (ret < 0) - return ret; - else if (ret != 12) - return -EREMOTEIO; + if (ret != 12) + goto done; scan.chan = get_unaligned_be16(&buffer[10]); iio_push_to_buffers_with_ts(indio_dev, &scan, sizeof(scan), iio_get_time_ns(indio_dev)); +done: iio_trigger_notify_done(indio_dev->trig); vl53l0x_clear_irq(data); diff --git a/drivers/iio/resolver/ad2s1210.c b/drivers/iio/resolver/ad2s1210.c index 774728c804c01..1be19fe8aa3f6 100644 --- a/drivers/iio/resolver/ad2s1210.c +++ b/drivers/iio/resolver/ad2s1210.c @@ -1334,7 +1334,7 @@ static irqreturn_t ad2s1210_trigger_handler(int irq, void *p) ret = regmap_read(st->regmap, AD2S1210_REG_FAULT, ®_val); if (ret < 0) - return ret; + goto error_ret; st->sample.fault = reg_val; } diff --git a/drivers/iio/temperature/Makefile b/drivers/iio/temperature/Makefile index 07d6e65709f7f..0850bf6918204 100644 --- a/drivers/iio/temperature/Makefile +++ b/drivers/iio/temperature/Makefile @@ -13,7 +13,7 @@ obj-$(CONFIG_MAX31865) += max31865.o obj-$(CONFIG_MCP9600) += mcp9600.o obj-$(CONFIG_MLX90614) += mlx90614.o obj-$(CONFIG_MLX90632) += mlx90632.o -obj-$(CONFIG_MLX90632) += mlx90635.o +obj-$(CONFIG_MLX90635) += mlx90635.o obj-$(CONFIG_TMP006) += tmp006.o obj-$(CONFIG_TMP007) += tmp007.o obj-$(CONFIG_TMP117) += tmp117.o diff --git a/drivers/iio/temperature/ltc2983.c b/drivers/iio/temperature/ltc2983.c index 38e6f8dfd3b80..2bc5cd46a72f3 100644 --- a/drivers/iio/temperature/ltc2983.c +++ b/drivers/iio/temperature/ltc2983.c @@ -741,7 +741,7 @@ ltc2983_rtd_new(const struct fwnode_handle *child, struct ltc2983_data *st, struct ltc2983_rtd *rtd; int ret = 0; struct device *dev = &st->spi->dev; - u32 excitation_current = 0, n_wires = 0; + u32 excitation_current = 0, n_wires = 2; rtd = devm_kzalloc(dev, sizeof(*rtd), GFP_KERNEL); if (!rtd) @@ -1177,12 +1177,11 @@ static int ltc2983_chan_read(struct ltc2983_data *st, start_conversion |= LTC2983_STATUS_CHAN_SEL(sensor->chan); dev_dbg(&st->spi->dev, "Start conversion on chan:%d, status:%02X\n", sensor->chan, start_conversion); + reinit_completion(&st->completion); /* start conversion */ ret = regmap_write(st->regmap, LTC2983_STATUS_REG, start_conversion); if (ret) return ret; - - reinit_completion(&st->completion); /* * wait for conversion to complete. * 300 ms should be more than enough to complete the conversion. diff --git a/drivers/iio/temperature/tmp006.c b/drivers/iio/temperature/tmp006.c index d8d8c8936d177..bf62143fa7191 100644 --- a/drivers/iio/temperature/tmp006.c +++ b/drivers/iio/temperature/tmp006.c @@ -350,7 +350,7 @@ static int tmp006_probe(struct i2c_client *client) data->drdy_trig->ops = &tmp006_trigger_ops; iio_trigger_set_drvdata(data->drdy_trig, indio_dev); - ret = iio_trigger_register(data->drdy_trig); + ret = devm_iio_trigger_register(&client->dev, data->drdy_trig); if (ret) return ret; diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 27992c38ad902..d9306c0414b58 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -438,7 +438,7 @@ static int addr6_resolve(struct sockaddr *src_sock, static bool is_dst_local(const struct dst_entry *dst) { if (dst->ops->family == AF_INET) - return !!(dst_rtable(dst)->rt_type & RTN_LOCAL); + return dst_rtable(dst)->rt_type == RTN_LOCAL; else if (dst->ops->family == AF_INET6) return !!(dst_rt6_info(dst)->rt6i_flags & RTF_LOCAL); else diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 6ab9a0aee1ec6..1a2c2775b14d5 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -530,6 +530,7 @@ static int cm_init_av_by_path(struct sa_path_rec *path, struct rdma_ah_attr new_ah_attr; struct cm_device *cm_dev; struct cm_port *port; + u16 pkey_index; int ret; port = get_cm_port_from_path(path, sgid_attr); @@ -538,12 +539,10 @@ static int cm_init_av_by_path(struct sa_path_rec *path, cm_dev = port->cm_dev; ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num, - be16_to_cpu(path->pkey), &av->pkey_index); + be16_to_cpu(path->pkey), &pkey_index); if (ret) return ret; - cm_set_av_port(av, port); - /* * av->ah_attr might be initialized based on wc or during * request processing time which might have reference to sgid_attr. @@ -558,6 +557,8 @@ static int cm_init_av_by_path(struct sa_path_rec *path, if (ret) return ret; + av->pkey_index = pkey_index; + cm_set_av_port(av, port); av->timeout = path->packet_life_time + 1; rdma_move_ah_attr(&av->ah_attr, &new_ah_attr); return 0; @@ -2184,8 +2185,10 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv->av.ah_attr.roce.dmac); work->path[0].hop_limit = grh->hop_limit; - /* This destroy call is needed to pair with cm_init_av_for_response */ - cm_destroy_av(&cm_id_priv->av); + /* + * cm_init_av_by_path() will internally pair with the above + * cm_init_av_for_response() if it succeeds. + */ ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av); if (ret) { int err; diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index c3aa6d7fc66b6..5dad5d77ce274 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -661,7 +661,7 @@ void rdma_counter_init(struct ib_device *dev) fail: for (i = port; i >= rdma_start_port(dev); i--) { - port_counter = &dev->port_data[port].port_counter; + port_counter = &dev->port_data[i].port_counter; rdma_free_hw_stats_struct(port_counter->hstats); port_counter->hstats = NULL; mutex_destroy(&port_counter->lock); diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index 5e992ff3d7cff..ce8ae4305b9c8 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -97,13 +97,44 @@ static void destroy_all_handles_in_queue(struct ib_device *device, } } +/* + * Bulk-move all handles from @src into @dst without allocating new pages. + * If @dst has a partial tail page, fill it handle-by-handle from @src first + * to preserve the invariant that only the tail page is partial, then splice + * the remaining @src pages onto @dst. On return @src is empty. + * + * Caller must hold the lock protecting both queues. + */ +static void splice_frmr_queue_locked(struct frmr_queue *dst, + struct frmr_queue *src) +{ + u32 free_in_tail = dst->ci % NUM_HANDLES_PER_PAGE; + u32 handle; + + if (free_in_tail) { + free_in_tail = NUM_HANDLES_PER_PAGE - free_in_tail; + while (free_in_tail && src->ci) { + handle = pop_handle_from_queue_locked(src); + push_handle_to_queue_locked(dst, handle); + free_in_tail--; + } + } + + if (src->ci > 0) { + list_splice_tail_init(&src->pages_list, &dst->pages_list); + dst->num_pages += src->num_pages; + dst->ci += src->ci; + src->num_pages = 0; + src->ci = 0; + } +} + static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) { struct ib_frmr_pools *pools = device->frmr_pools; u32 total, to_destroy, destroyed = 0; bool has_work = false; u32 *handles; - u32 handle; spin_lock(&pool->lock); total = pool->queue.ci + pool->inactive_queue.ci + pool->in_use; @@ -112,7 +143,7 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) return false; } - to_destroy = total - pool->pinned_handles; + to_destroy = min(total - pool->pinned_handles, pool->inactive_queue.ci); handles = kcalloc(to_destroy, sizeof(*handles), GFP_ATOMIC); if (!handles) { @@ -121,15 +152,13 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) } /* Destroy all excess handles in the inactive queue */ - while (pool->inactive_queue.ci && destroyed < to_destroy) { - handles[destroyed++] = pop_handle_from_queue_locked( + for (; destroyed < to_destroy; destroyed++) + handles[destroyed] = pop_handle_from_queue_locked( &pool->inactive_queue); - } /* Move all handles from regular queue to inactive queue */ - while (pool->queue.ci) { - handle = pop_handle_from_queue_locked(&pool->queue); - push_handle_to_queue_locked(&pool->inactive_queue, handle); + if (pool->queue.ci > 0) { + splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue); has_work = true; } @@ -158,13 +187,7 @@ static void pool_aging_work(struct work_struct *work) /* Move all pages from regular queue to inactive queue */ spin_lock(&pool->lock); if (pool->queue.ci > 0) { - list_splice_tail_init(&pool->queue.pages_list, - &pool->inactive_queue.pages_list); - pool->inactive_queue.num_pages = pool->queue.num_pages; - pool->inactive_queue.ci = pool->queue.ci; - - pool->queue.num_pages = 0; - pool->queue.ci = 0; + splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue); has_work = true; } spin_unlock(&pool->lock); @@ -426,7 +449,7 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key, if (!handles) return -ENOMEM; - ret = pools->pool_ops->create_frmrs(device, key, handles, + ret = pools->pool_ops->create_frmrs(device, &driver_key, handles, needed_handles); if (ret) { kfree(handles); @@ -438,11 +461,16 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key, ret = push_handle_to_queue_locked(&pool->queue, handles[i]); if (ret) - goto end; + break; } - -end: spin_unlock(&pool->lock); + + if (ret) { + /* Destroy handles created but never pushed to the pool. */ + pools->pool_ops->destroy_frmrs(device, &handles[i], + needed_handles - i); + } + kfree(handles); schedule_aging: @@ -501,7 +529,9 @@ int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr) struct ib_frmr_pools *pools = device->frmr_pools; struct ib_frmr_pool *pool; - WARN_ON_ONCE(!device->frmr_pools); + if (WARN_ON_ONCE(!pools)) + return -EINVAL; + pool = ib_frmr_pool_find(pools, &mr->frmr.key); if (!pool) { pool = create_frmr_pool(device, &mr->frmr.key); @@ -519,9 +549,8 @@ EXPORT_SYMBOL(ib_frmr_pool_pop); * @device: The device to push the FRMR handle to. * @mr: The MR containing the FRMR handle to push back to the pool. * - * Returns 0 on success, negative error code on failure. */ -int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) +void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) { struct ib_frmr_pool *pool = mr->frmr.pool; struct ib_frmr_pools *pools = device->frmr_pools; @@ -529,19 +558,38 @@ int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) int ret; spin_lock(&pool->lock); + pool->in_use--; + ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle); + /* Schedule aging every time an empty pool becomes non-empty */ - if (pool->queue.ci == 0) + if (!ret && pool->queue.ci == 1) schedule_aging = true; - ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle); - if (ret == 0) - pool->in_use--; spin_unlock(&pool->lock); - if (ret == 0 && schedule_aging) + if (ret) { + pools->pool_ops->destroy_frmrs(device, &mr->frmr.handle, 1); + return; + } + + if (schedule_aging) queue_delayed_work(pools->aging_wq, &pool->aging_work, secs_to_jiffies(READ_ONCE(pools->aging_period_sec))); - return ret; } EXPORT_SYMBOL(ib_frmr_pool_push); + +/* + * Drop a handle previously popped from the pool without returning it for + * reuse. The caller is responsible for destroying the underlying hardware + * resource. + */ +void ib_frmr_pool_drop(struct ib_mr *mr) +{ + struct ib_frmr_pool *pool = mr->frmr.pool; + + spin_lock(&pool->lock); + pool->in_use--; + spin_unlock(&pool->lock); +} +EXPORT_SYMBOL(ib_frmr_pool_drop); diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 5aaba2b9746ba..02a0a9c0a4a6a 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -695,7 +695,7 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (has_cap_net_admin) { if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey)) @@ -711,9 +711,12 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) return -EMSGSIZE; - if (!rdma_is_kernel_res(res) && - nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id)) - return -EMSGSIZE; + if (!rdma_is_kernel_res(res)) { + struct ib_pd *pd = READ_ONCE(mr->pd); + + if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, pd->res.id)) + return -EMSGSIZE; + } if (fill_res_name_pid(msg, res)) return -EMSGSIZE; @@ -727,7 +730,7 @@ static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (!dev->ops.fill_res_mr_entry_raw) return -EINVAL; @@ -1017,7 +1020,7 @@ static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) goto err; diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index ac3688952cabb..cfee2071586c1 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -71,6 +71,8 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, xa_lock(&rt->xa); xas_for_each(&xas, e, U32_MAX) { + if (xa_is_zero(e)) + continue; if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) continue; cnt++; @@ -276,6 +278,53 @@ int rdma_restrack_put(struct rdma_restrack_entry *res) } EXPORT_SYMBOL(rdma_restrack_put); +/** + * rdma_restrack_sync() - Fence concurrent netlink dumps on an entry + * @res: resource entry + * + * After this returns any concurrent netlink dump threads will see the current + * value of the object. This is useful if the object has to be changed and there + * is not locking to protect the nl side. Eg for mr->pd. This effectively + * destroys the object from a kref/xarray perspective and then immediately + * restores it. The kref is acting like a lock to barrier concurrent nl threads. + * Callers must ensure rdma_restrack_del() is not concurrently called. + */ +void rdma_restrack_sync(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_entry *old; + struct rdma_restrack_root *rt; + struct task_struct *task; + struct ib_device *dev; + + if (!res->valid || res->no_track) + return; + + dev = res_to_dev(res); + if (WARN_ON(!dev)) + return; + + rt = &dev->res[res->type]; + if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) + return; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); + if (WARN_ON(old != res)) + return; + + task = res->task; + if (task) + get_task_struct(task); + rdma_restrack_put(res); + wait_for_completion(&res->comp); + reinit_completion(&res->comp); + if (task) + res->task = task; + kref_init(&res->kref); + + xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); +} +EXPORT_SYMBOL(rdma_restrack_sync); + /** * rdma_restrack_del() - delete object from the resource tracking database * @res: resource entry diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 6a04fc41f7380..75b8d1005a984 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -27,6 +27,7 @@ int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); +void rdma_restrack_sync(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 91a62d2ade4dd..22793e4b1895e 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -47,6 +47,7 @@ #include "uverbs.h" #include "core_priv.h" +#include "restrack.h" /* * Copy a response to userspace. If the provided 'resp' is larger than the @@ -819,6 +820,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) ret = PTR_ERR(new_pd); goto put_uobjs; } + if (new_pd == orig_pd) { + uobj_put_obj_read(new_pd); + cmd.flags &= ~IB_MR_REREG_PD; + } } else { new_pd = mr->pd; } @@ -866,9 +871,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) mr = new_mr; } else { if (cmd.flags & IB_MR_REREG_PD) { - atomic_dec(&orig_pd->usecnt); - mr->pd = new_pd; atomic_inc(&new_pd->usecnt); + WRITE_ONCE(mr->pd, new_pd); + rdma_restrack_sync(&mr->res); + atomic_dec(&orig_pd->usecnt); } if (cmd.flags & IB_MR_REREG_TRANS) { mr->iova = cmd.hca_va; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 365ec2767d253..5729063474636 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1024,6 +1024,9 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (rc) ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); + if (qp->dbr_obj) + kref_put(&qp->dbr_obj->usecnt, bnxt_re_dbr_kref_release); + if (rdma_is_kernel_res(&qp->ib_qp.res)) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_clean_qp(&qp->qplib_qp); @@ -1136,23 +1139,42 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, return 0; } -static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, - struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, - struct bnxt_re_qp_req *ureq) +static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, + struct ib_umem *umem, + struct bnxt_qplib_sg_info *sginfo) { - struct bnxt_qplib_qp *qplib_qp; - int bytes = 0, psn_sz; - struct ib_umem *umem; - int psn_nume; + unsigned long page_size; - qplib_qp = &qp->qplib_qp; + if (!umem) + return -EINVAL; - bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); - /* Consider mapping PSN search memory only for RC QPs. */ - if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { + page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); + if (!page_size || page_size != SZ_4K) + return -EINVAL; + + sginfo->umem = umem; + sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); + sginfo->pgsize = page_size; + sginfo->pgshft = __builtin_ctz(page_size); + return 0; +} + +static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev, + struct bnxt_re_ucontext *cntx, + struct bnxt_qplib_qp *qplib_qp, + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr) +{ + int psn_sz, psn_nume; + + if (rdev->dev_attr && + _is_host_msn_table(rdev->dev_attr->dev_cap_flags2)) + psn_sz = sizeof(struct sq_msn_search); + else psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? - sizeof(struct sq_psn_search_ext) : - sizeof(struct sq_psn_search); + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (!fixed_que_attr) { if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { psn_nume = ureq->sq_slots; } else { @@ -1162,9 +1184,31 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, } if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2)) psn_nume = roundup_pow_of_two(psn_nume); - bytes += (psn_nume * psn_sz); + } else { + psn_nume = ureq->sq_npsn; } + return psn_nume * psn_sz; +} + +static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, + struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr, + struct bnxt_re_dbr_obj *dbr_obj) +{ + struct bnxt_qplib_qp *qplib_qp; + struct ib_umem *umem; + int bytes; + int rc; + + qplib_qp = &qp->qplib_qp; + + bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + /* Consider mapping PSN search memory only for RC QPs. */ + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) + bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq, fixed_que_attr); + bytes = PAGE_ALIGN(bytes); umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes, IB_ACCESS_LOCAL_WRITE); @@ -1172,33 +1216,45 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, return PTR_ERR(umem); qp->sumem = umem; - qplib_qp->sq.sg_info.umem = umem; - qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; - qplib_qp->qp_handle = ureq->qp_handle; + rc = bnxt_re_setup_sginfo(rdev, qp->sumem, &qplib_qp->sq.sg_info); + if (rc) + goto fail; + + if (qp->qplib_qp.srq) + goto done; - if (!qp->qplib_qp.srq) { - bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); - bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) - goto rqfail; - qp->rumem = umem; - qplib_qp->rq.sg_info.umem = umem; - qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; + bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + goto fail; } - qplib_qp->dpi = &cntx->dpi; + qp->rumem = umem; + rc = bnxt_re_setup_sginfo(rdev, qp->rumem, &qplib_qp->rq.sg_info); + if (rc) + goto rqfail; + +done: + if (dbr_obj) + qplib_qp->dpi = &dbr_obj->dpi; + else + qplib_qp->dpi = &cntx->dpi; + qplib_qp->qp_handle = ureq->qp_handle; qplib_qp->is_user = true; return 0; + rqfail: + ib_umem_release(qp->rumem); + qp->rumem = NULL; + memset(&qplib_qp->rq.sg_info, 0, sizeof(qplib_qp->rq.sg_info)); +fail: ib_umem_release(qp->sumem); qp->sumem = NULL; memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); - - return PTR_ERR(umem); + return rc; } static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah @@ -1597,7 +1653,9 @@ static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, return qptype; } -static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) +static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp, + bool fixed_que_attr, + struct bnxt_re_qp_req *req) { struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; struct bnxt_qplib_q *sq = &qplib_qp->sq; @@ -1620,12 +1678,17 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) /* Update msn tbl size */ if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) { - if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); - else - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) / 2; + if (!fixed_que_attr) { + if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); + else + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) + / 2; + } else { + qplib_qp->msn_tbl_sz = req->sq_npsn; + } qplib_qp->msn = 0; } } @@ -1633,7 +1696,9 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + struct bnxt_re_dbr_obj *dbr_obj, + bool fixed_que_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1656,6 +1721,16 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, return qptype; qplqp->type = (u8)qptype; qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx); + if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE && + (!ureq->sq_slots || ureq->sq_slots > BNXT_RE_MAX_SQ_SLOTS)) + return -EINVAL; + if (fixed_que_attr) { + if (qplqp->wqe_mode != BNXT_QPLIB_WQE_MODE_VARIABLE) + return -EOPNOTSUPP; + if (!ureq->sq_npsn || + ureq->sq_npsn > roundup_pow_of_two(ureq->sq_slots / 2)) + return -EINVAL; + } qplqp->dev_cap_flags = dev_attr->dev_cap_flags; qplqp->cctx = rdev->chip_ctx; if (init_attr->qp_type == IB_QPT_RC) { @@ -1699,12 +1774,13 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx); if (uctx) { /* This will update DPI and qp_handle */ - rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq); + rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr, + dbr_obj); if (rc) return rc; } - bnxt_re_qp_calculate_msn_psn_size(qp); + bnxt_re_qp_calculate_msn_psn_size(qp, fixed_que_attr, ureq); rc = bnxt_re_setup_qp_hwqs(qp); if (rc) @@ -1835,8 +1911,11 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev) int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata) { + struct bnxt_re_dbr_obj *dbr_obj = NULL; struct bnxt_qplib_dev_attr *dev_attr; + struct uverbs_attr_bundle *attrs; struct bnxt_re_ucontext *uctx; + bool fixed_que_attr = false; struct bnxt_re_qp_req ureq; struct bnxt_re_dev *rdev; struct bnxt_re_pd *pd; @@ -1853,9 +1932,23 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); if (udata) { - rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0); + rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, + BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR); if (rc) return rc; + + attrs = rdma_udata_to_uverbs_attr_bundle(udata); + if (uverbs_attr_is_valid(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE)) { + dbr_obj = uverbs_attr_get_obj(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE); + if (IS_ERR(dbr_obj)) + return PTR_ERR(dbr_obj); + kref_get(&dbr_obj->usecnt); + qp->dbr_obj = dbr_obj; + } + if (ureq.comp_mask & BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR) + fixed_que_attr = true; } rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); @@ -1865,7 +1958,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, } qp->rdev = rdev; - rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq); + rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq, + dbr_obj, fixed_que_attr); if (rc) goto fail; @@ -1935,6 +2029,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); bnxt_re_qp_free_umem(qp); fail: + if (dbr_obj) + kref_put(&dbr_obj->usecnt, bnxt_re_dbr_kref_release); return rc; } @@ -2029,11 +2125,11 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) if (ret) return ret; - if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) { - free_page((unsigned long)srq->uctx_srq_page); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) hash_del(&srq->hash_entry); - } bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) + free_page((unsigned long)srq->uctx_srq_page); ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); return ib_respond_empty_udata(udata); @@ -3332,11 +3428,11 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (ret) return ret; - if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) { - free_page((unsigned long)cq->uctx_cq_page); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) hash_del(&cq->hash_entry); - } bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) + free_page((unsigned long)cq->uctx_cq_page); bnxt_re_put_nq(rdev, nq); @@ -3345,26 +3441,6 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) return ib_respond_empty_udata(udata); } -static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, - struct ib_umem *umem, - struct bnxt_qplib_sg_info *sginfo) -{ - unsigned long page_size; - - if (!umem) - return -EINVAL; - - page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); - if (!page_size || page_size != SZ_4K) - return -EINVAL; - - sginfo->umem = umem; - sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); - sginfo->pgsize = page_size; - sginfo->pgshft = __builtin_ctz(page_size); - return 0; -} - int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { @@ -4644,6 +4720,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) goto fail; } spin_lock_init(&uctx->sh_lock); + mutex_init(&uctx->wcdpi_lock); resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; chip_met_rev_num = rdev->chip_ctx->chip_num; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 08f71a94d55d7..acb40fc528d24 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -96,6 +96,7 @@ struct bnxt_re_qp { struct bnxt_re_cq *scq; struct bnxt_re_cq *rcq; struct dentry *dentry; + struct bnxt_re_dbr_obj *dbr_obj; /* doorbell region */ }; struct bnxt_re_cq { @@ -141,6 +142,7 @@ struct bnxt_re_ucontext { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_qplib_dpi wcdpi; + struct mutex wcdpi_lock; /* serialises WC DPI alloc/free */ void *shpg; spinlock_t sh_lock; /* protect shpg */ struct rdma_user_mmap_entry *shpage_mmap; @@ -167,7 +169,7 @@ struct bnxt_re_dbr_obj { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_re_user_mmap_entry *entry; - atomic_t usecnt; /* QPs using this dbr */ + struct kref usecnt; /* 1 (uobject) + n (QPs using this dbr) */ }; struct bnxt_re_flow { @@ -308,4 +310,5 @@ void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); struct bnxt_re_user_mmap_entry* bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset, enum bnxt_re_mmap_flag mmap_flag, u64 *offset); +void bnxt_re_dbr_kref_release(struct kref *ref); #endif /* __BNXT_RE_IB_VERBS_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index a892f11729179..d25fdc458120e 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -1093,8 +1093,6 @@ static int bnxt_re_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr) goto err; if (rdma_nl_put_driver_u32(msg, "element_size", mr_hwq->element_size)) goto err; - if (rdma_nl_put_driver_u64_hex(msg, "hwq", (unsigned long)mr_hwq)) - goto err; if (rdma_nl_put_driver_u64_hex(msg, "va", mr->qplib_mr.va)) goto err; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 95e0489c53c37..756f8b5f042a9 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -764,9 +764,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, break; case BNXT_QPLIB_DPI_TYPE_WC: dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; default: dpi->dbr = ioremap(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; } @@ -774,6 +778,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, mutex_unlock(&res->dpi_tbl_lock); return 0; +fail_ioremap: + /* Roll back the bit we just claimed. */ + set_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = NULL; + mutex_unlock(&res->dpi_tbl_lock); + return -ENOMEM; + } int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index 9fadd637cb5b1..c4193ae75b54d 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -369,6 +369,7 @@ int bnxt_qplib_destroy_flow(struct bnxt_qplib_res *res); #define BNXT_VAR_MAX_SLOT_ALIGN 256 #define BNXT_VAR_MAX_SGE 13 #define BNXT_RE_MAX_RQ_WQES 65536 +#define BNXT_RE_MAX_SQ_SLOTS 65536 #define BNXT_STATIC_MAX_SGE 6 diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 3eaee7101615b..ca6b1630cd83d 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -76,8 +76,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * struct ib_ucontext *ib_uctx; struct bnxt_re_dev *rdev; u64 mmap_offset; + u32 dpi = 0; u32 length; - u32 dpi; u64 addr; int err; @@ -98,26 +98,39 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * switch (alloc_type) { case BNXT_RE_ALLOC_WC_PAGE: - if (cctx->modes.db_push) { + if (cctx->modes.db_push) { + mutex_lock(&uctx->wcdpi_lock); + /* already allocated — one WC page per context */ + if (uctx->wcdpi.dbr) { + mutex_unlock(&uctx->wcdpi_lock); + return -EEXIST; + } if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi, - uctx, BNXT_QPLIB_DPI_TYPE_WC)) + uctx, BNXT_QPLIB_DPI_TYPE_WC)) { + mutex_unlock(&uctx->wcdpi_lock); return -ENOMEM; + } length = PAGE_SIZE; dpi = uctx->wcdpi.dpi; addr = (u64)uctx->wcdpi.umdbr; mmap_flag = BNXT_RE_MMAP_WC_DB; + mutex_unlock(&uctx->wcdpi_lock); } else { return -EINVAL; } break; case BNXT_RE_ALLOC_DBR_BAR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_bar_addr; mmap_flag = BNXT_RE_MMAP_DBR_BAR; break; case BNXT_RE_ALLOC_DBR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_page; mmap_flag = BNXT_RE_MMAP_DBR_PAGE; @@ -128,8 +141,15 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * } entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset); - if (!entry) + if (!entry) { + if (mmap_flag == BNXT_RE_MMAP_WC_DB) { + mutex_lock(&uctx->wcdpi_lock); + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + mutex_unlock(&uctx->wcdpi_lock); + } return -ENOMEM; + } uobj->object = entry; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); @@ -160,11 +180,16 @@ static int alloc_page_obj_cleanup(struct ib_uobject *uobject, switch (entry->mmap_flag) { case BNXT_RE_MMAP_WC_DB: - if (uctx && uctx->wcdpi.dbr) { + if (uctx) { struct bnxt_re_dev *rdev = uctx->rdev; - bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); - uctx->wcdpi.dbr = NULL; + mutex_lock(&uctx->wcdpi_lock); + if (uctx->wcdpi.dbr) { + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + } + mutex_unlock(&uctx->wcdpi_lock); } break; case BNXT_RE_MMAP_DBR_BAR: @@ -252,6 +277,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)cq->uctx_cq_page; + if (!addr) + return -EOPNOTSUPP; break; case BNXT_RE_SRQ_TOGGLE_MEM: srq = bnxt_re_search_for_srq(rdev, res_id); @@ -259,6 +286,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)srq->uctx_srq_page; + if (!addr) + return -EOPNOTSUPP; break; default: @@ -369,6 +398,7 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a } obj->rdev = rdev; + kref_init(&obj->usecnt); uobj->object = obj; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE); @@ -391,15 +421,33 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a return ret; } +void bnxt_re_dbr_kref_release(struct kref *ref) +{ + struct bnxt_re_dbr_obj *obj = + container_of(ref, struct bnxt_re_dbr_obj, usecnt); + struct bnxt_re_dev *rdev = obj->rdev; + + rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); + bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); + kfree(obj); +} + static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { struct bnxt_re_dbr_obj *obj = uobject->object; - struct bnxt_re_dev *rdev = obj->rdev; - rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); - bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); + /* If it is being destroyed explicitly while QPs still hold a + * reference (> 1), reject it with EBUSY. If no QP references + * or implicit teardown (process exit, driver removal), drop + * the uobject reference unconditionally. The object gets freed + * (bnxt_re_dbr_kref_release) when the usecnt goes to zero. + */ + if (why == RDMA_REMOVE_DESTROY && kref_read(&obj->usecnt) > 1) + return -EBUSY; + + kref_put(&obj->usecnt, bnxt_re_dbr_kref_release); return 0; } @@ -459,11 +507,26 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR, DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR, &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR)); +ADD_UVERBS_ATTRIBUTES_SIMPLE( + bnxt_re_qp_create, + UVERBS_OBJECT_QP, + UVERBS_METHOD_QP_CREATE, + UVERBS_ATTR_IDR(BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE, + BNXT_RE_OBJECT_DBR, + UVERBS_ACCESS_READ, + UA_OPTIONAL)); + +const struct uapi_definition bnxt_re_create_qp_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &bnxt_re_qp_create), + {}, +}; + const struct uapi_definition bnxt_re_uapi_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR), + UAPI_DEF_CHAIN(bnxt_re_create_qp_defs), {} }; diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 8b5a5b32b0fa2..b7fd8b1fbbbde 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -1206,6 +1206,7 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra) { struct hfi1_devdata *dd; + struct ib_device *ibdev; int ret, nports; /* extra is * number of ports */ @@ -1227,7 +1228,17 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, "Could not allocate unit ID: error %d\n", -ret); goto bail; } - rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit); + + /* + * FIXME: rvt and its users want to touch the ibdev before + * registration and have things like the name work. We don't have the + * infrastructure in the core to support this directly today, hack it + * to work by setting the name manually here. + */ + ibdev = &dd->verbs_dev.rdi.ibdev; + dev_set_name(&ibdev->dev, "%s_%d", class_name(), dd->unit); + strscpy(ibdev->name, dev_name(&ibdev->dev), IB_DEVICE_NAME_MAX); + /* * If the BIOS does not have the NUMA node information set, select * NUMA 0 so we get consistent performance. diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 621568e114054..f7e220ab42198 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -174,9 +174,9 @@ static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC, hr_cq->cqn); if (ret) - ibdev_err(ibdev, - "failed to send create cmd for CQ(0x%lx), ret = %d.\n", - hr_cq->cqn, ret); + ibdev_err_ratelimited(ibdev, + "failed to send create cmd for CQ(0x%lx), ret = %d.\n", + hr_cq->cqn, ret); hns_roce_free_cmd_mailbox(hr_dev, mailbox); diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index b869cdc541189..db32c5897640f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -26,17 +26,6 @@ static const struct file_operations hns_debugfs_seqfile_fops = { .llseek = seq_lseek }; -static void init_debugfs_seqfile(struct hns_debugfs_seqfile *seq, - const char *name, struct dentry *parent, - int (*read_fn)(struct seq_file *, void *), - void *data) -{ - debugfs_create_file(name, 0400, parent, seq, &hns_debugfs_seqfile_fops); - - seq->read = read_fn; - seq->data = data; -} - static const char * const sw_stat_info[] = { [HNS_ROCE_DFX_AEQE_CNT] = "aeqe", [HNS_ROCE_DFX_CEQE_CNT] = "ceqe", @@ -76,10 +65,12 @@ static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev, { struct hns_sw_stat_debugfs *dbgfs = &hr_dev->dbgfs.sw_stat_root; - dbgfs->root = debugfs_create_dir("sw_stat", parent); + dbgfs->sw_stat.read = sw_stat_debugfs_show; + dbgfs->sw_stat.data = hr_dev; - init_debugfs_seqfile(&dbgfs->sw_stat, "sw_stat", dbgfs->root, - sw_stat_debugfs_show, hr_dev); + dbgfs->root = debugfs_create_dir("sw_stat", parent); + debugfs_create_file("sw_stat", 0400, dbgfs->root, &dbgfs->sw_stat, + &hns_debugfs_seqfile_fops); } /* debugfs for device */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index e7c9e30ad2d8b..ccb40f8a48b72 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -314,14 +314,14 @@ static int calc_hem_config(struct hns_roce_dev *hr_dev, bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); switch (bt_num) { case 3: - index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l1 = (u64)l0_idx * chunk_ba_num + l1_idx; index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num * chunk_ba_num + - l1_idx * chunk_ba_num + l2_idx; + index->buf = (u64)l0_idx * chunk_ba_num * chunk_ba_num + + (u64)l1_idx * chunk_ba_num + l2_idx; break; case 2: index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num + l1_idx; + index->buf = (u64)l0_idx * chunk_ba_num + l1_idx; break; case 1: index->buf = l0_idx; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index fa36700d0db2b..60478382e82ff 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -6192,9 +6192,9 @@ static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn); hns_roce_free_cmd_mailbox(hr_dev, mailbox); if (ret) - ibdev_err(&hr_dev->ib_dev, - "failed to handle cmd of modifying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to handle cmd of modifying SRQ, ret = %d.\n", + ret); } out: @@ -6220,9 +6220,9 @@ static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC, srq->srqn); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd of querying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd of querying SRQ, ret = %d.\n", + ret); goto out; } @@ -6328,9 +6328,9 @@ static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT, key_to_hw_index(key)); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd when querying MPT, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd when querying MPT, ret = %d.\n", + ret); goto err_mailbox; } @@ -7586,8 +7586,8 @@ static int __init hns_roce_hw_v2_init(void) static void __exit hns_roce_hw_v2_exit(void) { - hns_roce_dealloc_bond_grp(); hnae3_unregister_client(&hns_roce_hw_v2_client); + hns_roce_dealloc_bond_grp(); hns_roce_cleanup_debugfs(); } diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 0dbe99aab6ad2..1e54710354542 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -1113,7 +1113,7 @@ static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq) unsigned long flags; spin_lock_irqsave(&hr_cq->lock, flags); - if (cq->comp_handler) { + if (cq->comp_handler && hr_cq->ib_cq.poll_ctx != IB_POLL_DIRECT) { if (!hr_cq->is_armed) { hr_cq->is_armed = 1; list_add_tail(&hr_cq->node, cq_list); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 25bfd3970f5b6..c13d29cd0897b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -173,7 +173,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx & (hr_dev->caps.num_mtpts - 1)); if (ret) { - dev_err(dev, "failed to create mpt, ret = %d.\n", ret); + dev_err_ratelimited(dev, "failed to create mpt, ret = %d.\n", ret); goto err_page; } @@ -319,7 +319,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT, mtpt_idx); if (ret) - ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); + ibdev_warn_ratelimited(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); mr->enabled = 0; mr->iova = virt_addr; @@ -350,7 +350,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx); if (ret) { - ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret); + ibdev_err_ratelimited(ib_dev, "failed to create MPT, ret = %d.\n", ret); goto free_cmd_mbox; } diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 8b94cbdfa54df..8feb5851067d8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -103,7 +103,7 @@ static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ, srq->srqn); if (ret) - ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret); + ibdev_err_ratelimited(ibdev, "failed to config SRQC, ret = %d.\n", ret); err_mbox: hns_roce_free_cmd_mailbox(hr_dev, mailbox); diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index f9be467d137f8..c345cc6542566 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -235,8 +235,7 @@ static void irdma_complete_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request) { if (cqp_request->waiting) { - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } else if (cqp_request->callback_fcn) { cqp_request->callback_fcn(cqp_request); } @@ -1107,9 +1106,9 @@ static int irdma_create_cqp(struct irdma_pci_f *rf) INIT_LIST_HEAD(&cqp->cqp_avail_reqs); INIT_LIST_HEAD(&cqp->cqp_pending_reqs); - /* init the waitqueue of the cqp_requests and add them to the list */ + /* init the completion of the cqp_requests and add them to the list */ for (i = 0; i < sqsize; i++) { - init_waitqueue_head(&cqp->cqp_requests[i].waitq); + init_completion(&cqp->cqp_requests[i].comp); list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs); } init_waitqueue_head(&cqp->remove_wq); diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h index 3d49bd57bae7c..8c17a201c1fd2 100644 --- a/drivers/infiniband/hw/irdma/main.h +++ b/drivers/infiniband/hw/irdma/main.h @@ -161,13 +161,12 @@ struct irdma_cqp_compl_info { struct irdma_cqp_request { struct cqp_cmds_info info; - wait_queue_head_t waitq; + struct completion comp; struct list_head list; refcount_t refcnt; void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; - bool request_done; /* READ/WRITE_ONCE macros operate on it */ bool waiting:1; bool dynamic:1; bool pending:1; diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 495e5daff4b45..8e9e159f19ffe 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -442,7 +442,7 @@ struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, if (cqp_request) { cqp_request->dynamic = true; if (wait) - init_waitqueue_head(&cqp_request->waitq); + init_completion(&cqp_request->comp); } } if (!cqp_request) { @@ -480,7 +480,7 @@ void irdma_free_cqp_request(struct irdma_cqp *cqp, if (cqp_request->dynamic) { kfree(cqp_request); } else { - WRITE_ONCE(cqp_request->request_done, false); + reinit_completion(&cqp_request->comp); cqp_request->callback_fcn = NULL; cqp_request->waiting = false; cqp_request->pending = false; @@ -515,8 +515,7 @@ irdma_free_pending_cqp_request(struct irdma_cqp *cqp, { if (cqp_request->waiting) { cqp_request->compl_info.error = true; - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } wait_event_timeout(cqp->remove_wq, refcount_read(&cqp_request->refcnt) == 1, 1000); @@ -609,9 +608,8 @@ static int irdma_wait_event(struct irdma_pci_f *rf, cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); - if (wait_event_timeout(cqp_request->waitq, - READ_ONCE(cqp_request->request_done), - msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) + if (wait_for_completion_timeout(&cqp_request->comp, + msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) break; if (cqp_request->pending) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 8cd4275328052..7da7a7e8b30ca 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -2572,7 +2572,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, } cqmr_shadow = &iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; - cqmr->split = true; } else { info.shadow_area_pa = cqmr->shadow; } @@ -2781,10 +2780,11 @@ static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, * irdma_copy_user_pgaddrs - copy user page address to pble's os locally * @iwmr: iwmr for IB's user page addresses * @pbl: ple pointer to save 1 level or 0 level pble + * @pbl_len: Max number of PBL entries to populate * @level: indicated level 0, 1 or 2 */ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, - enum irdma_pble_level level) + u32 pbl_len, enum irdma_pble_level level) { struct ib_umem *region = iwmr->region; struct irdma_pbl *iwpbl = &iwmr->iwpbl; @@ -2792,7 +2792,9 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, struct irdma_pble_info *pinfo; struct ib_block_iter biter; u32 idx = 0; - u32 pbl_cnt = 0; + + if (!pbl_len) + return; pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; @@ -2801,7 +2803,7 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { *pbl = rdma_block_iter_dma_address(&biter); - if (++pbl_cnt == palloc->total_cnt) + if (!--pbl_len) break; pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); } @@ -2877,6 +2879,7 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, u64 *pbl; int status; enum irdma_pble_level level = PBLE_LEVEL_1; + u32 pbl_len; if (lvl) { status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, @@ -2884,16 +2887,18 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, if (status) return status; + pbl_len = palloc->total_cnt; iwpbl->pbl_allocated = true; level = palloc->level; pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf; pbl = pinfo->addr; } else { + pbl_len = IRDMA_MAX_SAVED_PHY_PGADDR; pbl = iwmr->pgaddrmem; } - irdma_copy_user_pgaddrs(iwmr, pbl, level); + irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); if (lvl) iwmr->pgaddrmem[0] = *pbl; @@ -2974,7 +2979,8 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev, case IRDMA_MEMREG_TYPE_CQ: hmc_p = &cqmr->cq_pbl; - if (!cqmr->split) + if (!(iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & + IRDMA_FEATURE_CQ_RESIZE)) cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; if (lvl) @@ -3319,6 +3325,7 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, int err; lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; + iwmr->access = access; err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); if (err) diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index aabbb3442098b..289ebc9b23ca7 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -65,7 +65,6 @@ struct irdma_hmc_pble { struct irdma_cq_mr { struct irdma_hmc_pble cq_pbl; dma_addr_t shadow; - bool split; }; struct irdma_srq_mr { diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index afc2fc124fee3..307ae01bf26f3 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -600,8 +600,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, props->phys_state = IB_PORT_PHYS_STATE_DISABLED; } - props->active_width = IB_WIDTH_4X; - props->active_speed = IB_SPEED_EDR; + ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); props->pkey_tbl_len = 1; if (mana_ib_is_rnic(dev)) { props->gid_tbl_len = 16; diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 5a799d6df93eb..898c8363422ab 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -135,6 +135,7 @@ struct mlx4_ib_mr { dma_addr_t page_map; u32 npages; u32 max_pages; + int access_flags; struct mlx4_mr mmr; struct ib_umem *umem; size_t page_map_size; diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 6747bca306777..67b0e8954b5e5 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -181,6 +181,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (err) goto err_mr; + mr->access_flags = access_flags; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; mr->ibmr.page_size = 1U << shift; @@ -241,6 +242,8 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, if (err) goto release_mpt_entry; + } else { + mr_access_flags = mmr->access_flags; } if (flags & IB_MR_REREG_TRANS) { @@ -282,8 +285,10 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, * return a failure. But dereg_mr will free the resources. */ err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry); - if (!err && flags & IB_MR_REREG_ACCESS) - mmr->mmr.access = mr_access_flags; + if (!err && flags & IB_MR_REREG_ACCESS) { + mmr->access_flags = mr_access_flags; + mmr->mmr.access = convert_access(mr_access_flags); + } release_mpt_entry: mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry); diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c index b8fa4ecfc9610..e688ad66a895f 100644 --- a/drivers/infiniband/hw/mlx4/sysfs.c +++ b/drivers/infiniband/hw/mlx4/sysfs.c @@ -636,12 +636,6 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) p->port_num = port_num; p->slave = slave; - ret = kobject_init_and_add(&p->kobj, &port_type, - kobject_get(dev->dev_ports_parent[slave]), - "%d", port_num); - if (ret) - goto err_alloc; - p->pkey_group.name = "pkey_idx"; p->pkey_group.attrs = alloc_group_attrs(show_port_pkey, @@ -649,13 +643,9 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) dev->dev->caps.pkey_table_len[port_num]); if (!p->pkey_group.attrs) { ret = -ENOMEM; - goto err_alloc; + goto err_free_port; } - ret = sysfs_create_group(&p->kobj, &p->pkey_group); - if (ret) - goto err_free_pkey; - p->gid_group.name = "gid_idx"; p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1); if (!p->gid_group.attrs) { @@ -663,28 +653,47 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) goto err_free_pkey; } + ret = kobject_init_and_add(&p->kobj, &port_type, + kobject_get(dev->dev_ports_parent[slave]), + "%d", port_num); + if (ret) + goto err_put; + + ret = sysfs_create_group(&p->kobj, &p->pkey_group); + if (ret) + goto err_del; + ret = sysfs_create_group(&p->kobj, &p->gid_group); if (ret) - goto err_free_gid; + goto err_remove_pkey; ret = add_vf_smi_entries(p); if (ret) - goto err_free_gid; + goto err_remove_gid; list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]); return 0; -err_free_gid: - kfree(p->gid_group.attrs[0]); - kfree(p->gid_group.attrs); +err_remove_gid: + sysfs_remove_group(&p->kobj, &p->gid_group); + +err_remove_pkey: + sysfs_remove_group(&p->kobj, &p->pkey_group); + +err_del: + kobject_del(&p->kobj); + +err_put: + kobject_put(dev->dev_ports_parent[slave]); + kobject_put(&p->kobj); + return ret; err_free_pkey: for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i) kfree(p->pkey_group.attrs[i]); kfree(p->pkey_group.attrs); -err_alloc: - kobject_put(dev->dev_ports_parent[slave]); +err_free_port: kfree(p); return ret; } diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 645ebcc0832d7..c2ae5a1404711 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -1913,6 +1913,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( return err; } +static bool devx_key_in_sub_list(struct list_head *list, u32 key_level1) +{ + struct devx_event_subscription *s; + + list_for_each_entry(s, list, event_list) + if (s->xa_key_level1 == key_level1) + return true; + + return false; +} + static void subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table, u32 key_level1, @@ -2160,10 +2171,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( event_sub = kzalloc_obj(*event_sub); if (!event_sub) { + if (!devx_key_in_sub_list(&sub_list, key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + key_level1, + obj, + obj_id); err = -ENOMEM; goto err; } + event_sub->ev_file = ev_file; + event_sub->xa_key_level1 = key_level1; list_add_tail(&event_sub->event_list, &sub_list); uverbs_uobject_get(&ev_file->uobj); if (use_eventfd) { @@ -2178,9 +2196,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( } event_sub->cookie = cookie; - event_sub->ev_file = ev_file; - /* May be needed upon cleanup the devx object/subscription */ - event_sub->xa_key_level1 = key_level1; event_sub->xa_key_level2 = obj_id; INIT_LIST_HEAD(&event_sub->obj_list); } @@ -2225,10 +2240,11 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { list_del(&event_sub->event_list); - subscribe_event_xa_dealloc(devx_event_table, - event_sub->xa_key_level1, - obj, - obj_id); + if (!devx_key_in_sub_list(&sub_list, event_sub->xa_key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + event_sub->xa_key_level1, + obj, + obj_id); if (event_sub->eventfd) eventfd_ctx_put(event_sub->eventfd); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 61078281953d6..1144bfee58048 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2069,9 +2069,13 @@ static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn, if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) || (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) && !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - return err; + return 0; + + err = mlx5_ib_enable_lb(dev, true, false); + if (err) + mlx5_cmd_dealloc_transport_domain(dev->mdev, *tdn, uid); - return mlx5_ib_enable_lb(dev, true, false); + return err; } static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn, @@ -2658,7 +2662,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd, if (!dyn_uar) return err; - mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid); + mlx5_cmd_uar_dealloc(dev->mdev, uar_index, context->devx_uid); free_bfreg: mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx); @@ -4488,6 +4492,8 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) dev->port[i].roce.last_port_state = IB_PORT_DOWN; } + mutex_init(&dev->lb.mutex); + err = mlx5r_cmd_query_special_mkeys(dev); if (err) return err; @@ -4788,11 +4794,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) if (err) return err; - if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) && - (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) || - MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - mutex_init(&dev->lb.mutex); - if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) { err = mlx5_ib_init_var_region(dev); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e156dc4d75299..0a2b8ede0d818 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -331,6 +331,10 @@ struct mlx5_ib_flow_db { #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4 #define MLX5_IB_WR_UMR IB_WR_RESERVED1 +/* + * A valid pdn is required when flags include MLX5_IB_UPD_XLT_ENABLE, + * MLX5_IB_UPD_XLT_PD or MLX5_IB_UPD_XLT_ACCESS. + */ #define MLX5_IB_UPD_XLT_ZAP BIT(0) #define MLX5_IB_UPD_XLT_ENABLE BIT(1) #define MLX5_IB_UPD_XLT_ATOMIC BIT(2) @@ -1209,6 +1213,11 @@ static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) return container_of(ibpd, struct mlx5_ib_pd, ibpd); } +static inline u32 mlx5_mr_pdn(struct mlx5_ib_mr *mr) +{ + return to_mpd(mr->ibmr.pd)->pdn; +} + static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq) { return container_of(ibsrq, struct mlx5_ib_srq, ibsrq); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index fb40b44496f47..4118cd2d15fd5 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -31,6 +31,7 @@ * SOFTWARE. */ +#include #include #include #include @@ -163,9 +164,8 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, #define MLX5_FRMR_POOLS_KEY_VENDOR_KEY_SUPPORTED \ MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT 16 -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK 0xFF0000 -#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK 0xFFFF +#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK GENMASK_ULL(23, 16) +#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK GENMASK_ULL(15, 0) static struct mlx5_ib_mr * _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, @@ -194,7 +194,8 @@ _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, ph ^= MLX5_IB_NO_PH; mr->ibmr.frmr.key.kernel_vendor_key = - st_index | (ph << MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT); + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) | + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, ph); err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr); if (err) { kfree(mr); @@ -271,9 +272,10 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, get_mkc_octo_size(access_mode, key->num_dma_blocks)); MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); - st_index = key->kernel_vendor_key & - MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK; - ph = key->kernel_vendor_key & MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK; + st_index = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, + key->kernel_vendor_key); + ph = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, + key->kernel_vendor_key); if (ph) { /* Normalize ph: swap MLX5_IB_NO_PH for 0 */ if (ph == MLX5_IB_NO_PH) @@ -294,7 +296,7 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, free_in: kfree(in); if (err) - for (; i > 0; i--) + for (i--; i >= 0; i--) mlx5_core_destroy_mkey(dev->mdev, handles[i]); return err; } @@ -781,7 +783,8 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, * configured properly but left disabled. It is safe to go ahead * and configure it again via UMR while enabling it. */ - err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE); + err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE, + to_mpd(pd)->pdn); if (err) { mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ERR_PTR(err); @@ -890,7 +893,8 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach) if (!umem_dmabuf->sgt || !mr) return; - mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP); + /* MLX5_IB_UPD_XLT_ZAP does not change the pdn */ + mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP, 0); ib_umem_dmabuf_unmap_pages(umem_dmabuf); } @@ -1128,10 +1132,8 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, if (err) return err; - if (flags & IB_MR_REREG_PD) { - mr->ibmr.pd = pd; + if (flags & IB_MR_REREG_PD) upd_flags |= MLX5_IB_UPD_XLT_PD; - } if (flags & IB_MR_REREG_ACCESS) { mr->access_flags = access_flags; upd_flags |= MLX5_IB_UPD_XLT_ACCESS; @@ -1141,7 +1143,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, mr->ibmr.length = new_umem->length; mr->page_shift = order_base_2(page_size); mr->umem = new_umem; - err = mlx5r_umr_update_mr_pas(mr, upd_flags); + err = mlx5r_umr_update_mr_pas(mr, upd_flags, to_mpd(pd)->pdn); if (err) { /* * The MR is revoked at this point so there is no issue to free @@ -1188,6 +1190,21 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, if (!(flags & IB_MR_REREG_PD)) new_pd = ib_mr->pd; + if (mr->is_odp_implicit && !(flags & IB_MR_REREG_TRANS)) { + if (!(new_access_flags & IB_ACCESS_ON_DEMAND)) + return ERR_PTR(-EOPNOTSUPP); + + /* + * Due to all the child mkeys we cannot actually change an + * implicit MR in place. If the user did not specify a new + * translation then force the fixed implicit MR values. + */ + start = 0; + iova = 0; + length = U64_MAX; + flags |= IB_MR_REREG_TRANS; + } + if (!(flags & IB_MR_REREG_TRANS)) { struct ib_umem *umem; @@ -1202,7 +1219,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, } /* DM or ODP MR's don't have a normal umem so we can't re-use it */ if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) - goto recreate; + return ERR_PTR(-EOPNOTSUPP); /* * Only one active MR can refer to a umem at one time, revoke @@ -1381,9 +1398,12 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) bool is_odp = is_odp_mr(mr); int ret; - if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr) && - !ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr)) - return 0; + if (mr->ibmr.frmr.pool) { + if (!mlx5_umr_revoke_mr_with_lock(mr)) { + ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr); + return 0; + } + } if (is_odp) mutex_lock(&to_ib_umem_odp(mr->umem)->umem_mutex); @@ -1404,6 +1424,10 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) dma_resv_unlock( to_ib_umem_dmabuf(mr->umem)->attach->dmabuf->resv); } + + if (mr->ibmr.frmr.pool && !ret) + ib_frmr_pool_drop(&mr->ibmr); + return ret; } diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1119ce163ea78..42235b9dd6ab1 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -833,12 +833,14 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, u32 *bytes_mapped, u32 flags) { struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); + struct mlx5_ib_dev *dev = mr_to_mdev(mr); int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT; unsigned int old_page_shift = mr->page_shift; unsigned int page_shift; unsigned long page_size; u32 xlt_flags = 0; + u32 pdn = 0; int err; if (flags & MLX5_PF_FLAGS_ENABLE) @@ -857,8 +859,12 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = -EINVAL; } else { page_shift = order_base_2(page_size); + if (mr->data_direct) + pdn = dev->ddr.pdn; + else + pdn = mlx5_mr_pdn(mr); if (page_shift != mr->page_shift && mr->dmabuf_faulted) { - err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, + err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, pdn, page_shift); } else { mr->page_shift = page_shift; @@ -866,8 +872,8 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = mlx5r_umr_update_data_direct_ksm_pas( mr, xlt_flags); else - err = mlx5r_umr_update_mr_pas(mr, - xlt_flags); + err = mlx5r_umr_update_mr_pas(mr, xlt_flags, + pdn); } } dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 8fd05532c09cc..3dc0c85587fff 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -451,16 +451,13 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, if (ucmd) { qp->rq.wqe_cnt = ucmd->rq_wqe_count; - if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift)) - return -EINVAL; qp->rq.wqe_shift = ucmd->rq_wqe_shift; - if ((1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) < - wq_sig) + if (check_shl_overflow(1, qp->rq.wqe_shift, &wqe_size)) + return -EINVAL; + if (wqe_size / sizeof(struct mlx5_wqe_data_seg) < wq_sig) return -EINVAL; qp->rq.max_gs = - (1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) - + wqe_size / sizeof(struct mlx5_wqe_data_seg) - wq_sig; qp->rq.max_post = qp->rq.wqe_cnt; } else { @@ -3109,12 +3106,14 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, int err; if (params->is_rss_raw) { + rdma_restrack_no_track(&qp->ibqp.res); err = create_rss_raw_qp_tir(dev, pd, qp, params); goto out; } switch (qp->type) { case MLX5_IB_QPT_DCT: + rdma_restrack_no_track(&qp->ibqp.res); err = create_dct(dev, pd, qp, params); break; case MLX5_IB_QPT_DCI: diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c index 67841922c7b87..00a9bcb2603f0 100644 --- a/drivers/infiniband/hw/mlx5/restrack.c +++ b/drivers/infiniband/hw/mlx5/restrack.c @@ -178,9 +178,6 @@ static int fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp) ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "REG_UMR"); break; - case MLX5_IB_QPT_DCT: - ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCT"); - break; case MLX5_IB_QPT_DCI: ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCI"); break; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index f2139474be375..f3f428f5e1b66 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -603,11 +603,11 @@ mlx5r_umr_set_update_xlt_ctrl_seg(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg, static void mlx5r_umr_set_update_xlt_mkey_seg(struct mlx5_ib_dev *dev, struct mlx5_mkey_seg *mkey_seg, - struct mlx5_ib_mr *mr, + struct mlx5_ib_mr *mr, u32 pdn, unsigned int page_shift) { mlx5r_umr_set_access_flags(dev, mkey_seg, mr->access_flags); - MLX5_SET(mkc, mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); + MLX5_SET(mkc, mkey_seg, pd, pdn); MLX5_SET64(mkc, mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, mkey_seg, len, mr->ibmr.length); MLX5_SET(mkc, mkey_seg, log_page_size, page_shift); @@ -670,23 +670,22 @@ static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev, wqe->data_seg.byte_count = cpu_to_be32(sg->length); } -static void -_mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, struct mlx5r_umr_wqe *wqe, - struct ib_sge *sg, unsigned int flags, - unsigned int page_shift, bool dd) +static void _mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, + struct mlx5r_umr_wqe *wqe, struct ib_sge *sg, + unsigned int flags, u32 pdn, + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); mlx5r_umr_set_update_xlt_ctrl_seg(&wqe->ctrl_seg, flags, sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, page_shift); - if (dd) /* Use the data direct internal kernel PD */ - MLX5_SET(mkc, &wqe->mkey_seg, pd, dev->ddr.pdn); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe->data_seg, sg); } -static int -_mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, - size_t start_block, size_t nblocks) +static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, + u32 pdn, bool dd, size_t start_block, + size_t nblocks) { size_t ent_size = dd ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt); struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -720,7 +719,7 @@ _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, orig_sg_length = sg.length; - _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, mr->page_shift, dd); + _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, pdn, mr->page_shift); /* Set initial translation offset to start_block */ offset = (u64)start_block * ent_size; @@ -811,7 +810,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, true, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + true, start_block, nblocks); } int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, @@ -821,12 +821,13 @@ int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, } int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks) + u32 pdn, size_t start_block, size_t nblocks) { if (WARN_ON(mr->umem->is_odp)) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, false, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, pdn, false, start_block, + nblocks); } /* @@ -834,9 +835,9 @@ int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, * Dmabuf MR is handled in a similar way, except that the MLX5_IB_UPD_XLT_ZAP * flag may be used. */ -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn) { - return mlx5r_umr_update_mr_pas_range(mr, flags, 0, 0); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, 0, 0); } static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) @@ -861,6 +862,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, size_t orig_sg_length; size_t pages_iter; struct ib_sge sg; + u32 pdn = mlx5_mr_pdn(mr); int err = 0; void *xlt; @@ -895,7 +897,8 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, } mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, page_shift); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg); for (pages_mapped = 0; @@ -915,7 +918,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, */ err = mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); if (err) - return err; + break; dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE); sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT); @@ -937,8 +940,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, * pinned and the HW can switch from 4K to huge-page alignment). */ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd) + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); struct mlx5r_umr_wqe wqe = {}; @@ -953,16 +955,8 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, /* Fill mkey segment with the new page size, keep the rest unchanged */ MLX5_SET(mkc, &wqe.mkey_seg, log_page_size, page_shift); - if (dd) - MLX5_SET(mkc, &wqe.mkey_seg, pd, dev->ddr.pdn); - else - MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); - MLX5_SET64(mkc, &wqe.mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, &wqe.mkey_seg, len, mr->ibmr.length); - MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff); - MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0, - mlx5_mkey_variant(mr->mmkey.key)); err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false); if (!err) @@ -971,17 +965,18 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, return err; } -static inline int -_mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks, bool dd) +static inline int _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, + unsigned int flags, u32 pdn, + size_t start_block, + size_t nblocks, bool dd) { if (dd) return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags, start_block, nblocks); else - return mlx5r_umr_update_mr_pas_range(mr, flags, start_block, - nblocks); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, + start_block, nblocks); } /** @@ -995,11 +990,9 @@ _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, * Return: On success, returns the number of entries that were zapped. * On error, returns a negative error code. */ -static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, - unsigned int flags, - unsigned int page_shift, - size_t *nblocks, - bool dd) +static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, unsigned int flags, + unsigned int page_shift, size_t *nblocks, + u32 pdn, bool dd) { unsigned int old_page_shift = mr->page_shift; struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -1039,7 +1032,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, */ if (*nblocks) mr->page_shift = max_page_shift; - err = _mlx5r_dmabuf_umr_update_pas(mr, flags, 0, *nblocks, dd); + err = _mlx5r_dmabuf_umr_update_pas(mr, flags, pdn, 0, *nblocks, dd); if (err) { mr->page_shift = old_page_shift; return err; @@ -1049,7 +1042,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * non-present. */ if (*nblocks) { - err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift, dd); + err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift); if (err) { mr->page_shift = old_page_shift; return err; @@ -1064,6 +1057,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * entries accordingly * @mr: The memory region to update * @xlt_flags: Translation table update flags + * @pdn: Protection domain number * @page_shift: The new (optimized) page shift to use * * This function updates the page size and mkey translation entries for a DMABUF @@ -1083,7 +1077,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * * Returns 0 on success or a negative error code on failure. */ -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift) { unsigned int old_page_shift = mr->page_shift; @@ -1092,7 +1086,7 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, int err; err = _mlx5r_umr_zap_mkey(mr, xlt_flags, page_shift, &zapped_blocks, - mr->data_direct); + pdn, mr->data_direct); if (err) return err; @@ -1105,20 +1099,17 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, * the page size in the mkey yet. */ err = _mlx5r_dmabuf_umr_update_pas( - mr, - xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, - zapped_blocks, - total_blocks - zapped_blocks, + mr, xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, pdn, + zapped_blocks, total_blocks - zapped_blocks, mr->data_direct); if (err) goto err; } - err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift, - mr->data_direct); + err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift); if (err) goto err; - err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, 0, zapped_blocks, + err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, pdn, 0, zapped_blocks, mr->data_direct); if (err) goto err; diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h index 7eeaf6a94c974..99192ec67957c 100644 --- a/drivers/infiniband/hw/mlx5/umr.h +++ b/drivers/infiniband/hw/mlx5/umr.h @@ -101,14 +101,13 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, size_t nblocks); int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, unsigned int flags); int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks); -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags); + u32 pdn, size_t start_block, size_t nblocks); +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn); int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, int page_shift, int flags); int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd); -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, + unsigned int page_shift); +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift); #endif /* _MLX5_IB_UMR_H */ diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c index db380302149e5..7f723a2f37005 100644 --- a/drivers/infiniband/sw/rxe/rxe_mmap.c +++ b/drivers/infiniband/sw/rxe/rxe_mmap.c @@ -93,18 +93,31 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) goto done; found_it: + /* + * Increment refcount and check whether it is being freed atm while + * holding lock to prevent UAF + */ + if (!kref_get_unless_zero(&ip->ref)) { + spin_unlock_bh(&rxe->pending_lock); + ret = -ENXIO; + goto done; + } + list_del_init(&ip->pending_mmaps); spin_unlock_bh(&rxe->pending_lock); + vma->vm_ops = &rxe_vm_ops; + vma->vm_private_data = ip; + ret = remap_vmalloc_range(vma, ip->obj, 0); if (ret) { + vma->vm_private_data = NULL; + vma->vm_ops = NULL; + kref_put(&ip->ref, rxe_mmap_release); rxe_dbg_dev(rxe, "err %d from remap_vmalloc_range\n", ret); goto done; } - vma->vm_ops = &rxe_vm_ops; - vma->vm_private_data = ip; - rxe_vma_open(vma); done: return ret; } diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 50a2cb5405e22..082ff387d081a 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -283,12 +283,12 @@ static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port, tnl_cfg.encap_rcv = rxe_udp_encap_recv; /* Setup UDP tunnel */ - setup_udp_tunnel_sock(net, sock, &tnl_cfg); + setup_udp_tunnel_sock(net, sock->sk, &tnl_cfg); return sock; } -static void rxe_release_udp_tunnel(struct socket *sk) +static void rxe_release_udp_tunnel(struct sock *sk) { if (sk) udp_tunnel_sock_release(sk); @@ -636,7 +636,7 @@ static void rxe_sock_put(struct sock *sk, if (refcount_read(&sk->sk_refcnt) > SK_REF_FOR_TUNNEL) { __sock_put(sk); } else { - rxe_release_udp_tunnel(sk->sk_socket); + rxe_release_udp_tunnel(sk); sk = NULL; set_sk(net, sk); } diff --git a/drivers/infiniband/sw/rxe/rxe_ns.c b/drivers/infiniband/sw/rxe/rxe_ns.c index 8b9d734229b24..64621c89f8bf1 100644 --- a/drivers/infiniband/sw/rxe/rxe_ns.c +++ b/drivers/infiniband/sw/rxe/rxe_ns.c @@ -47,7 +47,7 @@ static void rxe_ns_exit(struct net *net) rcu_read_unlock(); if (sk) { rcu_assign_pointer(ns_sk->rxe_sk4, NULL); - udp_tunnel_sock_release(sk->sk_socket); + udp_tunnel_sock_release(sk); } #if IS_ENABLED(CONFIG_IPV6) @@ -56,7 +56,7 @@ static void rxe_ns_exit(struct net *net) rcu_read_unlock(); if (sk) { rcu_assign_pointer(ns_sk->rxe_sk6, NULL); - udp_tunnel_sock_release(sk->sk_socket); + udp_tunnel_sock_release(sk); } #endif } diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 9cb2f6fbf2dd6..e0294b7c95693 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -264,6 +264,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) struct rxe_recv_wqe *wqe; struct ib_event ev; unsigned int count; + unsigned int num_sge; size_t size; unsigned long flags; @@ -279,12 +280,13 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) } /* don't trust user space data */ - if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) { + num_sge = wqe->dma.num_sge; + if (unlikely(num_sge > srq->rq.max_sge)) { spin_unlock_irqrestore(&srq->rq.consumer_lock, flags); rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n"); return RESPST_ERR_MALFORMED_WQE; } - size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge); + size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); memcpy(&qp->resp.srq_wqe, wqe, size); qp->resp.wqe = &qp->resp.srq_wqe.wqe; @@ -308,6 +310,29 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) return RESPST_CHK_LENGTH; } +static enum resp_states rxe_get_recv_wqe(struct rxe_qp *qp) +{ + struct rxe_queue *q = qp->rq.queue; + struct rxe_recv_wqe *wqe; + unsigned int num_sge; + size_t size; + + wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT); + if (!wqe) + return RESPST_ERR_RNR; + + num_sge = wqe->dma.num_sge; + if (unlikely(num_sge > qp->rq.max_sge)) { + rxe_dbg_qp(qp, "invalid num_sge in recv WQE\n"); + return RESPST_ERR_MALFORMED_WQE; + } + size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); + memcpy(&qp->resp.srq_wqe, wqe, size); + + qp->resp.wqe = &qp->resp.srq_wqe.wqe; + return RESPST_CHK_LENGTH; +} + static enum resp_states check_resource(struct rxe_qp *qp, struct rxe_pkt_info *pkt) { @@ -328,9 +353,7 @@ static enum resp_states check_resource(struct rxe_qp *qp, if (srq) return get_srq_wqe(qp); - qp->resp.wqe = queue_head(qp->rq.queue, - QUEUE_TYPE_FROM_CLIENT); - return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR; + return rxe_get_recv_wqe(qp); } return RESPST_CHK_LENGTH; diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c index f7ac81c0f267a..87c79527ac090 100644 --- a/drivers/infiniband/sw/siw/siw_cm.c +++ b/drivers/infiniband/sw/siw/siw_cm.c @@ -138,6 +138,7 @@ static void siw_socket_disassoc(struct socket *s) cep = sk_to_cep(sk); if (cep) { siw_sk_restore_upcalls(sk, cep); + cep->sock = NULL; siw_cep_put(cep); } else { pr_warn("siw: cannot restore sk callbacks: no ep\n"); @@ -418,10 +419,11 @@ static void siw_free_cm_id(struct siw_cep *cep) static void siw_destroy_cep_sock(struct siw_cep *cep) { - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + struct socket *s = cep->sock; + + if (s) { + siw_socket_disassoc(s); + sock_release(s); } } @@ -1050,7 +1052,6 @@ static void siw_accept_newconn(struct siw_cep *cep) if (new_s) { siw_socket_disassoc(new_s); sock_release(new_s); - new_cep->sock = NULL; } siw_dbg_cep(cep, "error %d\n", rv); } @@ -1202,6 +1203,8 @@ static void siw_cm_work_handler(struct work_struct *w) WARN(1, "Undefined CM work type: %d\n", work->type); } if (release_cep) { + struct socket *s = cep->sock; + siw_dbg_cep(cep, "release: timer=%s, QP[%u]\n", cep->mpa_timer ? "y" : "n", @@ -1227,10 +1230,9 @@ static void siw_cm_work_handler(struct work_struct *w) cep->qp = NULL; siw_qp_put(qp); } - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + if (s) { + siw_socket_disassoc(s); + sock_release(s); } if (cep->cm_id) { siw_free_cm_id(cep); @@ -1561,7 +1563,6 @@ int siw_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) if (cep) { siw_socket_disassoc(s); sock_release(s); - cep->sock = NULL; cep->qp = NULL; @@ -1937,7 +1938,6 @@ int siw_create_listen(struct iw_cm_id *id, int backlog) siw_cep_set_inuse(cep); siw_free_cm_id(cep); - cep->sock = NULL; siw_socket_disassoc(s); cep->state = SIW_EPSTATE_CLOSED; @@ -1959,6 +1959,7 @@ static void siw_drop_listeners(struct iw_cm_id *id) */ list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) { struct siw_cep *cep = list_entry(p, struct siw_cep, listenq); + struct socket *s = cep->sock; list_del(p); @@ -1967,10 +1968,9 @@ static void siw_drop_listeners(struct iw_cm_id *id) siw_cep_set_inuse(cep); siw_free_cm_id(cep); - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + if (s) { + siw_socket_disassoc(s); + sock_release(s); } cep->state = SIW_EPSTATE_CLOSED; siw_cep_set_free_and_put(cep); diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index 34d03584160c2..b566d163c5aab 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -844,6 +844,15 @@ int siw_proc_rresp(struct siw_qp *qp) } mem_p = *mem; + if (unlikely(wqe->processed + srx->fpdu_part_rem > wqe->bytes)) { + siw_dbg_qp(qp, "rresp len: %d + %d > %d\n", + wqe->processed, srx->fpdu_part_rem, wqe->bytes); + wqe->wc_status = SIW_WC_LOC_LEN_ERR; + siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, + DDP_ETYPE_TAGGED_BUF, + DDP_ECODE_T_BASE_BOUNDS, 0); + return -EINVAL; + } bytes = min(srx->fpdu_part_rem, srx->skb_new); rv = siw_rx_data(mem_p, srx, &frx->pbl_idx, sge->laddr + wqe->processed, bytes); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 6482ad859bd10..f81e122a3ccb6 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -225,8 +225,9 @@ static int rdma_write_sg(struct rtrs_srv_op *id) /* WR will fail with length error * if this is 0 */ - if (plist->length == 0) { - rtrs_err(s, "Invalid RDMA-Write sg list length 0\n"); + if (plist->length == 0 || plist->length > max_chunk_size) { + rtrs_err(s, "Invalid RDMA-Write sg list length %u\n", + plist->length); return -EINVAL; } diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 9aec5d80117f1..f66cfd70c2636 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1129,9 +1129,10 @@ static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx, struct srp_imm_buf *imm_buf = srpt_get_desc_buf(srp_cmd); void *data = (void *)srp_cmd + imm_data_offset; uint32_t len = be32_to_cpu(imm_buf->len); - uint32_t req_size = imm_data_offset + len; + uint32_t req_size; - if (req_size > srp_max_req_size) { + if (check_add_overflow((uint32_t)imm_data_offset, len, &req_size) || + req_size > srp_max_req_size) { pr_err("Immediate data (length %d + %d) exceeds request size %d\n", imm_data_offset, len, srp_max_req_size); return -EINVAL; diff --git a/drivers/input/joystick/iforce/iforce-packets.c b/drivers/input/joystick/iforce/iforce-packets.c index fd1cd731d781a..effa76bfd8f9a 100644 --- a/drivers/input/joystick/iforce/iforce-packets.c +++ b/drivers/input/joystick/iforce/iforce-packets.c @@ -186,14 +186,18 @@ void iforce_process_packet(struct iforce *iforce, /* Check if an effect was just started or stopped */ i = data[1] & 0x7f; - if (data[1] & 0x80) { - if (!test_and_set_bit(FF_CORE_IS_PLAYED, iforce->core_effects[i].flags)) { - /* Report play event */ - input_report_ff_status(dev, i, FF_STATUS_PLAYING); + if (i < IFORCE_EFFECTS_MAX) { + if (data[1] & 0x80) { + if (!test_and_set_bit(FF_CORE_IS_PLAYED, + iforce->core_effects[i].flags)) { + /* Report play event */ + input_report_ff_status(dev, i, FF_STATUS_PLAYING); + } + } else if (test_and_clear_bit(FF_CORE_IS_PLAYED, + iforce->core_effects[i].flags)) { + /* Report stop event */ + input_report_ff_status(dev, i, FF_STATUS_STOPPED); } - } else if (test_and_clear_bit(FF_CORE_IS_PLAYED, iforce->core_effects[i].flags)) { - /* Report stop event */ - input_report_ff_status(dev, i, FF_STATUS_STOPPED); } for (j = 3; j < len; j += 2) diff --git a/drivers/input/joystick/maplecontrol.c b/drivers/input/joystick/maplecontrol.c index 7f36f73844a91..6293b6e8148b3 100644 --- a/drivers/input/joystick/maplecontrol.c +++ b/drivers/input/joystick/maplecontrol.c @@ -112,6 +112,8 @@ static int probe_maple_controller(struct device *dev) pad->dev = idev; pad->mdev = mdev; + maple_set_drvdata(mdev, pad); + idev->open = dc_pad_open; idev->close = dc_pad_close; @@ -146,7 +148,6 @@ static int probe_maple_controller(struct device *dev) goto fail; mdev->driver = mdrv; - maple_set_drvdata(mdev, pad); return 0; diff --git a/drivers/input/keyboard/maple_keyb.c b/drivers/input/keyboard/maple_keyb.c index 80a5181313e15..3d5538dd4f237 100644 --- a/drivers/input/keyboard/maple_keyb.c +++ b/drivers/input/keyboard/maple_keyb.c @@ -166,6 +166,8 @@ static int probe_maple_kbd(struct device *dev) kbd->dev = idev; memcpy(kbd->keycode, dc_kbd_keycode, sizeof(kbd->keycode)); + maple_set_drvdata(mdev, kbd); + idev->name = mdev->product_name; idev->evbit[0] = BIT(EV_KEY) | BIT(EV_REP); idev->keycode = kbd->keycode; @@ -190,8 +192,6 @@ static int probe_maple_kbd(struct device *dev) mdev->driver = mdrv; - maple_set_drvdata(mdev, kbd); - return error; fail_register: diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c index 7a1cb9333f53c..b1ff8c70877fe 100644 --- a/drivers/input/misc/ims-pcu.c +++ b/drivers/input/misc/ims-pcu.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -406,7 +407,16 @@ static void ims_pcu_destroy_gamepad(struct ims_pcu *pcu) static void ims_pcu_report_events(struct ims_pcu *pcu) { - u32 data = get_unaligned_be32(&pcu->read_buf[3]); + u32 data; + + /* 6-axis setting (1 byte) + button data + checksum */ + if (pcu->read_pos < IMS_PCU_DATA_OFFSET + 1 + sizeof(data) + 1) { + dev_warn(pcu->dev, "Short buttons report: %d bytes\n", + pcu->read_pos); + return; + } + + data = get_unaligned_be32(&pcu->read_buf[IMS_PCU_DATA_OFFSET + 1]); ims_pcu_buttons_report(pcu, data & ~IMS_PCU_GAMEPAD_MASK); if (pcu->gamepad) @@ -440,7 +450,7 @@ static void ims_pcu_handle_response(struct ims_pcu *pcu) static void ims_pcu_reset_packet(struct ims_pcu *pcu) { - pcu->have_stx = true; + pcu->have_stx = false; pcu->have_dle = false; pcu->read_pos = 0; pcu->check_sum = 0; @@ -686,11 +696,19 @@ static int __ims_pcu_execute_bl_command(struct ims_pcu *pcu, return error; } - if (expected_response && pcu->cmd_buf[2] != expected_response) { - dev_err(pcu->dev, - "Unexpected response from bootloader: 0x%02x, wanted 0x%02x\n", - pcu->cmd_buf[2], expected_response); - return -EINVAL; + if (expected_response) { + if (pcu->cmd_buf_len < 3) { + dev_err(pcu->dev, "Short response from bootloader: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + + if (pcu->cmd_buf[2] != expected_response) { + dev_err(pcu->dev, + "Unexpected response from bootloader: 0x%02x, wanted 0x%02x\n", + pcu->cmd_buf[2], expected_response); + return -EINVAL; + } } return 0; @@ -718,6 +736,12 @@ static int ims_pcu_get_info(struct ims_pcu *pcu) return error; } + if (pcu->cmd_buf_len < IMS_PCU_DATA_OFFSET + IMS_PCU_SET_INFO_SIZE + 1) { + dev_err(pcu->dev, "Short GET_INFO response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + memcpy(pcu->part_number, &pcu->cmd_buf[IMS_PCU_INFO_PART_OFFSET], sizeof(pcu->part_number)); @@ -815,6 +839,12 @@ static int ims_pcu_verify_block(struct ims_pcu *pcu, return error; } + if (pcu->cmd_buf_len < IMS_PCU_BL_DATA_OFFSET + sizeof(*fragment) + len + 1) { + dev_err(pcu->dev, "Short READ_APP response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + fragment = (void *)&pcu->cmd_buf[IMS_PCU_BL_DATA_OFFSET]; if (get_unaligned_le32(&fragment->addr) != addr || fragment->len != len) { @@ -934,9 +964,10 @@ static int ims_pcu_handle_firmware_update(struct ims_pcu *pcu, return retval; } -static void ims_pcu_process_async_firmware(const struct firmware *fw, +static void ims_pcu_process_async_firmware(const struct firmware *_fw, void *context) { + const struct firmware *fw __free(firmware) = _fw; struct ims_pcu *pcu = context; int error; @@ -956,8 +987,6 @@ static void ims_pcu_process_async_firmware(const struct firmware *fw, scoped_guard(mutex, &pcu->cmd_mutex) ims_pcu_handle_firmware_update(pcu, fw); - release_firmware(fw); - out: complete(&pcu->async_firmware_done); } @@ -1009,6 +1038,10 @@ ims_pcu_backlight_get_brightness(struct led_classdev *cdev) error); /* Assume the LED is OFF */ brightness = LED_OFF; + } else if (pcu->cmd_buf_len < IMS_PCU_DATA_OFFSET + 2 + 1) { + dev_err(pcu->dev, "Short GET_BRIGHTNESS response: %d bytes\n", + pcu->cmd_buf_len); + brightness = LED_OFF; } else { brightness = get_unaligned_le16(&pcu->cmd_buf[IMS_PCU_DATA_OFFSET]); @@ -1153,6 +1186,8 @@ static ssize_t ims_pcu_reset_device(struct device *dev, dev_info(pcu->dev, "Attempting to reset device\n"); + guard(mutex)(&pcu->cmd_mutex); + error = ims_pcu_execute_command(pcu, PCU_RESET, &reset_byte, 1); if (error) { dev_info(pcu->dev, @@ -1249,6 +1284,9 @@ static umode_t ims_pcu_is_attr_visible(struct kobject *kobj, struct ims_pcu *pcu = usb_get_intfdata(intf); umode_t mode = attr->mode; + if (intf != pcu->ctrl_intf) + return 0; + if (pcu->bootloader_mode) { if (attr != &dev_attr_update_firmware_status.attr && attr != &dev_attr_update_firmware.attr && @@ -1282,6 +1320,12 @@ static int ims_pcu_read_ofn_config(struct ims_pcu *pcu, u8 addr, u8 *data) if (error) return error; + if (pcu->cmd_buf_len < OFN_REG_RESULT_OFFSET + 2 + 1) { + dev_err(pcu->dev, "Short OFN_GET_CONFIG response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + result = (s16)get_unaligned_le16(pcu->cmd_buf + OFN_REG_RESULT_OFFSET); if (result < 0) return -EIO; @@ -1302,6 +1346,12 @@ static int ims_pcu_write_ofn_config(struct ims_pcu *pcu, u8 addr, u8 data) if (error) return error; + if (pcu->cmd_buf_len < OFN_REG_RESULT_OFFSET + 2 + 1) { + dev_err(pcu->dev, "Short OFN_SET_CONFIG response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + result = (s16)get_unaligned_le16(pcu->cmd_buf + OFN_REG_RESULT_OFFSET); if (result < 0) return -EIO; @@ -1488,6 +1538,9 @@ static umode_t ims_pcu_ofn_is_attr_visible(struct kobject *kobj, struct ims_pcu *pcu = usb_get_intfdata(intf); umode_t mode = attr->mode; + if (intf != pcu->ctrl_intf) + return SYSFS_GROUP_INVISIBLE; + /* * PCU-B devices, both GEN_1 and GEN_2 do not have OFN sensor. */ @@ -1528,7 +1581,7 @@ static void ims_pcu_irq(struct urb *urb) } dev_dbg(pcu->dev, "%s: received %d: %*ph\n", __func__, - urb->actual_length, urb->actual_length, pcu->urb_in_buf); + urb->actual_length, urb->actual_length, urb->transfer_buffer); if (urb == pcu->urb_in) ims_pcu_process_data(pcu, urb); @@ -1656,8 +1709,9 @@ ims_pcu_get_cdc_union_desc(struct usb_interface *intf) while (buflen >= sizeof(*union_desc)) { union_desc = (struct usb_cdc_union_desc *)buf; - if (union_desc->bLength > buflen) { - dev_err(&intf->dev, "Too large descriptor\n"); + if (union_desc->bLength < 2 || union_desc->bLength > buflen) { + dev_err(&intf->dev, "Invalid descriptor length: %d\n", + union_desc->bLength); return NULL; } @@ -1693,7 +1747,7 @@ static int ims_pcu_parse_cdc_data(struct usb_interface *intf, struct ims_pcu *pc pcu->ctrl_intf = usb_ifnum_to_if(pcu->udev, union_desc->bMasterInterface0); - if (!pcu->ctrl_intf) + if (pcu->ctrl_intf != intf) return -EINVAL; alt = pcu->ctrl_intf->cur_altsetting; @@ -1702,6 +1756,12 @@ static int ims_pcu_parse_cdc_data(struct usb_interface *intf, struct ims_pcu *pc return -ENODEV; pcu->ep_ctrl = &alt->endpoint[0].desc; + if (!usb_endpoint_is_int_in(pcu->ep_ctrl)) { + dev_err(pcu->dev, + "Control endpoint is not INTERRUPT IN\n"); + return -EINVAL; + } + pcu->max_ctrl_size = usb_endpoint_maxp(pcu->ep_ctrl); pcu->data_intf = usb_ifnum_to_if(pcu->udev, @@ -1783,11 +1843,16 @@ static void ims_pcu_stop_io(struct ims_pcu *pcu) static int ims_pcu_line_setup(struct ims_pcu *pcu) { struct usb_host_interface *interface = pcu->ctrl_intf->cur_altsetting; - struct usb_cdc_line_coding *line = (void *)pcu->cmd_buf; + struct usb_cdc_line_coding *line __free(kfree) = + kmalloc(sizeof(*line), GFP_KERNEL); int error; - memset(line, 0, sizeof(*line)); + if (!line) + return -ENOMEM; + line->dwDTERate = cpu_to_le32(57600); + line->bCharFormat = USB_CDC_1_STOP_BITS; + line->bParityType = USB_CDC_NO_PARITY; line->bDataBits = 8; error = usb_control_msg(pcu->udev, usb_sndctrlpipe(pcu->udev, 0), @@ -1831,6 +1896,12 @@ static int ims_pcu_get_device_info(struct ims_pcu *pcu) return error; } + if (pcu->cmd_buf_len < IMS_PCU_DATA_OFFSET + 6 + 1) { + dev_err(pcu->dev, "Short GET_FW_VERSION response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + snprintf(pcu->fw_version, sizeof(pcu->fw_version), "%02d%02d%02d%02d.%c%c", pcu->cmd_buf[2], pcu->cmd_buf[3], pcu->cmd_buf[4], pcu->cmd_buf[5], @@ -1843,6 +1914,12 @@ static int ims_pcu_get_device_info(struct ims_pcu *pcu) return error; } + if (pcu->cmd_buf_len < IMS_PCU_DATA_OFFSET + 6 + 1) { + dev_err(pcu->dev, "Short GET_BL_VERSION response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + snprintf(pcu->bl_version, sizeof(pcu->bl_version), "%02d%02d%02d%02d.%c%c", pcu->cmd_buf[2], pcu->cmd_buf[3], pcu->cmd_buf[4], pcu->cmd_buf[5], @@ -1855,6 +1932,12 @@ static int ims_pcu_get_device_info(struct ims_pcu *pcu) return error; } + if (pcu->cmd_buf_len < IMS_PCU_DATA_OFFSET + 1 + 1) { + dev_err(pcu->dev, "Short RESET_REASON response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + snprintf(pcu->reset_reason, sizeof(pcu->reset_reason), "%02x", pcu->cmd_buf[IMS_PCU_DATA_OFFSET]); @@ -1881,6 +1964,12 @@ static int ims_pcu_identify_type(struct ims_pcu *pcu, u8 *device_id) return error; } + if (pcu->cmd_buf_len < IMS_PCU_DATA_OFFSET + 1 + 1) { + dev_err(pcu->dev, "Short GET_DEVICE_ID response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + *device_id = pcu->cmd_buf[IMS_PCU_DATA_OFFSET]; dev_dbg(pcu->dev, "Detected device ID: %d\n", *device_id); @@ -1972,6 +2061,12 @@ static int ims_pcu_init_bootloader_mode(struct ims_pcu *pcu) return error; } + if (pcu->cmd_buf_len < IMS_PCU_DATA_OFFSET + 15 + 4 + 1) { + dev_err(pcu->dev, "Short QUERY_DEVICE response: %d bytes\n", + pcu->cmd_buf_len); + return -EIO; + } + pcu->fw_start_addr = get_unaligned_le32(&pcu->cmd_buf[IMS_PCU_DATA_OFFSET + 11]); pcu->fw_end_addr = @@ -2071,7 +2166,6 @@ static int ims_pcu_probe(struct usb_interface *intf, static void ims_pcu_disconnect(struct usb_interface *intf) { struct ims_pcu *pcu = usb_get_intfdata(intf); - struct usb_host_interface *alt = intf->cur_altsetting; usb_set_intfdata(intf, NULL); @@ -2079,7 +2173,7 @@ static void ims_pcu_disconnect(struct usb_interface *intf) * See if we are dealing with control or data interface. The cleanup * happens when we unbind primary (control) interface. */ - if (alt->desc.bInterfaceClass != USB_CLASS_COMM) + if (intf != pcu->ctrl_intf) return; ims_pcu_stop_io(pcu); @@ -2090,6 +2184,7 @@ static void ims_pcu_disconnect(struct usb_interface *intf) ims_pcu_destroy_application_mode(pcu); ims_pcu_buffers_free(pcu); + usb_driver_release_interface(&ims_pcu_driver, pcu->data_intf); kfree(pcu); } diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c index 5cba02a156ce4..edb0a28d25aae 100644 --- a/drivers/input/mouse/elan_i2c_core.c +++ b/drivers/input/mouse/elan_i2c_core.c @@ -428,8 +428,17 @@ static int elan_query_device_parameters(struct elan_tp_data *data) if (error) return error; } - data->width_x = data->max_x / x_traces; - data->width_y = data->max_y / y_traces; + + if (!x_traces || !y_traces) { + dev_warn(&client->dev, + "invalid trace numbers: x=%u, y=%u\n", + x_traces, y_traces); + data->width_x = 1; + data->width_y = 1; + } else { + data->width_x = data->max_x / x_traces; + data->width_y = data->max_y / y_traces; + } if (device_property_read_u32(&client->dev, "touchscreen-x-mm", &x_mm) || @@ -443,8 +452,16 @@ static int elan_query_device_parameters(struct elan_tp_data *data) data->x_res = elan_convert_resolution(hw_x_res, data->pattern); data->y_res = elan_convert_resolution(hw_y_res, data->pattern); } else { - data->x_res = (data->max_x + 1) / x_mm; - data->y_res = (data->max_y + 1) / y_mm; + if (unlikely(x_mm == 0 || y_mm == 0)) { + dev_warn(&client->dev, + "invalid physical dimensions: x_mm=%u, y_mm=%u\n", + x_mm, y_mm); + data->x_res = 1; + data->y_res = 1; + } else { + data->x_res = (data->max_x + 1) / x_mm; + data->y_res = (data->max_y + 1) / y_mm; + } } if (device_property_read_bool(&client->dev, "elan,clickpad")) @@ -956,6 +973,7 @@ static void elan_report_contact(struct elan_tp_data *data, int contact_num, if (data->report_features & ETP_FEATURE_REPORT_MK) { unsigned int mk_x, mk_y, area_x, area_y; + int adj_width_x, adj_width_y; u8 mk_data = high_precision ? packet[ETP_MK_DATA_OFFSET + contact_num] : finger_data[3]; @@ -967,8 +985,14 @@ static void elan_report_contact(struct elan_tp_data *data, int contact_num, * To avoid treating large finger as palm, let's reduce * the width x and y per trace. */ - area_x = mk_x * (data->width_x - ETP_FWIDTH_REDUCE); - area_y = mk_y * (data->width_y - ETP_FWIDTH_REDUCE); + + adj_width_x = data->width_x > ETP_FWIDTH_REDUCE ? + data->width_x - ETP_FWIDTH_REDUCE : 0; + adj_width_y = data->width_y > ETP_FWIDTH_REDUCE ? + data->width_y - ETP_FWIDTH_REDUCE : 0; + + area_x = mk_x * adj_width_x; + area_y = mk_y * adj_width_y; input_report_abs(input, ABS_TOOL_WIDTH, mk_x); input_report_abs(input, ABS_MT_TOUCH_MAJOR, diff --git a/drivers/input/mouse/maplemouse.c b/drivers/input/mouse/maplemouse.c index c99f7e2342191..0c8f7d1b02aa3 100644 --- a/drivers/input/mouse/maplemouse.c +++ b/drivers/input/mouse/maplemouse.c @@ -48,7 +48,7 @@ static void dc_mouse_callback(struct mapleq *mq) static int dc_mouse_open(struct input_dev *dev) { - struct dc_mouse *mse = maple_get_drvdata(to_maple_dev(&dev->dev)); + struct dc_mouse *mse = input_get_drvdata(dev); maple_getcond_callback(mse->mdev, dc_mouse_callback, HZ/50, MAPLE_FUNC_MOUSE); @@ -58,7 +58,7 @@ static int dc_mouse_open(struct input_dev *dev) static void dc_mouse_close(struct input_dev *dev) { - struct dc_mouse *mse = maple_get_drvdata(to_maple_dev(&dev->dev)); + struct dc_mouse *mse = input_get_drvdata(dev); maple_getcond_callback(mse->mdev, dc_mouse_callback, 0, MAPLE_FUNC_MOUSE); @@ -88,6 +88,9 @@ static int probe_maple_mouse(struct device *dev) mse->dev = input_dev; mse->mdev = mdev; + maple_set_drvdata(mdev, mse); + + input_set_drvdata(input_dev, mse); input_dev->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_REL); input_dev->keybit[BIT_WORD(BTN_MOUSE)] = BIT_MASK(BTN_LEFT) | BIT_MASK(BTN_RIGHT) | BIT_MASK(BTN_MIDDLE); @@ -102,12 +105,12 @@ static int probe_maple_mouse(struct device *dev) goto fail_register; mdev->driver = mdrv; - maple_set_drvdata(mdev, mse); return error; fail_register: input_free_device(input_dev); + maple_set_drvdata(mdev, NULL); fail_nomem: kfree(mse); fail: diff --git a/drivers/input/rmi4/rmi_bus.c b/drivers/input/rmi4/rmi_bus.c index 687cb987bc137..ade57e2a72014 100644 --- a/drivers/input/rmi4/rmi_bus.c +++ b/drivers/input/rmi4/rmi_bus.c @@ -455,11 +455,13 @@ static int __init rmi_bus_init(void) if (error) { pr_err("%s: error registering the RMI physical driver: %d\n", __func__, error); - goto err_unregister_bus; + goto err_unregister_function_handlers; } return 0; +err_unregister_function_handlers: + rmi_unregister_function_handlers(); err_unregister_bus: bus_unregister(&rmi_bus_type); return error; diff --git a/drivers/input/rmi4/rmi_f30.c b/drivers/input/rmi4/rmi_f30.c index 35045f161dc2f..b2155c8e20e75 100644 --- a/drivers/input/rmi4/rmi_f30.c +++ b/drivers/input/rmi4/rmi_f30.c @@ -233,7 +233,7 @@ static int rmi_f30_map_gpios(struct rmi_function *fn, int button_count = min_t(u8, f30->gpioled_count, TRACKSTICK_RANGE_END); f30->gpioled_key_map = devm_kcalloc(&fn->dev, - button_count, + f30->gpioled_count, sizeof(f30->gpioled_key_map[0]), GFP_KERNEL); if (!f30->gpioled_key_map) { diff --git a/drivers/input/rmi4/rmi_f3a.c b/drivers/input/rmi4/rmi_f3a.c index 0e8baed84dbb5..a0777644eef07 100644 --- a/drivers/input/rmi4/rmi_f3a.c +++ b/drivers/input/rmi4/rmi_f3a.c @@ -132,7 +132,7 @@ static int rmi_f3a_map_gpios(struct rmi_function *fn, struct f3a_data *f3a, int button_count = min_t(u8, f3a->gpio_count, TRACKSTICK_RANGE_END); f3a->gpio_key_map = devm_kcalloc(&fn->dev, - button_count, + f3a->gpio_count, sizeof(f3a->gpio_key_map[0]), GFP_KERNEL); if (!f3a->gpio_key_map) { diff --git a/drivers/input/serio/gscps2.c b/drivers/input/serio/gscps2.c index 22b2f57fd91f5..bf9b993f57330 100644 --- a/drivers/input/serio/gscps2.c +++ b/drivers/input/serio/gscps2.c @@ -219,6 +219,7 @@ static void gscps2_read_data(struct gscps2port *ps2port) ps2port->buffer[ps2port->append].str = status; ps2port->buffer[ps2port->append].data = gscps2_readb_input(ps2port->addr); + ps2port->append = (ps2port->append + 1) & BUFFER_SIZE; } while (true); } diff --git a/drivers/input/touchscreen/ads7846.c b/drivers/input/touchscreen/ads7846.c index 4b39f7212d35c..1ae1ae42582af 100644 --- a/drivers/input/touchscreen/ads7846.c +++ b/drivers/input/touchscreen/ads7846.c @@ -325,7 +325,6 @@ struct ser_req { u8 ref_on; u8 command; u8 ref_off; - u16 scratch; struct spi_message msg; struct spi_transfer xfer[8]; /* @@ -333,6 +332,7 @@ struct ser_req { * transfer buffers to live in their own cache lines. */ __be16 sample ____cacheline_aligned; + u16 scratch; }; struct ads7845_ser_req { @@ -403,8 +403,7 @@ static int ads7846_read12_ser(struct device *dev, unsigned command) spi_message_add_tail(&req->xfer[5], &req->msg); /* clear the command register */ - req->scratch = 0; - req->xfer[6].tx_buf = &req->scratch; + req->xfer[6].rx_buf = &req->scratch; req->xfer[6].len = 1; spi_message_add_tail(&req->xfer[6], &req->msg); diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index f8798d11ec030..17fcfe45988c2 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -1057,7 +1057,8 @@ static void goodix_read_config(struct goodix_ts_data *ts) } ts->int_trigger_type = ts->config[TRIGGER_LOC] & 0x03; - ts->max_touch_num = ts->config[MAX_CONTACTS_LOC] & 0x0f; + ts->max_touch_num = min(ts->config[MAX_CONTACTS_LOC] & 0x0f, + GOODIX_MAX_CONTACTS); x_max = get_unaligned_le16(&ts->config[RESOLUTION_LOC]); y_max = get_unaligned_le16(&ts->config[RESOLUTION_LOC + 2]); diff --git a/drivers/input/touchscreen/mms114.c b/drivers/input/touchscreen/mms114.c index af462086a65ca..41859bfa28443 100644 --- a/drivers/input/touchscreen/mms114.c +++ b/drivers/input/touchscreen/mms114.c @@ -165,7 +165,7 @@ static void mms114_process_mt(struct mms114_data *data, struct mms114_touch *tou unsigned int x; unsigned int y; - if (touch->id > MMS114_MAX_TOUCH) { + if (touch->id == 0 || touch->id > MMS114_MAX_TOUCH) { dev_err(&client->dev, "Wrong touch id (%d)\n", touch->id); return; } @@ -217,7 +217,9 @@ static irqreturn_t mms114_interrupt(int irq, void *dev_id) struct mms114_data *data = dev_id; struct i2c_client *client = data->client; struct mms114_touch touch[MMS114_MAX_TOUCH]; + struct mms114_touch *t; int packet_size; + int event_size; int touch_size; int index; int error; @@ -226,11 +228,19 @@ static irqreturn_t mms114_interrupt(int irq, void *dev_id) if (packet_size <= 0) goto out; + if (packet_size > sizeof(touch)) { + dev_err(&client->dev, "Invalid packet size %d (max %zu)\n", + packet_size, sizeof(touch)); + goto out; + } + /* MMS136 has slightly different event size */ if (data->type == TYPE_MMS134S || data->type == TYPE_MMS136) - touch_size = packet_size / MMS136_EVENT_SIZE; + event_size = MMS136_EVENT_SIZE; else - touch_size = packet_size / MMS114_EVENT_SIZE; + event_size = MMS114_EVENT_SIZE; + + touch_size = packet_size / event_size; error = __mms114_read_reg(data, MMS114_INFORMATION, packet_size, (u8 *)touch); @@ -238,18 +248,20 @@ static irqreturn_t mms114_interrupt(int irq, void *dev_id) goto out; for (index = 0; index < touch_size; index++) { - switch (touch[index].type) { + t = (struct mms114_touch *)((u8 *)touch + index * event_size); + + switch (t->type) { case MMS114_TYPE_TOUCHSCREEN: - mms114_process_mt(data, touch + index); + mms114_process_mt(data, t); break; case MMS114_TYPE_TOUCHKEY: - mms114_process_touchkey(data, touch + index); + mms114_process_touchkey(data, t); break; default: dev_err(&client->dev, "Wrong touch type (%d)\n", - touch[index].type); + t->type); break; } } diff --git a/drivers/input/touchscreen/touchwin.c b/drivers/input/touchscreen/touchwin.c index 099fd88e65d84..3ed33378dfcd8 100644 --- a/drivers/input/touchscreen/touchwin.c +++ b/drivers/input/touchscreen/touchwin.c @@ -63,12 +63,15 @@ static irqreturn_t tw_interrupt(struct serio *serio, if (data) { /* touch */ tw->touched = 1; tw->data[tw->idx++] = data; - /* verify length and that the two Y's are the same */ - if (tw->idx == TW_LENGTH && tw->data[1] == tw->data[2]) { - input_report_abs(dev, ABS_X, tw->data[0]); - input_report_abs(dev, ABS_Y, tw->data[1]); - input_report_key(dev, BTN_TOUCH, 1); - input_sync(dev); + /* a full packet ends the accumulation, valid or not */ + if (tw->idx == TW_LENGTH) { + /* report only if the two Y's are the same */ + if (tw->data[1] == tw->data[2]) { + input_report_abs(dev, ABS_X, tw->data[0]); + input_report_abs(dev, ABS_Y, tw->data[1]); + input_report_key(dev, BTN_TOUCH, 1); + input_sync(dev); + } tw->idx = 0; } } else if (tw->touched) { /* untouch */ diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f9f7180878930..c726d115939a6 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -948,12 +948,13 @@ static inline int get_hpet_devid(int id) } enum amd_iommu_intr_mode_type { - AMD_IOMMU_GUEST_IR_LEGACY, - - /* This mode is not visible to users. It is used when - * we cannot fully enable vAPIC and fallback to only support - * legacy interrupt remapping via 128-bit IRTE. + /* + * The legacy format mode is not visible to users to prevent the user + * from crashing x2APIC systems, which for all intents and purposes + * require 128-bit IRTEs. The legacy format will be forced as needed + * when hardware doesn't support 128-bit IRTEs. */ + AMD_IOMMU_GUEST_IR_LEGACY, AMD_IOMMU_GUEST_IR_LEGACY_GA, AMD_IOMMU_GUEST_IR_VAPIC, }; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 3bdb380d23e9a..9a846dcd03063 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1939,12 +1939,11 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, /* XT and GAM require GA mode. */ if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) { amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; - break; + } else { + if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) + amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; } - if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) - amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; - if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { pr_warn_once("Host Address Translation is not supported.\n"); amd_iommu_hatdis = true; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 57dc8fabc7d9b..15ffc4742183e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1774,7 +1774,8 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, { lockdep_assert_held(&domain->lock); - if (likely(!amd_iommu_np_cache)) { + if (likely(!amd_iommu_np_cache) || + size >= (1ULL<<52)) { __domain_flush_pages(domain, address, size); /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index edd41b5a3b6ac..2d006049dd610 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -39,7 +39,7 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,adreno-smmu", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, - { .compatible = "qcom,fastrpc", + { .compatible = "qcom,fastrpc-compute-cb", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,qcm2290-mdss", .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 381b60d9e7cea..3ca2c5da2843b 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1465,7 +1465,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, */ s->dma_address = pci_p2pdma_bus_addr_map( p2pdma_state.mem, sg_phys(s)); - sg_dma_len(s) = sg->length; + sg_dma_len(s) = s->length; sg_dma_mark_bus_address(s); continue; default: diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 4d0e65bc131d7..849d06dfe1aec 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -157,7 +157,10 @@ static void device_rbtree_remove(struct device_domain_info *info) unsigned long flags; spin_lock_irqsave(&iommu->device_rbtree_lock, flags); - rb_erase(&info->node, &iommu->device_rbtree); + if (!RB_EMPTY_NODE(&info->node)) { + rb_erase(&info->node, &iommu->device_rbtree); + RB_CLEAR_NODE(&info->node); + } spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); } @@ -3254,6 +3257,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) info->dev = dev; info->iommu = iommu; + RB_CLEAR_NODE(&info->node); if (dev_is_pci(dev)) { if (ecap_dev_iotlb_support(iommu->ecap) && pci_ats_supported(pdev) && diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index ef145560aa987..775f1c4ae3463 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1254,18 +1254,29 @@ void intel_iommu_disable_iopf(struct device *dev); static inline int iopf_for_domain_set(struct iommu_domain *domain, struct device *dev) { + struct device_domain_info *info = dev_iommu_priv_get(dev); + if (!domain || !domain->iopf_handler) return 0; + /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ + if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported) + return 0; + return intel_iommu_enable_iopf(dev); } static inline void iopf_for_domain_remove(struct iommu_domain *domain, struct device *dev) { + struct device_domain_info *info = dev_iommu_priv_get(dev); + if (!domain || !domain->iopf_handler) return; + if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported) + return; + intel_iommu_disable_iopf(dev); } diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 89541b74ab8ca..40910dc7363b1 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -748,10 +748,12 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) } did = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); intel_context_flush_no_pasid(info, context, did); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); } static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 57cd1db7207ab..fea10acd4f021 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -164,12 +164,9 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, if (IS_ERR(dev_pasid)) return PTR_ERR(dev_pasid); - /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ - if (info->pri_supported) { - ret = iopf_for_domain_replace(domain, old, dev); - if (ret) - goto out_remove_dev_pasid; - } + ret = iopf_for_domain_replace(domain, old, dev); + if (ret) + goto out_remove_dev_pasid; /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; @@ -184,8 +181,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, return 0; out_unwind_iopf: - if (info->pri_supported) - iopf_for_domain_replace(old, domain, dev); + iopf_for_domain_replace(old, domain, dev); out_remove_dev_pasid: domain_remove_dev_pasid(domain, dev, pasid); return ret; diff --git a/drivers/iommu/iommufd/driver.c b/drivers/iommu/iommufd/driver.c index 61e6b02601d1a..3b8067976eac0 100644 --- a/drivers/iommu/iommufd/driver.c +++ b/drivers/iommu/iommufd/driver.c @@ -149,15 +149,18 @@ int iommufd_viommu_report_event(struct iommufd_viommu *viommu, goto out_unlock_veventqs; } - spin_lock(&veventq->common.lock); - if (veventq->num_events == veventq->depth) { + /* Pre-allocate to avoid GFP_ATOMIC; use GFP_NOWAIT to avoid sleeping */ + vevent = kzalloc_flex(*vevent, event_data, data_len, GFP_NOWAIT); + if (!vevent) { + spin_lock(&veventq->common.lock); vevent = &veventq->lost_events_header; + rc = -ENOMEM; goto out_set_header; } - vevent = kzalloc_flex(*vevent, event_data, data_len, GFP_ATOMIC); - if (!vevent) { - rc = -ENOMEM; + spin_lock(&veventq->common.lock); + if (veventq->num_events == veventq->depth) { + kfree(vevent); vevent = &veventq->lost_events_header; goto out_set_header; } diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index 710eef0b60045..5129e3bf5461c 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -139,9 +139,14 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, mutex_lock(&fault->mutex); while ((group = iommufd_fault_deliver_fetch(fault))) { + size_t group_done = done; + if (done >= count || group->fault_count * fault_size > count - done) { iommufd_fault_deliver_restore(fault, group); + /* Read count doesn't fit the first fault group */ + if (done == 0) + rc = -EINVAL; break; } @@ -157,14 +162,17 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, iommufd_compose_fault_message(&iopf->fault, &data, idev, group->cookie); - if (copy_to_user(buf + done, &data, fault_size)) { + if (copy_to_user(buf + group_done, &data, fault_size)) { xa_erase(&fault->response, group->cookie); iommufd_fault_deliver_restore(fault, group); rc = -EFAULT; break; } - done += fault_size; + group_done += fault_size; } + if (rc) + break; + done = group_done; } mutex_unlock(&fault->mutex); @@ -264,8 +272,10 @@ iommufd_veventq_deliver_fetch(struct iommufd_veventq *veventq) /* Make a copy of the lost_events_header for copy_to_user */ if (next == &veventq->lost_events_header) { vevent = kzalloc_obj(*vevent, GFP_ATOMIC); - if (!vevent) + if (!vevent) { + vevent = ERR_PTR(-ENOMEM); goto out_unlock; + } } list_del(&next->node); if (vevent) @@ -310,8 +320,17 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, if (*ppos) return -ESPIPE; + /* Minimum read count is a vEVENT header */ + if (count < sizeof(*hdr)) + return -EINVAL; while ((cur = iommufd_veventq_deliver_fetch(veventq))) { + if (IS_ERR(cur)) { + if (done == 0) + rc = PTR_ERR(cur); + break; + } + /* Validate the remaining bytes against the header size */ if (done >= count || sizeof(*hdr) > count - done) { iommufd_veventq_deliver_restore(veventq, cur); @@ -321,8 +340,11 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, /* If being a normal vEVENT, validate against the full size */ if (!vevent_for_lost_events_header(cur) && - sizeof(hdr) + cur->data_len > count - done) { + sizeof(*hdr) + cur->data_len > count - done) { iommufd_veventq_deliver_restore(veventq, cur); + /* Read count doesn't fit a single normal vEVENT */ + if (done == 0) + rc = -EINVAL; break; } @@ -336,6 +358,7 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, if (cur->data_len && copy_to_user(buf + done, cur->event_data, cur->data_len)) { iommufd_veventq_deliver_restore(veventq, cur); + done -= sizeof(*hdr); rc = -EFAULT; break; } @@ -473,6 +496,9 @@ int iommufd_fault_iopf_handler(struct iopf_group *group) static const struct file_operations iommufd_veventq_fops = INIT_EVENTQ_FOPS(iommufd_veventq_fops_read, NULL); +/* An arbitrary upper bound for veventq_depth that fits all existing HWs */ +#define VEVENTQ_MAX_DEPTH (1U << 19) + int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd) { struct iommu_veventq_alloc *cmd = ucmd->cmd; @@ -484,7 +510,7 @@ int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd) if (cmd->flags || cmd->__reserved || cmd->type == IOMMU_VEVENTQ_TYPE_DEFAULT) return -EOPNOTSUPP; - if (!cmd->veventq_depth) + if (!cmd->veventq_depth || cmd->veventq_depth > VEVENTQ_MAX_DEPTH) return -EINVAL; viommu = iommufd_get_viommu(ucmd, cmd->viommu_id); diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c index fe789c2dc0c97..623cc608ca0cd 100644 --- a/drivers/iommu/iommufd/hw_pagetable.c +++ b/drivers/iommu/iommufd/hw_pagetable.c @@ -489,6 +489,9 @@ int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd) return rc; } +/* An arbitrary entry_num cap, far above any realistic invalidation batch */ +#define IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX (1U << 19) + int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd) { struct iommu_hwpt_invalidate *cmd = ucmd->cmd; @@ -507,7 +510,13 @@ int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd) goto out; } - if (cmd->entry_num && (!cmd->data_uptr || !cmd->entry_len)) { + /* + * Bound entry_num and entry_len so a single call cannot pin the CPU; + * entry_len also caps the copy_struct_from_user() trailing-zero scan. + */ + if (cmd->entry_num && + (!cmd->data_uptr || !cmd->entry_len || cmd->entry_len > PAGE_SIZE || + cmd->entry_num > IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX)) { rc = -EINVAL; goto out; } diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 6ac1965199e9a..43fbc5bed8de3 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -602,7 +602,7 @@ struct iommufd_vevent { struct iommufd_vevent_header header; struct list_head node; /* for iommufd_eventq::deliver */ ssize_t data_len; - u64 event_data[] __counted_by(data_len); + u8 event_data[] __counted_by(data_len); }; #define vevent_for_lost_events_header(vevent) \ diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 9bdb2945afe1e..03c8379bbc347 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -1656,20 +1656,22 @@ void iopt_release_pages(struct kref *kref) WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root)); WARN_ON(pages->npinned); WARN_ON(!xa_empty(&pages->pinned_pfns)); - mmdrop(pages->source_mm); - mutex_destroy(&pages->mutex); - put_task_struct(pages->source_task); - free_uid(pages->source_user); if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) { struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf; + dma_resv_lock(dmabuf->resv, NULL); dma_buf_unpin(pages->dmabuf.attach); + dma_resv_unlock(dmabuf->resv); dma_buf_detach(dmabuf, pages->dmabuf.attach); dma_buf_put(dmabuf); WARN_ON(!list_empty(&pages->dmabuf.tracker)); } else if (pages->type == IOPT_ADDRESS_FILE) { fput(pages->file); } + mmdrop(pages->source_mm); + mutex_destroy(&pages->mutex); + put_task_struct(pages->source_task); + free_uid(pages->source_user); kfree(pages); } diff --git a/drivers/irqchip/exynos-combiner.c b/drivers/irqchip/exynos-combiner.c index 03cafcc5c8355..d9d408cb4711e 100644 --- a/drivers/irqchip/exynos-combiner.c +++ b/drivers/irqchip/exynos-combiner.c @@ -24,8 +24,6 @@ #define IRQ_IN_COMBINER 8 -static DEFINE_RAW_SPINLOCK(irq_controller_lock); - struct combiner_chip_data { unsigned int hwirq_offset; unsigned int irq_mask; @@ -72,9 +70,7 @@ static void combiner_handle_cascade_irq(struct irq_desc *desc) chained_irq_enter(chip, desc); - raw_spin_lock(&irq_controller_lock); status = readl_relaxed(chip_data->base + COMBINER_INT_STATUS); - raw_spin_unlock(&irq_controller_lock); status &= chip_data->irq_mask; if (status == 0) diff --git a/drivers/irqchip/irq-crossbar.c b/drivers/irqchip/irq-crossbar.c index cd1134101ace0..033b087821195 100644 --- a/drivers/irqchip/irq-crossbar.c +++ b/drivers/irqchip/irq-crossbar.c @@ -158,11 +158,17 @@ static void crossbar_domain_free(struct irq_domain *domain, unsigned int virq, for (i = 0; i < nr_irqs; i++) { struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + /* + * irq_map[] is indexed by GIC SPI number. The parent domain's + * hwirq contains the GIC interrupt number (GIC SPI + + * GIC_IRQ_START). + */ + cb->irq_map[d->parent_data->hwirq - GIC_IRQ_START] = IRQ_FREE; + cb->write(d->parent_data->hwirq - GIC_IRQ_START, cb->safe_map); irq_domain_reset_irq_data(d); - cb->irq_map[d->hwirq] = IRQ_FREE; - cb->write(d->hwirq, cb->safe_map); } raw_spin_unlock(&cb->lock); + irq_domain_free_irqs_parent(domain, virq, nr_irqs); } static int crossbar_domain_translate(struct irq_domain *d, diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 291d7668cc8da..5f9f58ff758cc 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3290,11 +3290,9 @@ static void its_cpu_init_collection(struct its_node *its) /* avoid cross node collections and its mapping */ if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { - struct device_node *cpu_node; + struct device_node *cpu_node __free(device_node) = of_get_cpu_node(cpu, NULL); - cpu_node = of_get_cpu_node(cpu, NULL); - if (its->numa_node != NUMA_NO_NODE && - its->numa_node != of_node_to_nid(cpu_node)) + if (its->numa_node != NUMA_NO_NODE && its->numa_node != of_node_to_nid(cpu_node)) return; } diff --git a/drivers/irqchip/irq-imgpdc.c b/drivers/irqchip/irq-imgpdc.c index e9ef2f5a7207e..4feef4ab5fecc 100644 --- a/drivers/irqchip/irq-imgpdc.c +++ b/drivers/irqchip/irq-imgpdc.c @@ -378,6 +378,7 @@ static int pdc_intc_probe(struct platform_device *pdev) dev_err(&pdev->dev, "cannot add IRQ domain\n"); return -ENOMEM; } + priv->domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC; /* * Set up 2 generic irq chips with 2 chip types. @@ -465,6 +466,11 @@ static void pdc_intc_remove(struct platform_device *pdev) { struct pdc_intc_priv *priv = platform_get_drvdata(pdev); + for (unsigned int i = 0; i < priv->nr_perips; ++i) + irq_set_chained_handler_and_data(priv->perip_irqs[i], NULL, NULL); + + irq_set_chained_handler_and_data(priv->syswake_irq, NULL, NULL); + irq_domain_remove(priv->domain); } diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c index a7a1852b548c4..12efd241ce880 100644 --- a/drivers/irqchip/irq-riscv-imsic-early.c +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -272,16 +272,13 @@ static int __init imsic_early_acpi_init(union acpi_subtable_headers *header, rc = imsic_setup_state(imsic_acpi_fwnode, imsic); if (rc) { pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc); - return rc; + goto cleanup; } /* Do early setup of IMSIC state and IPIs */ rc = imsic_early_probe(imsic_acpi_fwnode); - if (rc) { - irq_domain_free_fwnode(imsic_acpi_fwnode); - imsic_acpi_fwnode = NULL; - return rc; - } + if (rc) + goto cleanup; rc = imsic_platform_acpi_probe(imsic_acpi_fwnode); @@ -300,8 +297,12 @@ static int __init imsic_early_acpi_init(union acpi_subtable_headers *header, * DT where IPI works but MSI probe fails for some reason. */ return 0; -} +cleanup: + irq_domain_free_fwnode(imsic_acpi_fwnode); + imsic_acpi_fwnode = NULL; + return rc; +} IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL, 1, imsic_early_acpi_init); #endif diff --git a/drivers/irqchip/irq-ts4800.c b/drivers/irqchip/irq-ts4800.c index 2e4013c6834de..c7c0b155e353e 100644 --- a/drivers/irqchip/irq-ts4800.c +++ b/drivers/irqchip/irq-ts4800.c @@ -28,6 +28,7 @@ struct ts4800_irq_data { void __iomem *base; struct platform_device *pdev; struct irq_domain *domain; + unsigned int parent_irq; }; static void ts4800_irq_mask(struct irq_data *d) @@ -134,6 +135,7 @@ static int ts4800_ic_probe(struct platform_device *pdev) irq_set_chained_handler_and_data(parent_irq, ts4800_ic_chained_handle_irq, data); + data->parent_irq = parent_irq; platform_set_drvdata(pdev, data); return 0; @@ -142,6 +144,14 @@ static int ts4800_ic_probe(struct platform_device *pdev) static void ts4800_ic_remove(struct platform_device *pdev) { struct ts4800_irq_data *data = platform_get_drvdata(pdev); + unsigned int hwirq; + + irq_set_chained_handler_and_data(data->parent_irq, NULL, NULL); + + for (hwirq = 0; hwirq < 8; hwirq++) + irq_dispose_mapping(irq_find_mapping(data->domain, hwirq)); + + irq_dispose_mapping(data->parent_irq); irq_domain_remove(data->domain); } diff --git a/drivers/leds/uleds.c b/drivers/leds/uleds.c index ace71ffc0591f..41942cfca7051 100644 --- a/drivers/leds/uleds.c +++ b/drivers/leds/uleds.c @@ -102,7 +102,8 @@ static ssize_t uleds_write(struct file *file, const char __user *buffer, name = udev->user_dev.name; if (!name[0] || !strcmp(name, ".") || !strcmp(name, "..") || - strchr(name, '/')) { + strnchr(name, sizeof(udev->user_dev.name), '/') || + !strnchr(name, sizeof(udev->user_dev.name), '\0')) { ret = -EINVAL; goto out; } diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index 246a9a9e39520..0028073be4a71 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -227,6 +227,7 @@ static int imx_mu_generic_tx(struct imx_mu_priv *priv, u32 val; int ret, count; + ret = 0; switch (cp->type) { case IMX_MU_TYPE_TX: imx_mu_write(priv, *arg, priv->dcfg->xTR + cp->idx * 4); @@ -259,7 +260,7 @@ static int imx_mu_generic_tx(struct imx_mu_priv *priv, return -EINVAL; } - return 0; + return ret; } static int imx_mu_generic_rx(struct imx_mu_priv *priv, diff --git a/drivers/mailbox/mailbox-mpfs.c b/drivers/mailbox/mailbox-mpfs.c index d5d9effece979..ef40fe2be30d6 100644 --- a/drivers/mailbox/mailbox-mpfs.c +++ b/drivers/mailbox/mailbox-mpfs.c @@ -201,7 +201,7 @@ static irqreturn_t mpfs_mbox_inbox_isr(int irq, void *data) struct mbox_chan *chan = data; struct mpfs_mbox *mbox = (struct mpfs_mbox *)chan->con_priv; - if (mbox->control_scb) + if (mbox->sysreg_scb) regmap_write(mbox->sysreg_scb, MESSAGE_INT_OFFSET, 0); else writel_relaxed(0, mbox->int_reg); diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index bbc9fd75a95f7..006ea5a5c3207 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -327,6 +327,19 @@ int mbox_flush(struct mbox_chan *chan, unsigned long timeout) } EXPORT_SYMBOL_GPL(mbox_flush); +static void mbox_clean_and_put_channel(struct mbox_chan *chan) +{ + /* The queued TX requests are simply aborted, no callbacks are made */ + scoped_guard(spinlock_irqsave, &chan->lock) { + chan->cl = NULL; + chan->active_req = MBOX_NO_MSG; + if (chan->txdone_method == MBOX_TXDONE_BY_ACK) + chan->txdone_method = MBOX_TXDONE_BY_POLL; + } + + module_put(chan->mbox->dev->driver->owner); +} + static int __mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl) { struct device *dev = cl->dev; @@ -350,10 +363,9 @@ static int __mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl) if (chan->mbox->ops->startup) { ret = chan->mbox->ops->startup(chan); - if (ret) { dev_err(dev, "Unable to startup the chan (%d)\n", ret); - mbox_free_channel(chan); + mbox_clean_and_put_channel(chan); return ret; } } @@ -495,15 +507,7 @@ void mbox_free_channel(struct mbox_chan *chan) if (chan->mbox->ops->shutdown) chan->mbox->ops->shutdown(chan); - /* The queued TX requests are simply aborted, no callbacks are made */ - scoped_guard(spinlock_irqsave, &chan->lock) { - chan->cl = NULL; - chan->active_req = MBOX_NO_MSG; - if (chan->txdone_method == MBOX_TXDONE_BY_ACK) - chan->txdone_method = MBOX_TXDONE_BY_POLL; - } - - module_put(chan->mbox->dev->driver->owner); + mbox_clean_and_put_channel(chan); } EXPORT_SYMBOL_GPL(mbox_free_channel); diff --git a/drivers/mailbox/mtk-adsp-mailbox.c b/drivers/mailbox/mtk-adsp-mailbox.c index 91487aa4d7da0..8bcecddee0eb5 100644 --- a/drivers/mailbox/mtk-adsp-mailbox.c +++ b/drivers/mailbox/mtk-adsp-mailbox.c @@ -19,6 +19,7 @@ struct mtk_adsp_mbox_priv { struct mbox_controller mbox; void __iomem *va_mboxreg; const struct mtk_adsp_mbox_cfg *cfg; + int irq; }; struct mtk_adsp_mbox_cfg { @@ -67,6 +68,8 @@ static int mtk_adsp_mbox_startup(struct mbox_chan *chan) writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_in); writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_out); + enable_irq(priv->irq); + return 0; } @@ -74,6 +77,8 @@ static void mtk_adsp_mbox_shutdown(struct mbox_chan *chan) { struct mtk_adsp_mbox_priv *priv = get_mtk_adsp_mbox_priv(chan->mbox); + disable_irq(priv->irq); + /* Clear ADSP mbox command */ writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_in); writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_out); @@ -139,8 +144,10 @@ static int mtk_adsp_mbox_probe(struct platform_device *pdev) if (irq < 0) return irq; + priv->irq = irq; ret = devm_request_threaded_irq(dev, irq, mtk_adsp_mbox_irq, - mtk_adsp_mbox_isr, IRQF_TRIGGER_NONE, + mtk_adsp_mbox_isr, + IRQF_TRIGGER_NONE | IRQF_NO_AUTOEN, dev_name(dev), mbox->chans); if (ret < 0) return ret; diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 26fedf5883eff..a458b9fd2fcd6 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -2238,7 +2238,9 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c struct dm_io_region io_reg = { .bdev = c->bdev, .sector = block_to_sector(c, block), - .count = block_to_sector(c, count), + .count = likely(c->sectors_per_block_bits >= 0) ? + count << c->sectors_per_block_bits : + count * (c->block_size >> SECTOR_SHIFT), }; if (WARN_ON_ONCE(dm_bufio_in_request())) diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index 05285c04ff2cd..cef2881948043 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c @@ -810,8 +810,10 @@ static struct era_metadata *metadata_open(struct block_device *bdev, int r; struct era_metadata *md = kzalloc_obj(*md); - if (!md) - return NULL; + if (!md) { + DMERR("could not allocate metadata struct"); + return ERR_PTR(-ENOMEM); + } md->bdev = bdev; md->block_size = block_size; @@ -1229,6 +1231,7 @@ static dm_block_t get_block(struct era *era, struct bio *bio) static void remap_to_origin(struct era *era, struct bio *bio) { bio_set_dev(bio, era->origin_dev->bdev); + bio->bi_iter.bi_sector = dm_target_offset(era->ti, bio->bi_iter.bi_sector); } /* @@ -1560,7 +1563,7 @@ static void era_dtr(struct dm_target *ti) static int era_map(struct dm_target *ti, struct bio *bio) { struct era *era = ti->private; - dm_block_t block = get_block(era, bio); + dm_block_t block; /* * All bios get remapped to the origin device. We do this now, but @@ -1568,6 +1571,7 @@ static int era_map(struct dm_target *ti, struct bio *bio) * block is marked in this era. */ remap_to_origin(era, bio); + block = get_block(era, bio); /* * REQ_PREFLUSH bios carry no data, so we're not interested in them. diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 65c30dec82220..77c28367ee532 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -1480,9 +1480,6 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se *metadata_offset = 0; } - if (unlikely(!is_power_of_2(ic->tag_size))) - hash_offset = (hash_offset + to_copy) % ic->tag_size; - total_size -= to_copy; } while (unlikely(total_size)); @@ -2523,6 +2520,9 @@ static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map) if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0)) return DM_MAPIO_REMAPPED; + if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) + return DM_MAPIO_KILL; + retry: if (!dio->integrity_payload) { unsigned digest_size, extra_size; @@ -2587,10 +2587,6 @@ static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map) dio->bio_details.bi_iter = bio->bi_iter; - if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) { - return DM_MAPIO_KILL; - } - bio->bi_iter.bi_sector += ic->start + SB_SECTORS; bip = bio_integrity_alloc(bio, GFP_NOIO, 1); @@ -2606,7 +2602,7 @@ static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map) struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); const char *mem = integrity_kmap(ic, bv.bv_page); if (ic->tag_size < ic->tuple_size) - memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size); + memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tag_size); integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos); integrity_kunmap(ic, mem); pos += ic->tuple_size; diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index a529174c94cf3..69847fab0353e 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -698,7 +698,7 @@ static void list_version_get_info(struct target_type *tt, void *param) struct vers_iter *info = param; /* Check space - it might have changed since the first iteration */ - if ((char *)info->vers + sizeof(tt->version) + strlen(tt->name) + 1 > info->end) { + if ((char *)info->vers + sizeof(struct dm_target_versions) + strlen(tt->name) + 1 > info->end) { info->flags = DM_BUFFER_FULL_FLAG; return; } @@ -1816,8 +1816,11 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para goto out_argv; table = dm_get_live_table(md, &srcu_idx); - if (!table) + if (!table) { + DMERR("The device has no table."); + r = -EINVAL; goto out_table; + } if (dm_deleting_md(md)) { r = -ENXIO; @@ -2348,7 +2351,7 @@ int __init dm_early_create(struct dm_ioctl *dmi, /* resume device */ r = dm_resume(md); if (r) - goto err_destroy_table; + goto err_hash_remove; DMINFO("%s (%s) is ready", md->disk->disk_name, dmi->name); dm_put(md); diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index d316757a328be..2ddeb4250c592 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -425,6 +425,9 @@ static int create_log_context(struct dm_dirty_log *log, struct dm_target *ti, */ bitset_size = dm_round_up(region_count, BITS_PER_LONG); bitset_size >>= BYTE_SHIFT; + /* Handle dm_round_up rollover on 32-bit systems */ + if (!bitset_size) + bitset_size = 1UL << (BITS_PER_LONG - BYTE_SHIFT); lc->bitset_uint32_count = bitset_size / sizeof(*lc->clean_bits); diff --git a/drivers/md/dm-pcache/dm_pcache.c b/drivers/md/dm-pcache/dm_pcache.c index 81c795c0400e9..d5cfd162c0633 100644 --- a/drivers/md/dm-pcache/dm_pcache.c +++ b/drivers/md/dm-pcache/dm_pcache.c @@ -168,6 +168,10 @@ static int parse_cache_opts(struct dm_pcache *pcache, struct dm_arg_set *as, argc--; if (!strcmp(arg, "cache_mode")) { + if (!argc) { + *error = "Missing value for cache_mode"; + return -EINVAL; + } arg = dm_shift_arg(as); if (!strcmp(arg, "writeback")) { opts->cache_mode = PCACHE_CACHE_MODE_WRITEBACK; @@ -177,6 +181,10 @@ static int parse_cache_opts(struct dm_pcache *pcache, struct dm_arg_set *as, } argc--; } else if (!strcmp(arg, "data_crc")) { + if (!argc) { + *error = "Missing value for data_crc"; + return -EINVAL; + } arg = dm_shift_arg(as); if (!strcmp(arg, "true")) { opts->data_crc = true; diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index c53cf07ab7b0b..5df710061a116 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -692,10 +692,8 @@ void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, */ last = raw_cpu_ptr(stats->last); stats_aux->merged = - (bi_sector == (READ_ONCE(last->last_sector) && - ((bi_rw == WRITE) == - (READ_ONCE(last->last_rw) == WRITE)) - )); + bi_sector == READ_ONCE(last->last_sector) && + (bi_rw == WRITE) == (READ_ONCE(last->last_rw) == WRITE); WRITE_ONCE(last->last_sector, end_sector); WRITE_ONCE(last->last_rw, bi_rw); } else @@ -842,10 +840,10 @@ static unsigned long long dm_jiffies_to_msec64(struct dm_stat *s, unsigned long result = jiffies_to_msecs(j & 0x3fffff); if (j >= 1 << 22) { mult = jiffies_to_msecs(1 << 22); - result += (unsigned long long)mult * (unsigned long long)jiffies_to_msecs((j >> 22) & 0x3fffff); + result += (unsigned long long)mult * ((j >> 22) & 0x3fffff); } if (j >= 1ULL << 44) - result += (unsigned long long)mult * (unsigned long long)mult * (unsigned long long)jiffies_to_msecs(j >> 44); + result += (unsigned long long)mult * (unsigned long long)(1 << 22) * (j >> 44); return result; } diff --git a/drivers/md/dm-thin-metadata.c b/drivers/md/dm-thin-metadata.c index b6a2d2081a24b..e60e1326376a0 100644 --- a/drivers/md/dm-thin-metadata.c +++ b/drivers/md/dm-thin-metadata.c @@ -186,6 +186,7 @@ struct dm_pool_metadata { uint32_t time; dm_block_t root; dm_block_t details_root; + dm_block_t held_root; struct list_head thin_devices; uint64_t trans_id; unsigned long flags; @@ -748,6 +749,7 @@ static int __open_metadata(struct dm_pool_metadata *pmd) */ pmd->root = le64_to_cpu(disk_super->data_mapping_root); pmd->details_root = le64_to_cpu(disk_super->device_details_root); + pmd->held_root = le64_to_cpu(disk_super->held_root); __setup_btree_details(pmd); dm_bm_unlock(sblock); @@ -838,6 +840,7 @@ static int __begin_transaction(struct dm_pool_metadata *pmd) pmd->time = le32_to_cpu(disk_super->time); pmd->root = le64_to_cpu(disk_super->data_mapping_root); pmd->details_root = le64_to_cpu(disk_super->device_details_root); + pmd->held_root = le64_to_cpu(disk_super->held_root); pmd->trans_id = le64_to_cpu(disk_super->trans_id); pmd->flags = le32_to_cpu(disk_super->flags); pmd->data_block_size = le32_to_cpu(disk_super->data_block_size); @@ -928,6 +931,7 @@ static int __commit_transaction(struct dm_pool_metadata *pmd) disk_super->time = cpu_to_le32(pmd->time); disk_super->data_mapping_root = cpu_to_le64(pmd->root); disk_super->device_details_root = cpu_to_le64(pmd->details_root); + disk_super->held_root = cpu_to_le64(pmd->held_root); disk_super->trans_id = cpu_to_le64(pmd->trans_id); disk_super->flags = cpu_to_le32(pmd->flags); @@ -1333,9 +1337,14 @@ static int __reserve_metadata_snap(struct dm_pool_metadata *pmd) { int r, inc; struct thin_disk_superblock *disk_super; - struct dm_block *copy, *sblock; + struct dm_block *copy; dm_block_t held_root; + if (pmd->held_root) { + DMWARN("Pool metadata snapshot already exists: release this before taking another."); + return -EBUSY; + } + /* * We commit to ensure the btree roots which we increment in a * moment are up to date. @@ -1353,22 +1362,16 @@ static int __reserve_metadata_snap(struct dm_pool_metadata *pmd) dm_sm_inc_block(pmd->metadata_sm, THIN_SUPERBLOCK_LOCATION); r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION, &sb_validator, ©, &inc); - if (r) + if (r) { + dm_sm_dec_block(pmd->metadata_sm, THIN_SUPERBLOCK_LOCATION); return r; + } BUG_ON(!inc); held_root = dm_block_location(copy); disk_super = dm_block_data(copy); - if (le64_to_cpu(disk_super->held_root)) { - DMWARN("Pool metadata snapshot already exists: release this before taking another."); - - dm_tm_dec(pmd->tm, held_root); - dm_tm_unlock(pmd->tm, copy); - return -EBUSY; - } - /* * Wipe the spacemap since we're not publishing this. */ @@ -1384,18 +1387,8 @@ static int __reserve_metadata_snap(struct dm_pool_metadata *pmd) dm_tm_inc(pmd->tm, le64_to_cpu(disk_super->device_details_root)); dm_tm_unlock(pmd->tm, copy); - /* - * Write the held root into the superblock. - */ - r = superblock_lock(pmd, &sblock); - if (r) { - dm_tm_dec(pmd->tm, held_root); - return r; - } + pmd->held_root = held_root; - disk_super = dm_block_data(sblock); - disk_super->held_root = cpu_to_le64(held_root); - dm_bm_unlock(sblock); return 0; } @@ -1415,18 +1408,10 @@ static int __release_metadata_snap(struct dm_pool_metadata *pmd) { int r; struct thin_disk_superblock *disk_super; - struct dm_block *sblock, *copy; + struct dm_block *copy; dm_block_t held_root; - r = superblock_lock(pmd, &sblock); - if (r) - return r; - - disk_super = dm_block_data(sblock); - held_root = le64_to_cpu(disk_super->held_root); - disk_super->held_root = cpu_to_le64(0); - - dm_bm_unlock(sblock); + held_root = pmd->held_root; if (!held_root) { DMWARN("No pool metadata snapshot found: nothing to release."); @@ -1437,13 +1422,15 @@ static int __release_metadata_snap(struct dm_pool_metadata *pmd) if (r) return r; + pmd->held_root = 0; + disk_super = dm_block_data(copy); dm_btree_del(&pmd->info, le64_to_cpu(disk_super->data_mapping_root)); dm_btree_del(&pmd->details_info, le64_to_cpu(disk_super->device_details_root)); - dm_sm_dec_block(pmd->metadata_sm, held_root); - dm_tm_unlock(pmd->tm, copy); + dm_sm_dec_block(pmd->metadata_sm, held_root); + return 0; } @@ -1462,19 +1449,7 @@ int dm_pool_release_metadata_snap(struct dm_pool_metadata *pmd) static int __get_metadata_snap(struct dm_pool_metadata *pmd, dm_block_t *result) { - int r; - struct thin_disk_superblock *disk_super; - struct dm_block *sblock; - - r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION, - &sb_validator, &sblock); - if (r) - return r; - - disk_super = dm_block_data(sblock); - *result = le64_to_cpu(disk_super->held_root); - - dm_bm_unlock(sblock); + *result = pmd->held_root; return 0; } diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 85ad9dc210ff0..c79f60df3a905 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -220,7 +220,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, PTR_ERR(bbuf)); /* assume the block is corrupted */ - if (neras && *neras <= v->fec->roots) + if (neras && *neras < v->fec->roots) fio->erasures[(*neras)++] = i; continue; @@ -238,7 +238,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, * skip if we have already found the theoretical * maximum number (i.e. fec->roots) of erasures */ - if (neras && *neras <= v->fec->roots && + if (neras && *neras < v->fec->roots && fec_is_erasure(v, io, want_digest, bbuf)) fio->erasures[(*neras)++] = i; } diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h index 50b5e187d5cc1..3885b514fc23e 100644 --- a/drivers/md/dm-verity-fec.h +++ b/drivers/md/dm-verity-fec.h @@ -47,7 +47,7 @@ struct dm_verity_fec { /* per-bio data */ struct dm_verity_fec_io { struct rs_control *rs; /* Reed-Solomon state */ - int erasures[DM_VERITY_FEC_MAX_ROOTS + 1]; /* erasures for decode_rs8 */ + int erasures[DM_VERITY_FEC_MAX_ROOTS]; /* erasures for decode_rs8 */ u8 *output; /* buffer for corrected output */ unsigned int level; /* recursion level */ unsigned int nbufs; /* number of buffers allocated */ diff --git a/drivers/md/dm-verity-loadpin.c b/drivers/md/dm-verity-loadpin.c index 0666699b68581..9a64f575ae5f6 100644 --- a/drivers/md/dm-verity-loadpin.c +++ b/drivers/md/dm-verity-loadpin.c @@ -70,7 +70,7 @@ bool dm_verity_loadpin_is_bdev_trusted(struct block_device *bdev) table = dm_get_live_table(md, &srcu_idx); - if (table->num_targets != 1) + if (!table || table->num_targets != 1) goto out; ti = dm_table_get_target(table, 0); diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 9a9847f94c460..1b07630912540 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -26,7 +26,7 @@ #define DM_MSG_PREFIX "verity" -#define DM_VERITY_ENV_LENGTH 42 +#define DM_VERITY_ENV_LENGTH 46 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 @@ -180,14 +180,16 @@ static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, char *envp[] = { verity_env, NULL }; const char *type_str = ""; struct mapped_device *md = dm_table_get_md(v->ti->table); + int ce; /* Corruption should be visible in device status in all modes */ v->hash_failed = true; - if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) - goto out; - - v->corrupted_errs++; + ce = atomic_read(&v->corrupted_errs); + do { + if (ce >= DM_VERITY_MAX_CORRUPTED_ERRS) + goto out; + } while (!atomic_try_cmpxchg(&v->corrupted_errs, &ce, ce + 1)); switch (type) { case DM_VERITY_BLOCK_TYPE_DATA: @@ -203,7 +205,7 @@ static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, type_str, block); - if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) { + if (ce + 1 == DM_VERITY_MAX_CORRUPTED_ERRS) { DMERR("%s: reached maximum errors", v->data_dev->name); dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0); } @@ -1262,6 +1264,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, continue; } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { + if (v->use_bh_wq) + continue; v->use_bh_wq = true; static_branch_inc(&use_bh_wq_enabled); continue; diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index 2922263501f68..e104a651c6574 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -68,7 +68,7 @@ struct dm_verity { unsigned int digest_size; /* digest size for the current hash algorithm */ enum verity_mode mode; /* mode for handling verification errors */ enum verity_mode error_mode;/* mode for handling I/O errors */ - unsigned int corrupted_errs;/* Number of errors for corrupted blocks */ + atomic_t corrupted_errs;/* Number of errors for corrupted blocks */ struct workqueue_struct *verify_wq; diff --git a/drivers/md/dm.c b/drivers/md/dm.c index e178fe19973ea..1334c411c7224 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -735,7 +735,16 @@ static struct table_device *open_table_device(struct mapped_device *md, return ERR_PTR(-ENOMEM); refcount_set(&td->count, 1); - bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL); + /* + * Open the backing device with kernel rather than caller + * credentials. Otherwise the caller's credentials would be + * pinned in bdev_file->f_cred until the table device is closed. + * That would keep the caller's thread keyring alive long beyond the + * lifetime of the caller, breaking userspace expectation (e.g. + * cryptsetup(8) leaking the LUKS volume key). + */ + scoped_with_kernel_creds() + bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL); if (IS_ERR(bdev_file)) { r = PTR_ERR(bdev_file); goto out_free_td; @@ -2098,8 +2107,17 @@ static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob, WARN_ON_ONCE(!dm_tio_is_normal(&io->tio)); /* don't poll if the mapped io is done */ - if (atomic_read(&io->io_count) > 1) - bio_poll(&io->tio.clone, iob, flags); + if (atomic_read(&io->io_count) > 1) { + /* + * DM hides the target queues from the upper poller, which may + * decide it is safe to spin on a single stacked queue. Do not + * pass that spinning policy down to a target queue: one slow + * clone could keep the task inside dm_poll_bio() for a long + * time. Poll target bios once and let the caller decide + * whether to keep polling, reap completions or reschedule. + */ + bio_poll(&io->tio.clone, iob, flags | BLK_POLL_ONESHOT); + } /* bio_poll holds the last reference */ return atomic_read(&io->io_count) == 1; diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 028b9ca8ce52d..1206e31f323ad 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2063,18 +2063,23 @@ static void bitmap_end_behind_write(struct mddev *mddev) bitmap->mddev->bitmap_info.max_write_behind); } -static void bitmap_wait_behind_writes(struct mddev *mddev) +static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait) { struct bitmap *bitmap = mddev->bitmap; /* wait for behind writes to complete */ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { + if (nowait) + return false; + pr_debug("md:%s: behind writes in progress - waiting to stop.\n", mdname(mddev)); /* need to kick something here to make sure I/O goes? */ wait_event(bitmap->behind_wait, atomic_read(&bitmap->behind_writes) == 0); } + + return true; } static void bitmap_destroy(struct mddev *mddev) @@ -2084,7 +2089,7 @@ static void bitmap_destroy(struct mddev *mddev) if (!bitmap) /* there was no bitmap */ return; - bitmap_wait_behind_writes(mddev); + bitmap_wait_behind_writes(mddev, false); if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags)) mddev_destroy_serial_pool(mddev, NULL); diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index 214f623c7e790..f46674bdfeb91 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -98,7 +98,7 @@ struct bitmap_operations { void (*start_behind_write)(struct mddev *mddev); void (*end_behind_write)(struct mddev *mddev); - void (*wait_behind_writes)(struct mddev *mddev); + bool (*wait_behind_writes)(struct mddev *mddev, bool nowait); md_bitmap_fn *start_write; md_bitmap_fn *end_write; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1adc5b1178216..5a4e2abaa7577 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1574,16 +1574,19 @@ static void llbitmap_end_behind_write(struct mddev *mddev) wake_up(&llbitmap->behind_wait); } -static void llbitmap_wait_behind_writes(struct mddev *mddev) +static bool llbitmap_wait_behind_writes(struct mddev *mddev, bool nowait) { struct llbitmap *llbitmap = mddev->bitmap; - if (!llbitmap) - return; + if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) { + if (nowait) + return false; - wait_event(llbitmap->behind_wait, - atomic_read(&llbitmap->behind_writes) == 0); + wait_event(llbitmap->behind_wait, + atomic_read(&llbitmap->behind_writes) == 0); + } + return true; } static ssize_t bits_show(struct mddev *mddev, char *page) diff --git a/drivers/md/md.c b/drivers/md/md.c index 8b568eee87433..6d1ddbdbbf8d2 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -395,17 +395,24 @@ static bool is_suspended(struct mddev *mddev, struct bio *bio) bool md_handle_request(struct mddev *mddev, struct bio *bio) { check_suspended: - if (is_suspended(mddev, bio)) { - /* Bail out if REQ_NOWAIT is set for the bio */ - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return true; + if (unlikely(md_cloned_bio(mddev, bio))) { + /* + * This bio is an MD cloned bio and already holds an + * active_io reference, so percpu_ref_get() is safe here. + */ + percpu_ref_get(&mddev->active_io); + } else { + if (is_suspended(mddev, bio)) { + /* Bail out if REQ_NOWAIT is set for the bio */ + if (bio->bi_opf & REQ_NOWAIT) { + bio_wouldblock_error(bio); + return true; + } + wait_event(mddev->sb_wait, !is_suspended(mddev, bio)); } - wait_event(mddev->sb_wait, !is_suspended(mddev, bio)); + if (!percpu_ref_tryget_live(&mddev->active_io)) + goto check_suspended; } - if (!percpu_ref_tryget_live(&mddev->active_io)) - goto check_suspended; - if (!mddev->pers->make_request(mddev, bio)) { percpu_ref_put(&mddev->active_io); if (mddev_is_dm(mddev) && mddev->pers->prepare_suspend) @@ -7042,7 +7049,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); static void mddev_detach(struct mddev *mddev) { if (md_bitmap_enabled(mddev, false)) - mddev->bitmap_ops->wait_behind_writes(mddev); + mddev->bitmap_ops->wait_behind_writes(mddev, false); if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) { mddev->pers->quiesce(mddev, 1); mddev->pers->quiesce(mddev, 0); diff --git a/drivers/md/md.h b/drivers/md/md.h index 52c3780860464..aca5c06e5fe31 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -1042,6 +1042,11 @@ void mddev_update_io_opt(struct mddev *mddev, unsigned int nr_stripes); extern const struct block_device_operations md_fops; +static inline bool md_cloned_bio(struct mddev *mddev, struct bio *bio) +{ + return bio->bi_pool == &mddev->io_clone_set; +} + /* * MD devices can be used undeneath by DM, in which case ->gendisk is NULL. */ diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 64d970e2ef50f..1f67e4efa09af 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1341,21 +1341,34 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, int max_sectors; int rdisk; bool r1bio_existed = !!r1_bio; + bool nowait = bio->bi_opf & REQ_NOWAIT; /* - * If r1_bio is set, we are blocking the raid1d thread - * so there is a tiny risk of deadlock. So ask for + * An md cloned bio indicates we are in the error path. + * This is more reliable than checking r1_bio, which might + * be NULL even in the error path if a failed bio was split. + */ + bool err_path = md_cloned_bio(mddev, bio); + + /* + * If we are in the error path, we are blocking the raid1d + * thread so there is a tiny risk of deadlock. So ask for * emergency memory if needed. */ - gfp_t gfp = r1_bio ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; + gfp_t gfp = err_path ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; /* * Still need barrier for READ in case that whole * array is frozen. */ - if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, - bio->bi_opf & REQ_NOWAIT)) { + if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { bio_wouldblock_error(bio); + + if (r1bio_existed) { + set_bit(R1BIO_Returned, &r1_bio->state); + raid_end_bio_io(r1_bio); + } + return; } @@ -1395,7 +1408,11 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * over-take any writes that are 'behind' */ mddev_add_trace_msg(mddev, "raid1 wait behind writes"); - mddev->bitmap_ops->wait_behind_writes(mddev); + if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) { + bio_wouldblock_error(bio); + set_bit(R1BIO_Returned, &r1_bio->state); + goto err_handle; + } } if (max_sectors < bio_sectors(bio)) { @@ -1411,7 +1428,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, } r1_bio->read_disk = rdisk; - if (!r1bio_existed) { + if (likely(!md_cloned_bio(mddev, bio))) { md_account_bio(mddev, &bio); r1_bio->master_bio = bio; } @@ -1494,7 +1511,7 @@ static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, } -static void raid1_write_request(struct mddev *mddev, struct bio *bio, +static bool raid1_write_request(struct mddev *mddev, struct bio *bio, int max_write_sectors) { struct r1conf *conf = mddev->private; @@ -1505,6 +1522,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, int max_sectors; bool write_behind = false; bool is_discard = (bio_op(bio) == REQ_OP_DISCARD); + sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, @@ -1512,7 +1530,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, if (bio->bi_opf & REQ_NOWAIT) { bio_wouldblock_error(bio); - return; + return false; } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, @@ -1528,12 +1546,13 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, if (!wait_barrier(conf, bio->bi_iter.bi_sector, bio->bi_opf & REQ_NOWAIT)) { bio_wouldblock_error(bio); - return; + return false; } if (!wait_blocked_rdev(mddev, bio)) { bio_wouldblock_error(bio); - return; + allow_barrier(conf, sector); + return false; } r1_bio = alloc_r1bio(mddev, bio); @@ -1596,8 +1615,10 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, * complexity of supporting that is not worth * the benefit. */ - if (bio->bi_opf & REQ_ATOMIC) + if (bio->bi_opf & REQ_ATOMIC) { + rdev_dec_pending(rdev, mddev); goto err_handle; + } good_sectors = first_bad - r1_bio->sector; if (good_sectors < max_sectors) @@ -1690,7 +1711,8 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, /* In case raid1d snuck in to freeze_array */ wake_up_barrier(conf); - return; + return true; + err_handle: for (k = 0; k < i; k++) { if (r1_bio->bios[k]) { @@ -1700,6 +1722,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, } raid_end_bio_io(r1_bio); + return false; } static bool raid1_make_request(struct mddev *mddev, struct bio *bio) @@ -1723,8 +1746,9 @@ static bool raid1_make_request(struct mddev *mddev, struct bio *bio) if (bio_data_dir(bio) == READ) raid1_read_request(mddev, bio, sectors, NULL); else { - md_write_start(mddev,bio); - raid1_write_request(mddev, bio, sectors); + md_write_start(mddev, bio); + if (!raid1_write_request(mddev, bio, sectors)) + md_write_end(mddev); } return true; } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 39085e7dd6d26..aacf160ee9f20 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1146,7 +1146,7 @@ static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf, } static void raid10_read_request(struct mddev *mddev, struct bio *bio, - struct r10bio *r10_bio, bool io_accounting) + struct r10bio *r10_bio) { struct r10conf *conf = mddev->private; struct bio *read_bio; @@ -1155,7 +1155,20 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, char b[BDEVNAME_SIZE]; int slot = r10_bio->read_slot; struct md_rdev *err_rdev = NULL; - gfp_t gfp = GFP_NOIO; + + /* + * An md cloned bio indicates we are in the error path. + * This is more reliable than checking slot, which might + * be -1 even in the error path if a failed bio was split. + */ + bool err_path = md_cloned_bio(mddev, bio); + + /* + * If we are in the error path, we are blocking the raid10d + * thread so there is a tiny risk of deadlock. So ask for + * emergency memory if needed. + */ + gfp_t gfp = err_path ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; if (slot >= 0 && r10_bio->devs[slot].rdev) { /* @@ -1166,11 +1179,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, * we lose the device name in error messages. */ int disk; - /* - * As we are blocking raid10, it is a little safer to - * use __GFP_HIGH. - */ - gfp = GFP_NOIO | __GFP_HIGH; disk = r10_bio->devs[slot].devnum; err_rdev = conf->mirrors[disk].rdev; @@ -1218,7 +1226,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } slot = r10_bio->read_slot; - if (io_accounting) { + if (likely(!md_cloned_bio(mddev, bio))) { md_account_bio(mddev, &bio); r10_bio->master_bio = bio; } @@ -1341,7 +1349,7 @@ static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio) } } -static void raid10_write_request(struct mddev *mddev, struct bio *bio, +static bool raid10_write_request(struct mddev *mddev, struct bio *bio, struct r10bio *r10_bio) { struct r10conf *conf = mddev->private; @@ -1357,7 +1365,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, /* Bail out if REQ_NOWAIT is set for the bio */ if (bio->bi_opf & REQ_NOWAIT) { bio_wouldblock_error(bio); - return; + return false; } for (;;) { prepare_to_wait(&conf->wait_barrier, @@ -1373,7 +1381,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, sectors = r10_bio->sectors; if (!regular_request_wait(mddev, conf, bio, sectors)) { free_r10bio(r10_bio); - return; + return false; } if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && @@ -1390,7 +1398,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, if (bio->bi_opf & REQ_NOWAIT) { allow_barrier(conf); bio_wouldblock_error(bio); - return; + return false; } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape metadata"); @@ -1506,7 +1514,8 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, raid10_write_one_disk(mddev, r10_bio, bio, true, i); } one_write_done(r10_bio); - return; + return true; + err_handle: for (k = 0; k < i; k++) { int d = r10_bio->devs[k].devnum; @@ -1524,10 +1533,12 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, } raid_end_bio_io(r10_bio); + return false; } -static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) +static bool __make_request(struct mddev *mddev, struct bio *bio, int sectors) { + bool ret; struct r10conf *conf = mddev->private; struct r10bio *r10_bio; @@ -1543,10 +1554,13 @@ static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->geo.raid_disks); + ret = true; if (bio_data_dir(bio) == READ) - raid10_read_request(mddev, bio, r10_bio, true); + raid10_read_request(mddev, bio, r10_bio); else - raid10_write_request(mddev, bio, r10_bio); + ret = raid10_write_request(mddev, bio, r10_bio); + + return ret; } static void raid_end_discard_bio(struct r10bio *r10bio) @@ -1625,6 +1639,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { bio_wouldblock_error(bio); + md_write_end(mddev); return 0; } @@ -1667,6 +1682,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } @@ -1684,6 +1701,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } @@ -1727,6 +1746,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) r10_bio->mddev = mddev; r10_bio->state = 0; r10_bio->sectors = 0; + r10_bio->read_slot = -1; memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks); wait_blocked_dev(mddev, r10_bio); @@ -1891,7 +1911,8 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) sectors = chunk_sects - (bio->bi_iter.bi_sector & (chunk_sects - 1)); - __make_request(mddev, bio, sectors); + if (!__make_request(mddev, bio, sectors)) + md_write_end(mddev); /* In case raid10d snuck in to freeze_array */ wake_up_barrier(conf); @@ -2858,7 +2879,7 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) rdev_dec_pending(rdev, mddev); r10_bio->state = 0; - raid10_read_request(mddev, r10_bio->master_bio, r10_bio, false); + raid10_read_request(mddev, r10_bio->master_bio, r10_bio); /* * allow_barrier after re-submit to ensure no sync io * can be issued while regular io pending. diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 0d76e82f4506e..5ae7ed98b6548 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -4837,55 +4837,62 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, { struct stripe_head *sh, *next; int i; + unsigned long state; list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { list_del_init(&sh->batch_list); - WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | - (1 << STRIPE_SYNCING) | - (1 << STRIPE_REPLACED) | - (1 << STRIPE_DELAYED) | - (1 << STRIPE_BIT_DELAY) | - (1 << STRIPE_FULL_WRITE) | - (1 << STRIPE_BIOFILL_RUN) | - (1 << STRIPE_COMPUTE_RUN) | - (1 << STRIPE_DISCARD) | - (1 << STRIPE_BATCH_READY) | - (1 << STRIPE_BATCH_ERR)), - "stripe state: %lx\n", sh->state); - WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | - (1 << STRIPE_REPLACED)), - "head stripe state: %lx\n", head_sh->state); + state = READ_ONCE(sh->state); + WARN_ONCE(state & ((1 << STRIPE_ACTIVE) | + (1 << STRIPE_SYNCING) | + (1 << STRIPE_REPLACED) | + (1 << STRIPE_DELAYED) | + (1 << STRIPE_BIT_DELAY) | + (1 << STRIPE_FULL_WRITE) | + (1 << STRIPE_BIOFILL_RUN) | + (1 << STRIPE_COMPUTE_RUN) | + (1 << STRIPE_DISCARD) | + (1 << STRIPE_BATCH_READY) | + (1 << STRIPE_BATCH_ERR)), + "stripe state: %lx\n", state); + + state = READ_ONCE(head_sh->state); + WARN_ONCE(state & ((1 << STRIPE_DISCARD) | + (1 << STRIPE_REPLACED)), + "head stripe state: %lx\n", state); set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | (1 << STRIPE_PREREAD_ACTIVE) | (1 << STRIPE_ON_UNPLUG_LIST)), - head_sh->state & (1 << STRIPE_INSYNC)); + state & (1 << STRIPE_INSYNC)); sh->check_state = head_sh->check_state; sh->reconstruct_state = head_sh->reconstruct_state; spin_lock_irq(&sh->stripe_lock); - sh->batch_head = NULL; - spin_unlock_irq(&sh->stripe_lock); for (i = 0; i < sh->disks; i++) { if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) wake_up_bit(&sh->dev[i].flags, R5_Overlap); - sh->dev[i].flags = head_sh->dev[i].flags & + sh->dev[i].flags = READ_ONCE(head_sh->dev[i].flags) & (~((1 << R5_WriteError) | (1 << R5_Overlap))); } - if (handle_flags == 0 || - sh->state & handle_flags) + sh->batch_head = NULL; + spin_unlock_irq(&sh->stripe_lock); + + state = READ_ONCE(sh->state); + if (handle_flags == 0 || (state & handle_flags)) set_bit(STRIPE_HANDLE, &sh->state); raid5_release_stripe(sh); } spin_lock_irq(&head_sh->stripe_lock); - head_sh->batch_head = NULL; - spin_unlock_irq(&head_sh->stripe_lock); for (i = 0; i < head_sh->disks; i++) if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) wake_up_bit(&head_sh->dev[i].flags, R5_Overlap); - if (head_sh->state & handle_flags) + head_sh->batch_head = NULL; + spin_unlock_irq(&head_sh->stripe_lock); + + state = READ_ONCE(head_sh->state); + if (state & handle_flags) set_bit(STRIPE_HANDLE, &head_sh->state); } diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c index 8c684756d5fc2..d147ec4830811 100644 --- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c +++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c @@ -1202,7 +1202,8 @@ static int mtk_jpeg_release(struct file *file) struct mtk_jpeg_dev *jpeg = video_drvdata(file); struct mtk_jpeg_ctx *ctx = mtk_jpeg_file_to_ctx(file); - cancel_work_sync(&ctx->jpeg_work); + if (jpeg->variant->jpeg_worker) + cancel_work_sync(&ctx->jpeg_work); mutex_lock(&jpeg->lock); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c index 4bf8570e1b9e9..2d639b7899105 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c @@ -556,8 +556,8 @@ static void mxc_isi_remove(struct platform_device *pdev) mxc_isi_pipe_cleanup(pipe); } - mxc_isi_crossbar_cleanup(&isi->crossbar); mxc_isi_v4l2_cleanup(isi); + mxc_isi_crossbar_cleanup(&isi->crossbar); } static const struct of_device_id mxc_isi_of_match[] = { diff --git a/drivers/media/platform/qcom/camss/camss-vfe.c b/drivers/media/platform/qcom/camss/camss-vfe.c index 5baf0e3d4bc46..2ee4f9ae0ab50 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe.c +++ b/drivers/media/platform/qcom/camss/camss-vfe.c @@ -2053,7 +2053,7 @@ int msm_vfe_register_entities(struct vfe_device *vfe, v4l2_subdev_init(sd, &vfe_v4l2_ops); sd->internal_ops = &vfe_v4l2_internal_ops; sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; - if (i == VFE_LINE_PIX) + if (i == VFE_LINE_PIX && vfe->res->is_lite == false) snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d_%s", MSM_VFE_NAME, vfe->id, "pix"); else diff --git a/drivers/media/platform/qcom/iris/iris_platform_gen1.c b/drivers/media/platform/qcom/iris/iris_platform_gen1.c index df8e6bf9430ed..aa71f7f53ee34 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_gen1.c +++ b/drivers/media/platform/qcom/iris/iris_platform_gen1.c @@ -281,7 +281,7 @@ static const struct bw_info sm8250_bw_table_dec[] = { static const char * const sm8250_pmdomain_table[] = { "venus", "vcodec0" }; -static const char * const sm8250_opp_pd_table[] = { "mx" }; +static const char * const sm8250_opp_pd_table[] = { "mx", "mmcx" }; static const struct platform_clk_data sm8250_clk_table[] = { {IRIS_AXI_CLK, "iface" }, diff --git a/drivers/media/platform/qcom/iris/iris_probe.c b/drivers/media/platform/qcom/iris/iris_probe.c index ddaacda523ecb..487eb0917c0e2 100644 --- a/drivers/media/platform/qcom/iris/iris_probe.c +++ b/drivers/media/platform/qcom/iris/iris_probe.c @@ -64,6 +64,13 @@ static int iris_init_power_domains(struct iris_core *core) return ret; ret = devm_pm_domain_attach_list(core->dev, &iris_opp_pd_data, &core->opp_pmdomain_tbl); + /* backwards compatibility for incomplete ABI SM8250 */ + if (ret == -ENODEV && + of_device_is_compatible(core->dev->of_node, "qcom,sm8250-venus")) { + iris_opp_pd_data.num_pd_names--; + ret = devm_pm_domain_attach_list(core->dev, &iris_opp_pd_data, + &core->opp_pmdomain_tbl); + } if (ret < 0) return ret; diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index 7e639760c41d9..00fb6806c1298 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -882,6 +882,7 @@ static const struct venus_resources sdm845_res_v2 = { .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0", "vcodec1" }, .vcodec_pmdomains_num = 3, .opp_pmdomain = (const char *[]) { "cx" }, + .opp_pmdomain_num = 1, .vcodec_num = 2, .max_load = 3110400, /* 4096x2160@90 */ .hfi_version = HFI_VERSION_4XX, @@ -933,6 +934,7 @@ static const struct venus_resources sc7180_res = { .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "cx" }, + .opp_pmdomain_num = 1, .vcodec_num = 1, .hfi_version = HFI_VERSION_4XX, .vpu_version = VPU_VERSION_AR50, @@ -991,7 +993,8 @@ static const struct venus_resources sm8250_res = { .vcodec_clks_num = 1, .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, - .opp_pmdomain = (const char *[]) { "mx" }, + .opp_pmdomain = (const char *[]) { "mx", "mmcx" }, + .opp_pmdomain_num = 2, .vcodec_num = 1, .max_load = 7833600, .hfi_version = HFI_VERSION_6XX, @@ -1053,6 +1056,7 @@ static const struct venus_resources sc7280_res = { .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "cx" }, + .opp_pmdomain_num = 1, .vcodec_num = 1, .hfi_version = HFI_VERSION_6XX, .vpu_version = VPU_VERSION_IRIS2_1, @@ -1100,6 +1104,7 @@ static const struct venus_resources qcm2290_res = { .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "cx" }, + .opp_pmdomain_num = 1, .vcodec_num = 1, .hfi_version = HFI_VERSION_4XX, .vpu_version = VPU_VERSION_AR50_LITE, diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h index 7506f5d0f609a..70e7b40affa93 100644 --- a/drivers/media/platform/qcom/venus/core.h +++ b/drivers/media/platform/qcom/venus/core.h @@ -83,6 +83,7 @@ struct venus_resources { const char **vcodec_pmdomains; unsigned int vcodec_pmdomains_num; const char **opp_pmdomain; + unsigned int opp_pmdomain_num; unsigned int vcodec_num; const char * const resets[VIDC_RESETS_NUM_MAX]; unsigned int resets_num; diff --git a/drivers/media/platform/qcom/venus/helpers.c b/drivers/media/platform/qcom/venus/helpers.c index 747c388fe25fa..59eee3dd9e06c 100644 --- a/drivers/media/platform/qcom/venus/helpers.c +++ b/drivers/media/platform/qcom/venus/helpers.c @@ -954,8 +954,8 @@ static u32 get_framesize_raw_nv12(u32 width, u32 height) uv_sclines = ALIGN(((height + 1) >> 1), 16); y_plane = y_stride * y_sclines; - uv_plane = uv_stride * uv_sclines + SZ_4K; - size = y_plane + uv_plane + SZ_8K; + uv_plane = uv_stride * uv_sclines; + size = y_plane + uv_plane; return ALIGN(size, SZ_4K); } diff --git a/drivers/media/platform/qcom/venus/hfi_parser.c b/drivers/media/platform/qcom/venus/hfi_parser.c index 92765f9c88730..c4cf6cd50a9a0 100644 --- a/drivers/media/platform/qcom/venus/hfi_parser.c +++ b/drivers/media/platform/qcom/venus/hfi_parser.c @@ -268,7 +268,6 @@ static int hfi_platform_parser(struct venus_core *core, struct venus_inst *inst) const struct hfi_plat_caps *caps = NULL; u32 enc_codecs, dec_codecs, count = 0; unsigned int entries; - int ret; plat = hfi_platform_get(core->res->hfi_version); if (!plat) @@ -277,9 +276,8 @@ static int hfi_platform_parser(struct venus_core *core, struct venus_inst *inst) if (inst) return 0; - ret = hfi_platform_get_codecs(core, &enc_codecs, &dec_codecs, &count); - if (ret) - return ret; + if (plat->codecs) + plat->codecs(core, &enc_codecs, &dec_codecs, &count); if (plat->capabilities) caps = plat->capabilities(core, &entries); diff --git a/drivers/media/platform/qcom/venus/hfi_platform.c b/drivers/media/platform/qcom/venus/hfi_platform.c index cde7f93045ac4..f19572ab1d161 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform.c +++ b/drivers/media/platform/qcom/venus/hfi_platform.c @@ -2,9 +2,7 @@ /* * Copyright (c) 2020, The Linux Foundation. All rights reserved. */ -#include #include "hfi_platform.h" -#include "core.h" const struct hfi_platform *hfi_platform_get(enum hfi_version version) { @@ -73,25 +71,3 @@ hfi_platform_get_codec_lp_freq(struct venus_core *core, return freq; } - -int -hfi_platform_get_codecs(struct venus_core *core, u32 *enc_codecs, - u32 *dec_codecs, u32 *count) -{ - const struct hfi_platform *plat; - - plat = hfi_platform_get(core->res->hfi_version); - if (!plat) - return -EINVAL; - - if (plat->codecs) - plat->codecs(core, enc_codecs, dec_codecs, count); - - if (IS_IRIS2_1(core)) { - *enc_codecs &= ~HFI_VIDEO_CODEC_VP8; - *dec_codecs &= ~HFI_VIDEO_CODEC_VP8; - } - - return 0; -} - diff --git a/drivers/media/platform/qcom/venus/hfi_platform.h b/drivers/media/platform/qcom/venus/hfi_platform.h index 5e4f8013a6b1d..a0b6d19f3e1a0 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform.h +++ b/drivers/media/platform/qcom/venus/hfi_platform.h @@ -74,6 +74,4 @@ unsigned long hfi_platform_get_codec_vsp_freq(struct venus_core *core, unsigned long hfi_platform_get_codec_lp_freq(struct venus_core *core, enum hfi_version version, u32 codec, u32 session_type); -int hfi_platform_get_codecs(struct venus_core *core, u32 *enc_codecs, - u32 *dec_codecs, u32 *count); #endif diff --git a/drivers/media/platform/qcom/venus/hfi_platform_v4.c b/drivers/media/platform/qcom/venus/hfi_platform_v4.c index cda888b56b5d4..e0b3652bb4409 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform_v4.c +++ b/drivers/media/platform/qcom/venus/hfi_platform_v4.c @@ -136,8 +136,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_H264, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 1, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 120000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -173,8 +173,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_HEVC, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 1, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 120000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -195,8 +195,8 @@ static const struct hfi_plat_caps caps[] = { .caps[19] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, .caps[20] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[21] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, - .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .num_caps = 24, .pl[0] = {HFI_HEVC_PROFILE_MAIN, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, .pl[1] = {HFI_HEVC_PROFILE_MAIN10, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, @@ -210,8 +210,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_VP8, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 1, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 120000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -229,8 +229,8 @@ static const struct hfi_plat_caps caps[] = { .caps[16] = {HFI_CAPABILITY_P_FRAME_QP, 0, 127, 1}, .caps[17] = {HFI_CAPABILITY_MAX_WORKMODES, 1, 2, 1}, .caps[18] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, - .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .caps[21] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[22] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, .num_caps = 23, diff --git a/drivers/media/platform/qcom/venus/hfi_platform_v6.c b/drivers/media/platform/qcom/venus/hfi_platform_v6.c index d8568c08cc361..fb8d10ab34043 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform_v6.c +++ b/drivers/media/platform/qcom/venus/hfi_platform_v6.c @@ -173,8 +173,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_HEVC, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 8192, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 8192, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 8192, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 8192, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 64, 138240, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 160000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -195,8 +195,8 @@ static const struct hfi_plat_caps caps[] = { .caps[19] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, .caps[20] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[21] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, - .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .num_caps = 24, .pl[0] = {HFI_HEVC_PROFILE_MAIN, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, .pl[1] = {HFI_HEVC_PROFILE_MAIN10, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, @@ -210,8 +210,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_VP8, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 64, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 74000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -229,8 +229,8 @@ static const struct hfi_plat_caps caps[] = { .caps[16] = {HFI_CAPABILITY_P_FRAME_QP, 0, 127, 1}, .caps[17] = {HFI_CAPABILITY_MAX_WORKMODES, 1, 2, 1}, .caps[18] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, - .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .caps[21] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[22] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, .num_caps = 23, diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c index f0269524ac70e..14a4e8311a643 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.c +++ b/drivers/media/platform/qcom/venus/pm_helpers.c @@ -887,7 +887,7 @@ static int vcodec_domains_get(struct venus_core *core) }; struct dev_pm_domain_attach_data opp_pd_data = { .pd_names = res->opp_pmdomain, - .num_pd_names = 1, + .num_pd_names = res->opp_pmdomain_num, .pd_flags = PD_FLAG_DEV_LINK_ON | PD_FLAG_REQUIRED_OPP, }; @@ -904,6 +904,12 @@ static int vcodec_domains_get(struct venus_core *core) /* Attach the power domain for setting performance state */ ret = devm_pm_domain_attach_list(dev, &opp_pd_data, &core->opp_pmdomain); + /* backwards compatibility for incomplete ABI SM8250 */ + if (ret == -ENODEV && + of_device_is_compatible(dev->of_node, "qcom,sm8250-venus")) { + opp_pd_data.num_pd_names--; + ret = devm_pm_domain_attach_list(dev, &opp_pd_data, &core->opp_pmdomain); + } if (ret < 0) return ret; diff --git a/drivers/media/platform/rockchip/rga/rga-hw.c b/drivers/media/platform/rockchip/rga/rga-hw.c index 43ed742a16492..d1618bb247501 100644 --- a/drivers/media/platform/rockchip/rga/rga-hw.c +++ b/drivers/media/platform/rockchip/rga/rga-hw.c @@ -414,7 +414,7 @@ static void rga_cmd_set(struct rga_ctx *ctx, { struct rockchip_rga *rga = ctx->rga; - memset(rga->cmdbuf_virt, 0, RGA_CMDBUF_SIZE * 4); + memset(rga->cmdbuf_virt, 0, RGA_CMDBUF_SIZE); rga_cmd_set_src_addr(ctx, src->dma_desc_pa); /* diff --git a/drivers/media/platform/rockchip/rga/rga-hw.h b/drivers/media/platform/rockchip/rga/rga-hw.h index cc6bd7f5b0300..2b8537a5fd0d7 100644 --- a/drivers/media/platform/rockchip/rga/rga-hw.h +++ b/drivers/media/platform/rockchip/rga/rga-hw.h @@ -6,7 +6,7 @@ #ifndef __RGA_HW_H__ #define __RGA_HW_H__ -#define RGA_CMDBUF_SIZE 0x20 +#define RGA_CMDBUF_SIZE 0x80 /* Hardware limits */ #define MAX_WIDTH 8192 diff --git a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c index 02eb4a6cafade..0b80e84983f9e 100644 --- a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c +++ b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c @@ -311,7 +311,7 @@ dw_mipi_csi2rx_find_format(struct dw_mipi_csi2rx_device *csi2, u32 mbus_code) WARN_ON(csi2->formats_num == 0); for (unsigned int i = 0; i < csi2->formats_num; i++) { - const struct dw_mipi_csi2rx_format *format = &csi2->formats[i]; + const struct dw_mipi_csi2rx_format *format = &formats[i]; if (format->code == mbus_code) return format; @@ -433,7 +433,7 @@ dw_mipi_csi2rx_enum_mbus_code(struct v4l2_subdev *sd, if (code->index >= csi2->formats_num) return -EINVAL; - code->code = csi2->formats[code->index].code; + code->code = formats[code->index].code; return 0; default: return -EINVAL; @@ -470,6 +470,17 @@ static int dw_mipi_csi2rx_set_fmt(struct v4l2_subdev *sd, *src = *sink; + /* Store the CSIS format descriptor for active formats. */ + if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) { + csi2->formats = fmt ? : + dw_mipi_csi2rx_find_format(csi2, default_format.code); + + if (!csi2->formats) { + dev_err(csi2->dev, "Failed to find valid format\n"); + return -EINVAL; + } + } + return 0; } diff --git a/drivers/media/test-drivers/vidtv/vidtv_mux.c b/drivers/media/test-drivers/vidtv/vidtv_mux.c index f0134e38a1fb8..ea2ea53102f9f 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_mux.c +++ b/drivers/media/test-drivers/vidtv/vidtv_mux.c @@ -101,7 +101,8 @@ static int vidtv_mux_pid_ctx_init(struct vidtv_mux *m) /* add a ctx for all PMT sections */ while (p) { pid = vidtv_psi_get_pat_program_pid(p); - vidtv_mux_create_pid_ctx_once(m, pid); + if (!vidtv_mux_create_pid_ctx_once(m, pid)) + goto free; p = p->next; } @@ -170,6 +171,9 @@ static u32 vidtv_mux_push_si(struct vidtv_mux *m) nit_ctx = vidtv_mux_get_pid_ctx(m, VIDTV_NIT_PID); eit_ctx = vidtv_mux_get_pid_ctx(m, VIDTV_EIT_PID); + if (!pat_ctx || !sdt_ctx || !nit_ctx || !eit_ctx) + return 0; + pat_args.offset = m->mux_buf_offset; pat_args.continuity_counter = &pat_ctx->cc; @@ -186,6 +190,8 @@ static u32 vidtv_mux_push_si(struct vidtv_mux *m) } pmt_ctx = vidtv_mux_get_pid_ctx(m, pmt_pid); + if (!pmt_ctx) + continue; pmt_args.offset = m->mux_buf_offset; pmt_args.pmt = m->si.pmt_secs[i]; diff --git a/drivers/media/usb/uvc/uvc_status.c b/drivers/media/usb/uvc/uvc_status.c index 65f5356bebb39..b632cf5e3fe9b 100644 --- a/drivers/media/usb/uvc/uvc_status.c +++ b/drivers/media/usb/uvc/uvc_status.c @@ -316,6 +316,16 @@ static int uvc_status_start(struct uvc_device *dev, gfp_t flags) if (!dev->int_urb) return 0; + /* + * If the previous uvc_status_stop() call was from the async work, + * the work may still be running. Wait for it to finish before we submit + * the urb. + */ + flush_work(&dev->async_ctrl.work); + + /* Clear the flush status if we were previously stopped. */ + smp_store_release(&dev->flush_status, false); + return usb_submit_urb(dev->int_urb, flags); } @@ -336,6 +346,15 @@ static void uvc_status_stop(struct uvc_device *dev) */ smp_store_release(&dev->flush_status, true); + /* + * If we are called from the event work function, the URB is guaranteed + * to not be in flight as it has completed and has not been resubmitted. + * There's no need to cancel the work (which would deadlock), or to kill + * the URB. + */ + if (current_work() == &w->work) + return; + /* * Cancel any pending asynchronous work. If any status event was queued, * process it synchronously. @@ -354,15 +373,6 @@ static void uvc_status_stop(struct uvc_device *dev) */ if (cancel_work_sync(&w->work)) uvc_ctrl_status_event(w->chain, w->ctrl, w->data); - - /* - * From this point, there are no events on the queue and the status URB - * is dead. No events will be queued until uvc_status_start() is called. - * The barrier is needed to make sure that flush_status is visible to - * uvc_ctrl_status_event_work() when uvc_status_start() will be called - * again. - */ - smp_store_release(&dev->flush_status, false); } int uvc_status_resume(struct uvc_device *dev) diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index f6c8e32237968..aa3cb941762e3 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -494,6 +494,13 @@ static int uvc_commit_video(struct uvc_streaming *stream, * Clocks and timestamps */ +/* + * The accuracy of the hardware timestamping depends on having enough data to + * interpolate between the different clock domains. This value is sof cycles, + * this is, milliseconds. + */ +#define UVC_MIN_HW_TIMESTAMP_DIFF 100 + static inline ktime_t uvc_video_get_time(void) { if (uvc_clock_param == CLOCK_MONOTONIC) @@ -537,6 +544,15 @@ static void uvc_video_clock_add_sample(struct uvc_clock *clock, spin_unlock_irqrestore(&clock->lock, flags); } +static inline u16 sof_diff(u16 a, u16 b) +{ + /* + * Because the result is modulo 2048 (via & 2047), we do not need a + * special case for a < b. + */ + return (a - b) & 2047; +} + static void uvc_video_clock_decode(struct uvc_streaming *stream, struct uvc_buffer *buf, const u8 *data, int len) @@ -583,16 +599,7 @@ uvc_video_clock_decode(struct uvc_streaming *stream, struct uvc_buffer *buf, if (!has_scr) return; - /* - * To limit the amount of data, drop SCRs with an SOF identical to the - * previous one. This filtering is also needed to support UVC 1.5, where - * all the data packets of the same frame contains the same SOF. In that - * case only the first one will match the host_sof. - */ sample.dev_sof = get_unaligned_le16(&data[header_size - 2]); - if (sample.dev_sof == stream->clock.last_sof) - return; - sample.dev_stc = get_unaligned_le32(&data[header_size - 6]); /* @@ -664,6 +671,17 @@ uvc_video_clock_decode(struct uvc_streaming *stream, struct uvc_buffer *buf, } sample.dev_sof = (sample.dev_sof + stream->clock.sof_offset) & 2047; + + /* + * To limit the amount of data, drop SCRs with an SOF similar to the + * previous one. This filtering is also needed to support UVC 1.5, where + * all the data packets of the same frame contains the same SOF. In that + * case only the first one will match the host_sof. + */ + if (sof_diff(sample.dev_sof, stream->clock.last_sof) <= + (UVC_MIN_HW_TIMESTAMP_DIFF / stream->clock.size)) + return; + uvc_video_clock_add_sample(&stream->clock, &sample); stream->clock.last_sof = sample.dev_sof; } @@ -833,15 +851,22 @@ void uvc_video_clock_update(struct uvc_streaming *stream, y2 += 2048 << 16; /* - * Have at least 1/4 of a second of timestamps before we - * try to do any calculation. Otherwise we do not have enough - * precision. This value was determined by running Android CTS - * on different devices. + * If the buffer is not full, we want to gather at least 1/4th of + * timestamps before using HW timestamping. We do this to avoid jitter + * on the initial frames. + * + * If the buffer is full we would use it regardless of how much data + * it represents. This could be solved with an infinite big circular + * buffer, but RAM is expensive these days, specially the infinitely + * big. + * + * The value of UVC_MIN_HW_TIMESTAMP_DIFF was determined by running + * Android's CTS on different devices. * - * dev_sof runs at 1KHz, and we have a fixed point precision of - * 16 bits. + * y1 and y2 are dev_sof with a fixed point precision of 16 bits. */ - if ((y2 - y1) < ((1000 / 4) << 16)) + if (clock->size != clock->count && + (y2 - y1) < (UVC_MIN_HW_TIMESTAMP_DIFF << 16)) goto done; y = (u64)(y2 - y1) * (1ULL << 31) + (u64)y1 * (u64)x2 @@ -1149,7 +1174,9 @@ static void uvc_video_stats_stop(struct uvc_streaming *stream) * uvc_video_decode_end will never be called with a NULL buffer. */ static int uvc_video_decode_start(struct uvc_streaming *stream, - struct uvc_buffer *buf, const u8 *data, int len) + struct uvc_buffer *buf, + struct uvc_buffer *meta_buf, + const u8 *data, int len) { u8 header_len; u8 fid; @@ -1168,6 +1195,53 @@ static int uvc_video_decode_start(struct uvc_streaming *stream, header_len = data[0]; fid = data[1] & UVC_STREAM_FID; + /* + * Mark the buffer as done if we're at the beginning of a new frame. + * End of frame detection is better implemented by checking the EOF + * bit (FID bit toggling is delayed by one frame compared to the EOF + * bit), but some devices don't set the bit at end of frame (and the + * last payload can be lost anyway). We thus must check if the FID has + * been toggled. + * + * stream->last_fid is initialized to -1, and buf->bytesused to 0, + * so the first isochronous frame will never trigger an end of frame + * detection. + * + * Empty buffers (bytesused == 0) don't trigger end of frame detection + * as it doesn't make sense to return an empty buffer. This also + * avoids detecting end of frame conditions at FID toggling if the + * previous payload had the EOF bit set. + */ + if (fid != stream->last_fid && buf && buf->bytesused != 0) { + uvc_dbg(stream->dev, FRAME, + "Frame complete (FID bit toggled)\n"); + buf->state = UVC_BUF_STATE_READY; + + return -EAGAIN; + } + + /* + * Some cameras, when running two parallel streams (one MJPEG alongside + * another non-MJPEG stream), are known to lose the EOF packet for a frame. + * We can detect the end of a frame by checking for a new SOI marker, as + * the SOI always lies on the packet boundary between two frames for + * these devices. + */ + if (stream->dev->quirks & UVC_QUIRK_MJPEG_NO_EOF && + (stream->cur_format->fcc == V4L2_PIX_FMT_MJPEG || + stream->cur_format->fcc == V4L2_PIX_FMT_JPEG) && + buf && buf->bytesused != 0) { + const u8 *packet = data + header_len; + + if (len >= header_len + 2 && + packet[0] == 0xff && packet[1] == JPEG_MARKER_SOI) { + buf->state = UVC_BUF_STATE_READY; + buf->error = 1; + stream->last_fid ^= UVC_STREAM_FID; + return -EAGAIN; + } + } + /* * Increase the sequence number regardless of any buffer states, so * that discontinuous sequence numbers always indicate lost frames. @@ -1176,6 +1250,19 @@ static int uvc_video_decode_start(struct uvc_streaming *stream, stream->sequence++; if (stream->sequence) uvc_video_stats_update(stream); + + /* + * On a FID flip initialize sequence number and timestamp. + * + * The driver already takes care of injecting FID flips for + * UVC_QUIRK_STREAM_NO_FID and UVC_QUIRK_MJPEG_NO_EOF. + */ + if (buf) { + buf->buf.field = V4L2_FIELD_NONE; + buf->buf.sequence = stream->sequence; + buf->buf.vb2_buf.timestamp = + ktime_to_ns(uvc_video_get_time()); + } } uvc_video_clock_decode(stream, buf, data, len); @@ -1216,57 +1303,10 @@ static int uvc_video_decode_start(struct uvc_streaming *stream, return -ENODATA; } - buf->buf.field = V4L2_FIELD_NONE; - buf->buf.sequence = stream->sequence; - buf->buf.vb2_buf.timestamp = ktime_to_ns(uvc_video_get_time()); - /* TODO: Handle PTS and SCR. */ buf->state = UVC_BUF_STATE_ACTIVE; - } - - /* - * Mark the buffer as done if we're at the beginning of a new frame. - * End of frame detection is better implemented by checking the EOF - * bit (FID bit toggling is delayed by one frame compared to the EOF - * bit), but some devices don't set the bit at end of frame (and the - * last payload can be lost anyway). We thus must check if the FID has - * been toggled. - * - * stream->last_fid is initialized to -1, so the first isochronous - * frame will never trigger an end of frame detection. - * - * Empty buffers (bytesused == 0) don't trigger end of frame detection - * as it doesn't make sense to return an empty buffer. This also - * avoids detecting end of frame conditions at FID toggling if the - * previous payload had the EOF bit set. - */ - if (fid != stream->last_fid && buf->bytesused != 0) { - uvc_dbg(stream->dev, FRAME, - "Frame complete (FID bit toggled)\n"); - buf->state = UVC_BUF_STATE_READY; - return -EAGAIN; - } - - /* - * Some cameras, when running two parallel streams (one MJPEG alongside - * another non-MJPEG stream), are known to lose the EOF packet for a frame. - * We can detect the end of a frame by checking for a new SOI marker, as - * the SOI always lies on the packet boundary between two frames for - * these devices. - */ - if (stream->dev->quirks & UVC_QUIRK_MJPEG_NO_EOF && - (stream->cur_format->fcc == V4L2_PIX_FMT_MJPEG || - stream->cur_format->fcc == V4L2_PIX_FMT_JPEG)) { - const u8 *packet = data + header_len; - - if (len >= header_len + 2 && - packet[0] == 0xff && packet[1] == JPEG_MARKER_SOI && - buf->bytesused != 0) { - buf->state = UVC_BUF_STATE_READY; - buf->error = 1; - stream->last_fid ^= UVC_STREAM_FID; - return -EAGAIN; - } + if (meta_buf) + meta_buf->state = UVC_BUF_STATE_ACTIVE; } stream->last_fid = fid; @@ -1424,7 +1464,7 @@ static void uvc_video_decode_meta(struct uvc_streaming *stream, ktime_t time; const u8 *scr; - if (!meta_buf || length == 2) + if (length <= 2 || !meta_buf || meta_buf->state != UVC_BUF_STATE_ACTIVE) return; has_pts = mem[1] & UVC_STREAM_PTS; @@ -1541,7 +1581,7 @@ static void uvc_video_decode_isoc(struct uvc_urb *uvc_urb, /* Decode the payload header. */ mem = urb->transfer_buffer + urb->iso_frame_desc[i].offset; do { - ret = uvc_video_decode_start(stream, buf, mem, + ret = uvc_video_decode_start(stream, buf, meta_buf, mem, urb->iso_frame_desc[i].actual_length); if (ret == -EAGAIN) uvc_video_next_buffers(stream, &buf, &meta_buf); @@ -1590,7 +1630,8 @@ static void uvc_video_decode_bulk(struct uvc_urb *uvc_urb, */ if (stream->bulk.header_size == 0 && !stream->bulk.skip_payload) { do { - ret = uvc_video_decode_start(stream, buf, mem, len); + ret = uvc_video_decode_start(stream, buf, meta_buf, mem, + len); if (ret == -EAGAIN) uvc_video_next_buffers(stream, &buf, &meta_buf); } while (ret == -EAGAIN); diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c index 554c591e11133..55bcd5975d9fc 100644 --- a/drivers/media/v4l2-core/v4l2-common.c +++ b/drivers/media/v4l2-core/v4l2-common.c @@ -281,6 +281,7 @@ const struct v4l2_format_info *v4l2_format_info(u32 format) { .format = V4L2_PIX_FMT_Y212, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 }, { .format = V4L2_PIX_FMT_Y216, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 }, { .format = V4L2_PIX_FMT_YUV48_12, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 }, + { .format = V4L2_PIX_FMT_YUV24, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 }, { .format = V4L2_PIX_FMT_MT2110T, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2, .block_w = { 16, 8, 0, 0 }, .block_h = { 32, 16, 0, 0 }}, { .format = V4L2_PIX_FMT_MT2110R, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2, diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c index d620660da3311..64e41338cdf2d 100644 --- a/drivers/memory/tegra/mc.c +++ b/drivers/memory/tegra/mc.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1010,10 +1011,23 @@ static void tegra_mc_sync_state(struct device *dev) icc_sync_state(dev); } +static int tegra_mc_resume(struct device *dev) +{ + struct tegra_mc *mc = dev_get_drvdata(dev); + + if (mc->soc->ops && mc->soc->ops->resume) + mc->soc->ops->resume(mc); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(tegra_mc_pm_ops, NULL, tegra_mc_resume); + static struct platform_driver tegra_mc_driver = { .driver = { .name = "tegra-mc", .of_match_table = tegra_mc_of_match, + .pm = pm_sleep_ptr(&tegra_mc_pm_ops), .suppress_bind_attrs = true, .sync_state = tegra_mc_sync_state, }, diff --git a/drivers/memory/tegra/tegra186-emc.c b/drivers/memory/tegra/tegra186-emc.c index 03ebab6fbe68f..f71265b303b97 100644 --- a/drivers/memory/tegra/tegra186-emc.c +++ b/drivers/memory/tegra/tegra186-emc.c @@ -258,15 +258,13 @@ static int tegra186_emc_icc_get_init_bw(struct icc_node *node, u32 *avg, u32 *pe static int tegra186_emc_interconnect_init(struct tegra186_emc *emc) { - struct tegra_mc *mc = dev_get_drvdata(emc->dev->parent); - const struct tegra_mc_soc *soc = mc->soc; struct icc_node *node; int err; emc->provider.dev = emc->dev; emc->provider.set = tegra186_emc_icc_set_bw; emc->provider.data = &emc->provider; - emc->provider.aggregate = soc->icc_ops->aggregate; + emc->provider.aggregate = icc_std_aggregate; emc->provider.xlate = tegra186_emc_of_icc_xlate; emc->provider.get_bw = tegra186_emc_icc_get_init_bw; diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c index a01fe313558e7..ce33907bfc242 100644 --- a/drivers/memstick/core/ms_block.c +++ b/drivers/memstick/core/ms_block.c @@ -1338,6 +1338,10 @@ static int msb_ftl_initialize(struct msb_data *msb) return 0; msb->zone_count = msb->block_count / MS_BLOCKS_IN_ZONE; + if (msb->block_count > MS_MAX_ZONES * MS_BLOCKS_IN_ZONE) { + pr_err("Too many blocks: %d\n", msb->block_count); + return -EINVAL; + } msb->logical_block_count = msb->zone_count * 496 - 2; msb->used_blocks_bitmap = bitmap_zalloc(msb->block_count, GFP_KERNEL); diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c index 39430dd44e30c..56fb7cceafc6c 100644 --- a/drivers/mfd/cros_ec_dev.c +++ b/drivers/mfd/cros_ec_dev.c @@ -195,7 +195,6 @@ static int ec_device_probe(struct platform_device *pdev) if (!ec) return retval; - dev_set_drvdata(dev, ec); ec->ec_dev = dev_get_drvdata(dev->parent); ec->dev = dev; ec->cmd_offset = ec_platform->cmd_offset; @@ -237,6 +236,8 @@ static int ec_device_probe(struct platform_device *pdev) if (retval) goto failed; + dev_set_drvdata(dev, ec); + /* check whether this EC is a sensor hub. */ if (cros_ec_get_sensor_count(ec) > 0) { retval = mfd_add_hotplug_devices(ec->dev, diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c index 166881751e698..ed6d93893de04 100644 --- a/drivers/mfd/cs42l43.c +++ b/drivers/mfd/cs42l43.c @@ -722,7 +722,7 @@ static void cs42l43_mcu_load_firmware(const struct firmware *firmware, void *con unsigned int loadaddr, val; int ret; - if (!firmware) { + if (!firmware || firmware->size < sizeof(*hdr)) { dev_err(cs42l43->dev, "Failed to load firmware\n"); cs42l43->firmware_error = -ENODEV; goto err; diff --git a/drivers/mfd/rsmu_i2c.c b/drivers/mfd/rsmu_i2c.c index cba64f107a2fd..9e5fc8259eec2 100644 --- a/drivers/mfd/rsmu_i2c.c +++ b/drivers/mfd/rsmu_i2c.c @@ -134,14 +134,10 @@ static int rsmu_i2c_write_device(struct rsmu_ddata *rsmu, u8 reg, u8 *buf, u8 by static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u32 reg, rsmu_rw_device rsmu_write_device) { - u32 page = reg & RSMU_CM_PAGE_MASK; + u32 page = (reg | RSMU_CM_SCSR_BASE) & RSMU_CM_PAGE_MASK; u8 buf[4]; int err; - /* Do not modify offset register for none-scsr registers */ - if (reg < RSMU_CM_SCSR_BASE) - return 0; - /* Simply return if we are on the same page */ if (rsmu->page == page) return 0; diff --git a/drivers/mfd/rsmu_spi.c b/drivers/mfd/rsmu_spi.c index 39d9be1e141fb..c931d8cea0a1a 100644 --- a/drivers/mfd/rsmu_spi.c +++ b/drivers/mfd/rsmu_spi.c @@ -101,11 +101,8 @@ static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u32 reg) switch (rsmu->type) { case RSMU_CM: - /* Do not modify page register for none-scsr registers */ - if (reg < RSMU_CM_SCSR_BASE) - return 0; page_reg = RSMU_CM_PAGE_ADDR; - page = reg & RSMU_PAGE_MASK; + page = (reg | RSMU_CM_SCSR_BASE) & RSMU_PAGE_MASK; buf[0] = (u8)(page & 0xFF); buf[1] = (u8)((page >> 8) & 0xFF); buf[2] = (u8)((page >> 16) & 0xFF); diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c index 0ee6d8940e690..8276456b142fe 100644 --- a/drivers/mfd/sm501.c +++ b/drivers/mfd/sm501.c @@ -704,9 +704,11 @@ static int sm501_register_device(struct sm501_devdata *sm, if (ret >= 0) { dev_dbg(sm->dev, "registered %s\n", pdev->name); list_add_tail(&smdev->list, &sm->devices); - } else + } else { dev_err(sm->dev, "error registering %s (%d)\n", pdev->name, ret); + platform_device_put(pdev); + } return ret; } diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c index 8d5fe2b60bfa5..f5f8054466033 100644 --- a/drivers/mfd/tps6586x.c +++ b/drivers/mfd/tps6586x.c @@ -397,7 +397,7 @@ static int tps6586x_add_subdevs(struct tps6586x *tps6586x, pdev->dev.parent = tps6586x->dev; pdev->dev.platform_data = subdev->platform_data; - pdev->dev.of_node = subdev->of_node; + pdev->dev.of_node = of_node_get(subdev->of_node); ret = platform_device_add(pdev); if (ret) { diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 0274e8d07660d..54a923ba4f1e7 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -2715,7 +2715,6 @@ static void mmc_blk_rpmb_device_release(struct device *dev) { struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev); - rpmb_dev_unregister(rpmb->rdev); mmc_blk_put(rpmb->md); ida_free(&mmc_rpmb_ida, rpmb->id); kfree(rpmb); @@ -2930,8 +2929,8 @@ static int mmc_blk_alloc_rpmb_part(struct mmc_card *card, } static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb) - { + rpmb_dev_unregister(rpmb->rdev); cdev_device_del(&rpmb->chrdev, &rpmb->dev); put_device(&rpmb->dev); } diff --git a/drivers/mmc/core/mmc_test.c b/drivers/mmc/core/mmc_test.c index ab38e4c45a8da..4dc16649e61d4 100644 --- a/drivers/mmc/core/mmc_test.c +++ b/drivers/mmc/core/mmc_test.c @@ -318,9 +318,9 @@ static void mmc_test_free_mem(struct mmc_test_mem *mem) { if (!mem) return; - while (mem->cnt--) - __free_pages(mem->arr[mem->cnt].page, - mem->arr[mem->cnt].order); + for (unsigned int i = 0; i < mem->cnt; i++) + __free_pages(mem->arr[i].page, + mem->arr[i].order); kfree(mem); } @@ -341,6 +341,7 @@ static struct mmc_test_mem *mmc_test_alloc_mem(unsigned long min_sz, unsigned long page_cnt = 0; unsigned long limit = nr_free_buffer_pages() >> 4; struct mmc_test_mem *mem; + unsigned int idx = 0; if (max_page_cnt > limit) max_page_cnt = limit; @@ -375,23 +376,26 @@ static struct mmc_test_mem *mmc_test_alloc_mem(unsigned long min_sz, goto out_free; break; } - mem->arr[mem->cnt].page = page; - mem->arr[mem->cnt].order = order; - mem->cnt += 1; + mem->arr[idx].page = page; + mem->arr[idx].order = order; + idx += 1; if (max_page_cnt <= (1UL << order)) break; max_page_cnt -= 1UL << order; page_cnt += 1UL << order; - if (mem->cnt >= max_segs) { + if (idx >= mem->cnt) { if (page_cnt < min_page_cnt) goto out_free; break; } } + mem->cnt = idx; + return mem; out_free: + mem->cnt = idx; mmc_test_free_mem(mem); return NULL; } diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c index 18ecddd6df6f2..18f4905c15b9b 100644 --- a/drivers/mmc/host/sdhci-esdhc-imx.c +++ b/drivers/mmc/host/sdhci-esdhc-imx.c @@ -1326,19 +1326,21 @@ static int esdhc_change_pinstate(struct sdhci_host *host, dev_dbg(mmc_dev(host->mmc), "change pinctrl state for uhs %d\n", uhs); - if (IS_ERR(imx_data->pinctrl) || - IS_ERR(imx_data->pins_100mhz) || - IS_ERR(imx_data->pins_200mhz)) + if (IS_ERR(imx_data->pinctrl)) return -EINVAL; switch (uhs) { case MMC_TIMING_UHS_SDR50: case MMC_TIMING_UHS_DDR50: + if (IS_ERR(imx_data->pins_100mhz)) + return -EINVAL; pinctrl = imx_data->pins_100mhz; break; case MMC_TIMING_UHS_SDR104: case MMC_TIMING_MMC_HS200: case MMC_TIMING_MMC_HS400: + if (IS_ERR(imx_data->pins_200mhz)) + return -EINVAL; pinctrl = imx_data->pins_200mhz; break; default: @@ -1349,6 +1351,23 @@ static int esdhc_change_pinstate(struct sdhci_host *host, return pinctrl_select_state(imx_data->pinctrl, pinctrl); } +static void esdhc_set_dll_override(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host); + struct esdhc_platform_data *boarddata = &imx_data->boarddata; + u32 v; + + if (!boarddata->delay_line) + return; + + v = boarddata->delay_line << ESDHC_DLL_OVERRIDE_VAL_SHIFT | + (1 << ESDHC_DLL_OVERRIDE_EN_SHIFT); + if (is_imx53_esdhc(imx_data)) + v <<= 1; + writel(v, host->ioaddr + ESDHC_DLL_CTRL); +} + /* * For HS400 eMMC, there is a data_strobe line. This signal is generated * by the device and used for data output and CRC status response output @@ -1404,7 +1423,6 @@ static void esdhc_set_uhs_signaling(struct sdhci_host *host, unsigned timing) u32 m; struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host); - struct esdhc_platform_data *boarddata = &imx_data->boarddata; /* disable ddr mode and disable HS400 mode */ m = readl(host->ioaddr + ESDHC_MIX_CTRL); @@ -1425,15 +1443,7 @@ static void esdhc_set_uhs_signaling(struct sdhci_host *host, unsigned timing) m |= ESDHC_MIX_CTRL_DDREN; writel(m, host->ioaddr + ESDHC_MIX_CTRL); imx_data->is_ddr = 1; - if (boarddata->delay_line) { - u32 v; - v = boarddata->delay_line << - ESDHC_DLL_OVERRIDE_VAL_SHIFT | - (1 << ESDHC_DLL_OVERRIDE_EN_SHIFT); - if (is_imx53_esdhc(imx_data)) - v <<= 1; - writel(v, host->ioaddr + ESDHC_DLL_CTRL); - } + esdhc_set_dll_override(host); break; case MMC_TIMING_MMC_HS400: m |= ESDHC_MIX_CTRL_DDREN | ESDHC_MIX_CTRL_HS400_EN; @@ -2051,7 +2061,9 @@ static int sdhci_esdhc_suspend(struct device *dev) * 2, make sure the pm_runtime_force_resume() in sdhci_esdhc_resume() really * invoke its ->runtime_resume callback (needs_force_resume = 1). */ - pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; if ((imx_data->socdata->flags & ESDHC_FLAG_STATE_LOST_IN_LPMODE) && (host->tuning_mode != SDHCI_TUNING_MODE_1)) { @@ -2064,15 +2076,14 @@ static int sdhci_esdhc_suspend(struct device *dev) * to save the tuning delay value just in case the usdhc * lost power during system PM. */ - if (mmc_card_keep_power(host->mmc) && mmc_card_wake_sdio_irq(host->mmc) && - esdhc_is_usdhc(imx_data)) + if (mmc_card_keep_power(host->mmc) && esdhc_is_usdhc(imx_data)) sdhc_esdhc_tuning_save(host); + /* The irqs of imx are not shared. It is safe to disable */ + disable_irq(host->irq); + if (device_may_wakeup(dev)) { - /* The irqs of imx are not shared. It is safe to disable */ - disable_irq(host->irq); - ret = sdhci_enable_irq_wakeups(host); - if (!ret) + if (!sdhci_enable_irq_wakeups(host)) dev_warn(dev, "Failed to enable irq wakeup\n"); } else { /* @@ -2083,12 +2094,12 @@ static int sdhci_esdhc_suspend(struct device *dev) * other function like GPIO function to save power in PM, * which finally block the SDIO wakeup function. */ - ret = pinctrl_pm_select_sleep_state(dev); - if (ret) - return ret; + if (pinctrl_pm_select_sleep_state(dev)) + dev_warn(dev, "Failed to select sleep pinctrl state\n"); } - ret = mmc_gpio_set_cd_wake(host->mmc, true); + if (mmc_gpio_set_cd_wake(host->mmc, true)) + dev_warn(dev, "Failed to enable cd wake\n"); /* * Make sure invoke runtime_suspend to gate off clock. @@ -2096,7 +2107,7 @@ static int sdhci_esdhc_suspend(struct device *dev) */ pm_runtime_force_suspend(dev); - return ret; + return 0; } static int sdhci_esdhc_resume(struct device *dev) @@ -2106,31 +2117,45 @@ static int sdhci_esdhc_resume(struct device *dev) struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host); int ret; - pm_runtime_force_resume(dev); + if (!device_may_wakeup(dev)) { + ret = esdhc_change_pinstate(host, host->timing); + if (ret) + dev_warn(dev, "Failed to restore pinctrl state\n"); + } - ret = mmc_gpio_set_cd_wake(host->mmc, false); + ret = pm_runtime_force_resume(dev); if (ret) return ret; + mmc_gpio_set_cd_wake(host->mmc, false); + /* re-initialize hw state in case it's lost in low power mode */ sdhci_esdhc_imx_hwinit(host); - if (host->irq_wake_enabled) { + if (host->irq_wake_enabled) sdhci_disable_irq_wakeups(host); - enable_irq(host->irq); - } + + enable_irq(host->irq); /* * restore the saved tuning delay value for the device which keep * power during system PM. */ - if (mmc_card_keep_power(host->mmc) && mmc_card_wake_sdio_irq(host->mmc) && - esdhc_is_usdhc(imx_data)) + if (mmc_card_keep_power(host->mmc) && esdhc_is_usdhc(imx_data)) { sdhc_esdhc_tuning_restore(host); + /* + * Restore DLL override for DDR modes. hwinit unconditionally + * clears ESDHC_DLL_CTRL, but the card is still in DDR mode. + */ + if (host->timing == MMC_TIMING_UHS_DDR50 || + host->timing == MMC_TIMING_MMC_DDR52) + esdhc_set_dll_override(host); + } + pm_runtime_put_autosuspend(dev); - return ret; + return 0; } static int sdhci_esdhc_runtime_suspend(struct device *dev) diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index b9ecd91f44ad4..29af07561a861 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -2441,13 +2441,16 @@ static int dwcmshc_probe(struct platform_device *pdev) return err; priv->bus_clk = devm_clk_get(dev, "bus"); - if (!IS_ERR(priv->bus_clk)) - clk_prepare_enable(priv->bus_clk); + if (!IS_ERR(priv->bus_clk)) { + err = clk_prepare_enable(priv->bus_clk); + if (err) + goto err_clk; + } } err = mmc_of_parse(host->mmc); if (err) - goto err_clk; + goto err_bus_clk; sdhci_get_of_property(pdev); @@ -2461,7 +2464,7 @@ static int dwcmshc_probe(struct platform_device *pdev) if (pltfm_data->init) { err = pltfm_data->init(&pdev->dev, host, priv); if (err) - goto err_clk; + goto err_bus_clk; } #ifdef CONFIG_ACPI @@ -2507,9 +2510,10 @@ static int dwcmshc_probe(struct platform_device *pdev) err_rpm: pm_runtime_disable(dev); pm_runtime_put_noidle(dev); +err_bus_clk: + clk_disable_unprepare(priv->bus_clk); err_clk: clk_disable_unprepare(pltfm_host->clk); - clk_disable_unprepare(priv->bus_clk); clk_bulk_disable_unprepare(priv->num_other_clks, priv->other_clks); return err; } diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c index 6c3cb2f1c9d32..2dae474dcd067 100644 --- a/drivers/mmc/host/vub300.c +++ b/drivers/mmc/host/vub300.c @@ -1586,7 +1586,7 @@ static int __command_write_data(struct vub300_mmc_host *vub300, return linear_length; } -static void __vub300_command_response(struct vub300_mmc_host *vub300, +static bool __vub300_command_response(struct vub300_mmc_host *vub300, struct mmc_command *cmd, struct mmc_data *data, int data_length) { @@ -1598,17 +1598,11 @@ static void __vub300_command_response(struct vub300_mmc_host *vub300, msecs_to_jiffies(msec_timeout)); if (respretval == 0) { /* TIMED OUT */ /* we don't know which of "out" and "res" if any failed */ - int result; vub300->usb_timed_out = 1; usb_kill_urb(vub300->command_out_urb); usb_kill_urb(vub300->command_res_urb); cmd->error = -ETIMEDOUT; - result = usb_lock_device_for_reset(vub300->udev, - vub300->interface); - if (result == 0) { - result = usb_reset_device(vub300->udev); - usb_unlock_device(vub300->udev); - } + return true; } else if (respretval < 0) { /* we don't know which of "out" and "res" if any failed */ usb_kill_urb(vub300->command_out_urb); @@ -1704,6 +1698,8 @@ static void __vub300_command_response(struct vub300_mmc_host *vub300, } else { cmd->error = -EINVAL; } + + return false; } static void construct_request_response(struct vub300_mmc_host *vub300, @@ -1749,6 +1745,7 @@ static void vub300_cmndwork_thread(struct work_struct *work) struct mmc_request *req = vub300->req; struct mmc_command *cmd = vub300->cmd; struct mmc_data *data = vub300->data; + bool reset_device; int data_length; mutex_lock(&vub300->cmd_mutex); init_completion(&vub300->command_complete); @@ -1771,7 +1768,8 @@ static void vub300_cmndwork_thread(struct work_struct *work) data_length = __command_read_data(vub300, cmd, data); else data_length = __command_write_data(vub300, cmd, data); - __vub300_command_response(vub300, cmd, data, data_length); + reset_device = __vub300_command_response(vub300, cmd, + data, data_length); vub300->req = NULL; vub300->cmd = NULL; vub300->data = NULL; @@ -1779,6 +1777,16 @@ static void vub300_cmndwork_thread(struct work_struct *work) if (cmd->error == -ENOMEDIUM) check_vub300_port_status(vub300); mutex_unlock(&vub300->cmd_mutex); + if (reset_device) { + int result; + + result = usb_lock_device_for_reset(vub300->udev, + vub300->interface); + if (result == 0) { + result = usb_reset_device(vub300->udev); + usb_unlock_device(vub300->udev); + } + } mmc_request_done(vub300->mmc, req); kref_put(&vub300->kref, vub300_delete); return; @@ -2336,12 +2344,16 @@ static int vub300_probe(struct usb_interface *interface, interface_to_InterfaceNumber(interface)); retval = mmc_add_host(mmc); if (retval) - goto err_delete_timer; + goto err_stop_io; return 0; -err_delete_timer: - timer_delete_sync(&vub300->inactivity_timer); +err_stop_io: + vub300->interface = NULL; + kref_put(&vub300->kref, vub300_delete); + + return retval; + err_free_host: mmc_free_host(mmc); /* diff --git a/drivers/mtd/devices/mchp23k256.c b/drivers/mtd/devices/mchp23k256.c index cef5f9677d39e..66e058df4c328 100644 --- a/drivers/mtd/devices/mchp23k256.c +++ b/drivers/mtd/devices/mchp23k256.c @@ -188,7 +188,7 @@ static int mchp23k256_probe(struct spi_device *spi) data = dev_get_platdata(&spi->dev); - flash->caps = of_device_get_match_data(&spi->dev); + flash->caps = spi_get_device_match_data(spi); if (!flash->caps) flash->caps = &mchp23k256_caps; diff --git a/drivers/mtd/devices/slram.c b/drivers/mtd/devices/slram.c index 69cb63d99f573..48c2bc6b65eec 100644 --- a/drivers/mtd/devices/slram.c +++ b/drivers/mtd/devices/slram.c @@ -129,6 +129,7 @@ static int slram_write(struct mtd_info *mtd, loff_t to, size_t len, static int register_device(char *name, unsigned long start, unsigned long length) { slram_mtd_list_t **curmtd; + int ret = -ENOMEM; curmtd = &slram_mtdlist; while (*curmtd) { @@ -155,14 +156,15 @@ static int register_device(char *name, unsigned long start, unsigned long length if (!(*curmtd)->mtdinfo) { E("slram: Cannot allocate new MTD device.\n"); - return(-ENOMEM); + goto err_free_list; } if (!(((slram_priv_t *)(*curmtd)->mtdinfo->priv)->start = memremap(start, length, MEMREMAP_WB | MEMREMAP_WT | MEMREMAP_WC))) { E("slram: memremap failed\n"); - return -EIO; + ret = -EIO; + goto err_free_priv; } ((slram_priv_t *)(*curmtd)->mtdinfo->priv)->end = ((slram_priv_t *)(*curmtd)->mtdinfo->priv)->start + length; @@ -183,10 +185,8 @@ static int register_device(char *name, unsigned long start, unsigned long length if (mtd_device_register((*curmtd)->mtdinfo, NULL, 0)) { E("slram: Failed to register new device\n"); - memunmap(((slram_priv_t *)(*curmtd)->mtdinfo->priv)->start); - kfree((*curmtd)->mtdinfo->priv); - kfree((*curmtd)->mtdinfo); - return(-EAGAIN); + ret = -EAGAIN; + goto err_unmap; } T("slram: Registered device %s from %luKiB to %luKiB\n", name, (start / 1024), ((start + length) / 1024)); @@ -194,6 +194,16 @@ static int register_device(char *name, unsigned long start, unsigned long length ((slram_priv_t *)(*curmtd)->mtdinfo->priv)->start, ((slram_priv_t *)(*curmtd)->mtdinfo->priv)->end); return(0); + +err_unmap: + memunmap(((slram_priv_t *)(*curmtd)->mtdinfo->priv)->start); +err_free_priv: + kfree((*curmtd)->mtdinfo->priv); +err_free_list: + kfree((*curmtd)->mtdinfo); + kfree(*curmtd); + *curmtd = NULL; + return ret; } static void unregister_devices(void) diff --git a/drivers/mtd/maps/vmu-flash.c b/drivers/mtd/maps/vmu-flash.c index 75e06d249ce99..10244e6731d07 100644 --- a/drivers/mtd/maps/vmu-flash.c +++ b/drivers/mtd/maps/vmu-flash.c @@ -547,6 +547,7 @@ static void vmu_queryblocks(struct mapleq *mq) mpart->partition = card->partition; mtd_cur->priv = mpart; mtd_cur->owner = THIS_MODULE; + mtd_cur->dev.parent = &mdev->dev; pcache = kzalloc_obj(struct vmu_cache); if (!pcache) @@ -609,7 +610,7 @@ static int vmu_connect(struct maple_device *mdev) basic_flash_data = be32_to_cpu(mdev->devinfo.function_data[c - 1]); - card = kmalloc_obj(struct memcard); + card = kzalloc_obj(struct memcard); if (!card) { error = -ENOMEM; goto fail_nomem; @@ -627,13 +628,13 @@ static int vmu_connect(struct maple_device *mdev) * Not sure there are actually any multi-partition devices in the * real world, but the hardware supports them, so, so will we */ - card->parts = kmalloc_objs(struct vmupart, card->partitions); + card->parts = kzalloc_objs(struct vmupart, card->partitions); if (!card->parts) { error = -ENOMEM; goto fail_partitions; } - card->mtd = kmalloc_objs(struct mtd_info, card->partitions); + card->mtd = kzalloc_objs(struct mtd_info, card->partitions); if (!card->mtd) { error = -ENOMEM; goto fail_mtd_info; diff --git a/drivers/mtd/mtd_virt_concat.c b/drivers/mtd/mtd_virt_concat.c index 37075ead0f337..5db6e648927e2 100644 --- a/drivers/mtd/mtd_virt_concat.c +++ b/drivers/mtd/mtd_virt_concat.c @@ -321,8 +321,10 @@ int mtd_virt_concat_create_join(void) if (concat->mtd.name) { ret = memcmp(concat->mtd.name, name, name_sz); - if (ret == 0) + if (ret == 0) { + kfree(name); continue; + } } mtd = mtd_concat_create(concat->subdev, concat->num_subdev, name); if (!mtd) { diff --git a/drivers/mtd/nand/onenand/onenand_samsung.c b/drivers/mtd/nand/onenand/onenand_samsung.c index 6d6aa709a21f8..b7b7758ce4d86 100644 --- a/drivers/mtd/nand/onenand/onenand_samsung.c +++ b/drivers/mtd/nand/onenand/onenand_samsung.c @@ -554,6 +554,9 @@ static int s5pc110_dma_poll(dma_addr_t dst, dma_addr_t src, size_t count, int di } while (!(status & S5PC110_DMA_TRANS_STATUS_TD) && time_before(jiffies, timeout)); + if (!(status & S5PC110_DMA_TRANS_STATUS_TD)) + return -ETIMEDOUT; + writel(S5PC110_DMA_TRANS_CMD_TDC, base + S5PC110_DMA_TRANS_CMD); return 0; @@ -608,7 +611,9 @@ static int s5pc110_dma_irq(dma_addr_t dst, dma_addr_t src, size_t count, int dir writel(S5PC110_DMA_TRANS_CMD_TR, base + S5PC110_DMA_TRANS_CMD); - wait_for_completion_timeout(&onenand->complete, msecs_to_jiffies(20)); + if (!wait_for_completion_timeout(&onenand->complete, + msecs_to_jiffies(20))) + return -ETIMEDOUT; return 0; } diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c index fad0334f759dd..a88ac2cfaccdc 100644 --- a/drivers/mtd/nand/raw/fsl_ifc_nand.c +++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c @@ -684,8 +684,15 @@ static int fsl_ifc_read_page(struct nand_chip *chip, uint8_t *buf, return check_erased_page(chip, buf); } - if (ctrl->nand_stat != IFC_NAND_EVTER_STAT_OPC) + if (!ctrl->nand_stat) { mtd->ecc_stats.failed++; + return -ETIMEDOUT; + } + + if (ctrl->nand_stat != IFC_NAND_EVTER_STAT_OPC) { + mtd->ecc_stats.failed++; + return -EIO; + } return nctrl->max_bitflips; } diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c index 19b13ae536d48..8f6a89d9ba83f 100644 --- a/drivers/mtd/nand/raw/lpc32xx_mlc.c +++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c @@ -396,6 +396,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len, struct lpc32xx_nand_host *host = nand_get_controller_data(chip); struct dma_async_tx_descriptor *desc; int flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; + unsigned long time_left; int res; sg_init_one(&host->sgl, mem, len); @@ -410,6 +411,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len, flags); if (!desc) { dev_err(mtd->dev.parent, "Failed to prepare slave sg\n"); + res = -ENXIO; goto out1; } @@ -420,7 +422,13 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len, dmaengine_submit(desc); dma_async_issue_pending(host->dma_chan); - wait_for_completion_timeout(&host->comp_dma, msecs_to_jiffies(1000)); + time_left = wait_for_completion_timeout(&host->comp_dma, + msecs_to_jiffies(1000)); + if (!time_left) { + dmaengine_terminate_sync(host->dma_chan); + res = -ETIMEDOUT; + goto out1; + } dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1, DMA_BIDIRECTIONAL); @@ -428,7 +436,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len, out1: dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1, DMA_BIDIRECTIONAL); - return -ENXIO; + return res; } static int lpc32xx_read_page(struct nand_chip *chip, uint8_t *buf, diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c index 3ca30e7dce336..10c8080207f49 100644 --- a/drivers/mtd/nand/raw/lpc32xx_slc.c +++ b/drivers/mtd/nand/raw/lpc32xx_slc.c @@ -430,6 +430,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, dma_addr_t dma, struct dma_async_tx_descriptor *desc; int flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; int res; + unsigned long time_left; host->dma_slave_config.direction = dir; host->dma_slave_config.src_addr = dma; @@ -467,12 +468,19 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, dma_addr_t dma, dmaengine_submit(desc); dma_async_issue_pending(host->dma_chan); - wait_for_completion_timeout(&host->comp, msecs_to_jiffies(1000)); + time_left = wait_for_completion_timeout(&host->comp, + msecs_to_jiffies(1000)); + if (!time_left) { + dmaengine_terminate_sync(host->dma_chan); + res = -ETIMEDOUT; + } else { + res = 0; + } dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1, DMA_BIDIRECTIONAL); - return 0; + return res; out1: dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1, DMA_BIDIRECTIONAL); diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index d6d3e17ab4079..a5b278ab9384f 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -175,7 +175,7 @@ void nand_select_target(struct nand_chip *chip, unsigned int cs) * cs should always lie between 0 and nanddev_ntargets(), when that's * not the case it's a bug and the caller should be fixed. */ - if (WARN_ON(cs > nanddev_ntargets(&chip->base))) + if (WARN_ON(cs >= nanddev_ntargets(&chip->base))) return; chip->cur_cs = cs; @@ -1216,32 +1216,32 @@ static int nand_lp_exec_read_page_op(struct nand_chip *chip, unsigned int page, return nand_exec_op(chip, &op); } -static unsigned int rawnand_last_page_of_lun(unsigned int pages_per_lun, unsigned int lun) +static unsigned int rawnand_last_page_of_block(unsigned int ppb, unsigned int block) { - /* lun is expected to be very small */ - return (lun * pages_per_lun) + pages_per_lun - 1; + /* block is expected to be very small */ + return (block * ppb) + ppb - 1; } static void rawnand_cap_cont_reads(struct nand_chip *chip) { struct nand_memory_organization *memorg; - unsigned int ppl, first_lun, last_lun; + unsigned int ppb, first_block, last_block; memorg = nanddev_get_memorg(&chip->base); - ppl = memorg->pages_per_eraseblock * memorg->eraseblocks_per_lun; - first_lun = chip->cont_read.first_page / ppl; - last_lun = chip->cont_read.last_page / ppl; + ppb = memorg->pages_per_eraseblock; + first_block = chip->cont_read.first_page / ppb; + last_block = chip->cont_read.last_page / ppb; - /* Prevent sequential cache reads across LUN boundaries */ - if (first_lun != last_lun) - chip->cont_read.pause_page = rawnand_last_page_of_lun(ppl, first_lun); + /* Prevent sequential cache reads across block boundaries */ + if (first_block != last_block) + chip->cont_read.pause_page = rawnand_last_page_of_block(ppb, first_block); else chip->cont_read.pause_page = chip->cont_read.last_page; if (chip->cont_read.first_page == chip->cont_read.pause_page) { chip->cont_read.first_page++; chip->cont_read.pause_page = min(chip->cont_read.last_page, - rawnand_last_page_of_lun(ppl, first_lun + 1)); + rawnand_last_page_of_block(ppb, first_block + 1)); } if (chip->cont_read.first_page >= chip->cont_read.last_page) diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c index f2c65eb7a8d9a..06f8f1e14b9c2 100644 --- a/drivers/mtd/nand/raw/pl35x-nand-controller.c +++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c @@ -1155,7 +1155,7 @@ static int pl35x_nand_probe(struct platform_device *pdev) nfc->controller.ops = &pl35x_nandc_ops; INIT_LIST_HEAD(&nfc->chips); - nfc->conf_regs = devm_ioremap_resource(&smc_amba->dev, &smc_amba->res); + nfc->conf_regs = devm_ioremap_resource(nfc->dev, &smc_amba->res); if (IS_ERR(nfc->conf_regs)) return PTR_ERR(nfc->conf_regs); diff --git a/drivers/mtd/spi-nor/Kconfig b/drivers/mtd/spi-nor/Kconfig index 24cd25de2b8b7..fd05a24d64a96 100644 --- a/drivers/mtd/spi-nor/Kconfig +++ b/drivers/mtd/spi-nor/Kconfig @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only menuconfig MTD_SPI_NOR tristate "SPI NOR device support" - depends on MTD depends on MTD && SPI_MASTER select SPI_MEM help diff --git a/drivers/mtd/spi-nor/debugfs.c b/drivers/mtd/spi-nor/debugfs.c index 14ba1680c3154..c0bd8f1149a51 100644 --- a/drivers/mtd/spi-nor/debugfs.c +++ b/drivers/mtd/spi-nor/debugfs.c @@ -29,6 +29,7 @@ static const char *const snor_f_names[] = { SNOR_F_NAME(RWW), SNOR_F_NAME(ECC), SNOR_F_NAME(NO_WP), + SNOR_F_NAME(SWAP16), }; #undef SNOR_F_NAME diff --git a/drivers/mtd/spi-nor/spansion.c b/drivers/mtd/spi-nor/spansion.c index 8498c7003d888..b6023076903a3 100644 --- a/drivers/mtd/spi-nor/spansion.c +++ b/drivers/mtd/spi-nor/spansion.c @@ -674,7 +674,9 @@ static int s25hx_t_late_init(struct spi_nor *nor) params->ready = cypress_nor_sr_ready_and_clear; cypress_nor_ecc_init(nor); - params->die_erase_opcode = SPINOR_OP_CYPRESS_DIE_ERASE; + if (params->n_dice > 1) + params->die_erase_opcode = SPINOR_OP_CYPRESS_DIE_ERASE; + return 0; } diff --git a/drivers/mtd/spi-nor/swp.c b/drivers/mtd/spi-nor/swp.c index e67a81dbb6bf6..d5f4bf555cfc6 100644 --- a/drivers/mtd/spi-nor/swp.c +++ b/drivers/mtd/spi-nor/swp.c @@ -282,8 +282,15 @@ static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, u64 len) /* Prefer top, if both are valid */ use_top = can_be_top; - /* lock_len: length of region that should remain locked */ - if (use_top) + /* + * lock_len: length of region that should remain locked. + * + * When can_be_top and can_be_bottom booleans are true, both adjacent + * regions are unlocked, thus the entire flash can be unlocked. + */ + if (can_be_top && can_be_bottom) + lock_len = 0; + else if (use_top) lock_len = nor->params->size - (ofs + len); else lock_len = ofs; diff --git a/drivers/net/amt.c b/drivers/net/amt.c index f2f3139e38a59..0f4ff41d05352 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -2979,7 +2979,7 @@ static int amt_socket_create(struct amt_dev *amt) tunnel_cfg.encap_rcv = amt_rcv; tunnel_cfg.encap_err_lookup = amt_err_lookup; tunnel_cfg.encap_destroy = NULL; - setup_udp_tunnel_sock(amt->net, sock, &tunnel_cfg); + setup_udp_tunnel_sock(amt->net, sock->sk, &tunnel_cfg); rcu_assign_pointer(amt->sock, sock); return 0; @@ -3032,7 +3032,7 @@ static int amt_dev_stop(struct net_device *dev) RCU_INIT_POINTER(amt->sock, NULL); synchronize_net(); if (sock) - udp_tunnel_sock_release(sock); + udp_tunnel_sock_release(sock->sk); cancel_work_sync(&amt->event_wq); for (i = 0; i < AMT_MAX_EVENTS; i++) { @@ -3301,9 +3301,9 @@ static size_t amt_get_size(const struct net_device *dev) nla_total_size(sizeof(__u16)) + /* IFLA_AMT_GATEWAY_PORT */ nla_total_size(sizeof(__u32)) + /* IFLA_AMT_LINK */ nla_total_size(sizeof(__u32)) + /* IFLA_MAX_TUNNELS */ - nla_total_size(sizeof(struct iphdr)) + /* IFLA_AMT_DISCOVERY_IP */ - nla_total_size(sizeof(struct iphdr)) + /* IFLA_AMT_REMOTE_IP */ - nla_total_size(sizeof(struct iphdr)); /* IFLA_AMT_LOCAL_IP */ + nla_total_size(sizeof(__be32)) + /* IFLA_AMT_DISCOVERY_IP */ + nla_total_size(sizeof(__be32)) + /* IFLA_AMT_REMOTE_IP */ + nla_total_size(sizeof(__be32)); /* IFLA_AMT_LOCAL_IP */ } static int amt_fill_info(struct sk_buff *skb, const struct net_device *dev) diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c index da5866ba06999..169ab90393cc4 100644 --- a/drivers/net/bareudp.c +++ b/drivers/net/bareudp.c @@ -268,7 +268,7 @@ static int bareudp_socket_create(struct bareudp_dev *bareudp, __be16 port) tunnel_cfg.encap_rcv = bareudp_udp_encap_recv; tunnel_cfg.encap_err_lookup = bareudp_err_lookup; tunnel_cfg.encap_destroy = NULL; - setup_udp_tunnel_sock(bareudp->net, sock, &tunnel_cfg); + setup_udp_tunnel_sock(bareudp->net, sock->sk, &tunnel_cfg); rcu_assign_pointer(bareudp->sock, sock); return 0; @@ -290,7 +290,7 @@ static void bareudp_sock_release(struct bareudp_dev *bareudp) sock = bareudp->sock; rcu_assign_pointer(bareudp->sock, NULL); synchronize_net(); - udp_tunnel_sock_release(sock); + udp_tunnel_sock_release(sock->sk); } static int bareudp_stop(struct net_device *dev) diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c index 985ef66dc3331..acbba08dbdfad 100644 --- a/drivers/net/bonding/bond_3ad.c +++ b/drivers/net/bonding/bond_3ad.c @@ -745,6 +745,21 @@ static void __set_agg_ports_ready(struct aggregator *aggregator, int val) } } +static int __agg_usable_ports(struct aggregator *agg) +{ + struct port *port; + int valid = 0; + + for (port = agg->lag_ports; port; + port = port->next_port_in_aggregator) { + if (port->actor_oper_port_state & LACP_STATE_COLLECTING && + port->actor_oper_port_state & LACP_STATE_DISTRIBUTING) + valid++; + } + + return valid; +} + static int __agg_active_ports(struct aggregator *agg) { struct port *port; @@ -1179,10 +1194,10 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) switch (port->sm_mux_state) { case AD_MUX_DETACHED: port->actor_oper_port_state &= ~LACP_STATE_SYNCHRONIZATION; - ad_disable_collecting_distributing(port, - update_slave_arr); port->actor_oper_port_state &= ~LACP_STATE_COLLECTING; port->actor_oper_port_state &= ~LACP_STATE_DISTRIBUTING; + ad_disable_collecting_distributing(port, + update_slave_arr); port->ntt = true; break; case AD_MUX_WAITING: @@ -1322,6 +1337,7 @@ static void ad_rx_machine(struct lacpdu *lacpdu, struct port *port) fallthrough; case AD_RX_PORT_DISABLED: port->sm_vars &= ~AD_PORT_MATCHED; + port->partner_oper.port_state &= ~LACP_STATE_SYNCHRONIZATION; break; case AD_RX_LACP_DISABLED: port->sm_vars &= ~AD_PORT_SELECTED; @@ -2107,6 +2123,7 @@ static void ad_disable_distributing(struct port *port, bool *update_slave_arr) port->actor_port_number, aggregator->aggregator_identifier); __disable_distributing_port(port); + bond_3ad_set_carrier(port->slave->bond); /* Slave array needs an update */ *update_slave_arr = true; } @@ -2130,6 +2147,7 @@ static void ad_enable_collecting_distributing(struct port *port, port->actor_port_number, aggregator->aggregator_identifier); __enable_port(port); + bond_3ad_set_carrier(port->slave->bond); /* Slave array needs update */ *update_slave_arr = true; /* Should notify peers if possible */ @@ -2153,6 +2171,7 @@ static void ad_disable_collecting_distributing(struct port *port, port->actor_port_number, aggregator->aggregator_identifier); __disable_port(port); + bond_3ad_set_carrier(port->slave->bond); /* Slave array needs an update */ *update_slave_arr = true; } @@ -2832,7 +2851,9 @@ int bond_3ad_set_carrier(struct bonding *bond) active = __get_active_agg(&(SLAVE_AD_INFO(first_slave)->aggregator)); if (active) { /* are enough slaves available to consider link up? */ - if (__agg_active_ports(active) < bond->params.min_links) { + if ((bond->params.lacp_strict ? __agg_usable_ports(active) + : __agg_active_ports(active)) < + bond->params.min_links) { if (netif_carrier_ok(bond->dev)) { netif_carrier_off(bond->dev); goto out; diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 8e75453ce0efd..cd9b0a6d65217 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -6446,6 +6446,7 @@ static int __init bond_check_params(struct bond_params *params) params->ad_user_port_key = ad_user_port_key; params->coupled_control = 1; params->broadcast_neighbor = 0; + params->lacp_strict = 0; if (packets_per_slave > 0) { params->reciprocal_packets_per_slave = reciprocal_value(packets_per_slave); diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 90365d3f7ebff..4a11572f663d3 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -143,6 +143,7 @@ static const struct nla_policy bond_policy[IFLA_BOND_MAX + 1] = { [IFLA_BOND_NS_IP6_TARGET] = { .type = NLA_NESTED }, [IFLA_BOND_COUPLED_CONTROL] = { .type = NLA_U8 }, [IFLA_BOND_BROADCAST_NEIGH] = { .type = NLA_U8 }, + [IFLA_BOND_LACP_STRICT] = { .type = NLA_U8 }, }; static const struct nla_policy bond_slave_policy[IFLA_BOND_SLAVE_MAX + 1] = { @@ -599,6 +600,16 @@ static int bond_changelink(struct net_device *bond_dev, struct nlattr *tb[], return err; } + if (data[IFLA_BOND_LACP_STRICT]) { + int fallback_mode = nla_get_u8(data[IFLA_BOND_LACP_STRICT]); + + bond_opt_initval(&newval, fallback_mode); + err = __bond_opt_set(bond, BOND_OPT_LACP_STRICT, &newval, + data[IFLA_BOND_LACP_STRICT], extack); + if (err) + return err; + } + return 0; } @@ -671,6 +682,7 @@ static size_t bond_get_size(const struct net_device *bond_dev) nla_total_size(sizeof(struct in6_addr)) * BOND_MAX_NS_TARGETS + nla_total_size(sizeof(u8)) + /* IFLA_BOND_COUPLED_CONTROL */ nla_total_size(sizeof(u8)) + /* IFLA_BOND_BROADCAST_NEIGH */ + nla_total_size(sizeof(u8)) + /* IFLA_BOND_LACP_STRICT */ 0; } @@ -838,6 +850,10 @@ static int bond_fill_info(struct sk_buff *skb, bond->params.broadcast_neighbor)) goto nla_put_failure; + if (nla_put_u8(skb, IFLA_BOND_LACP_STRICT, + bond->params.lacp_strict)) + goto nla_put_failure; + if (BOND_MODE(bond) == BOND_MODE_8023AD) { struct ad_info info; diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index 7380cc4ee75a9..94b7b0851f16a 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -68,6 +68,8 @@ static int bond_option_lacp_active_set(struct bonding *bond, const struct bond_opt_value *newval); static int bond_option_lacp_rate_set(struct bonding *bond, const struct bond_opt_value *newval); +static int bond_option_lacp_strict_set(struct bonding *bond, + const struct bond_opt_value *newval); static int bond_option_ad_select_set(struct bonding *bond, const struct bond_opt_value *newval); static int bond_option_queue_id_set(struct bonding *bond, @@ -162,6 +164,12 @@ static const struct bond_opt_value bond_lacp_rate_tbl[] = { { NULL, -1, 0}, }; +static const struct bond_opt_value bond_lacp_strict_tbl[] = { + { "off", 0, BOND_VALFLAG_DEFAULT}, + { "on", 1, 0}, + { NULL, -1, 0 } +}; + static const struct bond_opt_value bond_ad_select_tbl[] = { { "stable", BOND_AD_STABLE, BOND_VALFLAG_DEFAULT}, { "bandwidth", BOND_AD_BANDWIDTH, 0}, @@ -363,6 +371,14 @@ static const struct bond_option bond_opts[BOND_OPT_LAST] = { .values = bond_lacp_rate_tbl, .set = bond_option_lacp_rate_set }, + [BOND_OPT_LACP_STRICT] = { + .id = BOND_OPT_LACP_STRICT, + .name = "lacp_strict", + .desc = "Define the LACP fallback mode when no slaves have negotiated", + .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)), + .values = bond_lacp_strict_tbl, + .set = bond_option_lacp_strict_set + }, [BOND_OPT_MINLINKS] = { .id = BOND_OPT_MINLINKS, .name = "min_links", @@ -1684,6 +1700,17 @@ static int bond_option_lacp_rate_set(struct bonding *bond, return 0; } +static int bond_option_lacp_strict_set(struct bonding *bond, + const struct bond_opt_value *newval) +{ + netdev_dbg(bond->dev, "Setting LACP fallback to %s (%llu)\n", + newval->string, newval->value); + bond->params.lacp_strict = newval->value; + bond_3ad_set_carrier(bond); + + return 0; +} + static int bond_option_ad_select_set(struct bonding *bond, const struct bond_opt_value *newval) { diff --git a/drivers/net/can/usb/esd_usb.c b/drivers/net/can/usb/esd_usb.c index d257440fa01ff..f41d4a0d140f7 100644 --- a/drivers/net/can/usb/esd_usb.c +++ b/drivers/net/can/usb/esd_usb.c @@ -1390,10 +1390,13 @@ static void esd_usb_disconnect(struct usb_interface *intf) netdev = dev->nets[i]->netdev; netdev_info(netdev, "unregister\n"); unregister_netdev(netdev); - free_candev(netdev); } } unlink_all_urbs(dev); + for (i = 0; i < dev->net_count; i++) { + if (dev->nets[i]) + free_candev(dev->nets[i]->netdev); + } kfree(dev); } } diff --git a/drivers/net/dsa/mxl862xx/mxl862xx-host.c b/drivers/net/dsa/mxl862xx/mxl862xx-host.c index d55f9dff6433e..4acd216f7cc00 100644 --- a/drivers/net/dsa/mxl862xx/mxl862xx-host.c +++ b/drivers/net/dsa/mxl862xx/mxl862xx-host.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "mxl862xx.h" #include "mxl862xx-host.h" @@ -40,12 +41,13 @@ static void mxl862xx_crc_err_work_fn(struct work_struct *work) crc_err_work); struct dsa_port *dp; - dev_warn(&priv->mdiodev->dev, - "MDIO CRC error detected, shutting down all ports\n"); - rtnl_lock(); - dsa_switch_for_each_cpu_port(dp, priv->ds) - dev_close(dp->conduit); + if (!test_bit(MXL862XX_FLAG_WORK_STOPPED, &priv->flags)) { + dev_warn(&priv->mdiodev->dev, + "MDIO CRC error detected, shutting down all ports\n"); + dsa_switch_for_each_cpu_port(dp, priv->ds) + dev_close(dp->conduit); + } rtnl_unlock(); clear_bit(MXL862XX_FLAG_CRC_ERR, &priv->flags); @@ -349,7 +351,7 @@ int mxl862xx_api_wrap(struct mxl862xx_priv *priv, u16 cmd, void *_data, * zero words individually. */ for (i = 0, zeros = 0; i < size / 2 && zeros < RST_DATA_THRESHOLD; i++) - if (!data[i]) + if (!get_unaligned_le16(&data[i])) zeros++; if (zeros < RST_DATA_THRESHOLD && (size & 1) && !*(u8 *)&data[i]) @@ -395,7 +397,7 @@ int mxl862xx_api_wrap(struct mxl862xx_priv *priv, u16 cmd, void *_data, */ val = *(u8 *)&data[i] | ((crc & 0xff) << 8); } else { - val = le16_to_cpu(data[i]); + val = get_unaligned_le16(&data[i]); } /* After RST_DATA, skip zero data words as the registers @@ -453,7 +455,7 @@ int mxl862xx_api_wrap(struct mxl862xx_priv *priv, u16 cmd, void *_data, *(uint8_t *)&data[i] = ret & 0xff; crc = (ret >> 8) & 0xff; } else { - data[i] = cpu_to_le16((u16)ret); + put_unaligned_le16((u16)ret, &data[i]); } } diff --git a/drivers/net/dsa/qca/qca8k-leds.c b/drivers/net/dsa/qca/qca8k-leds.c index 43ac68052baf9..ef496e345a4e7 100644 --- a/drivers/net/dsa/qca/qca8k-leds.c +++ b/drivers/net/dsa/qca/qca8k-leds.c @@ -429,7 +429,8 @@ qca8k_parse_port_leds(struct qca8k_priv *priv, struct fwnode_handle *port, int p init_data.fwnode = led; init_data.devname_mandatory = true; init_data.devicename = kasprintf(GFP_KERNEL, "%s:0%d", - priv->internal_mdio_bus->id, + priv->internal_mdio_bus ? + priv->internal_mdio_bus->id : priv->bus->id, port_num); if (!init_data.devicename) { fwnode_handle_put(led); diff --git a/drivers/net/dsa/realtek/rtl8366rb-leds.c b/drivers/net/dsa/realtek/rtl8366rb-leds.c index 509ffd3f8db5c..ba50d311cb151 100644 --- a/drivers/net/dsa/realtek/rtl8366rb-leds.c +++ b/drivers/net/dsa/realtek/rtl8366rb-leds.c @@ -89,6 +89,7 @@ static int rtl8366rb_setup_led(struct realtek_priv *priv, struct dsa_port *dp, struct led_init_data init_data = { }; enum led_default_state state; struct rtl8366rb_led *led; + char name[64]; u32 led_group; int ret; @@ -129,10 +130,9 @@ static int rtl8366rb_setup_led(struct realtek_priv *priv, struct dsa_port *dp, init_data.fwnode = led_fwnode; init_data.devname_mandatory = true; - init_data.devicename = kasprintf(GFP_KERNEL, "Realtek-%d:0%d:%d", - dp->ds->index, dp->index, led_group); - if (!init_data.devicename) - return -ENOMEM; + snprintf(name, sizeof(name), "Realtek-%d:0%d:%d", + dp->ds->index, dp->index, led_group); + init_data.devicename = name; ret = devm_led_classdev_register_ext(priv->dev, &led->cdev, &init_data); if (ret) { diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c index fefe46e2a5e61..350f958dcb2af 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.c +++ b/drivers/net/dsa/sja1105/sja1105_ptp.c @@ -755,7 +755,7 @@ static int sja1105_per_out_enable(struct sja1105_private *priv, * 2 edges on PTP_CLK. So check for truncation which happens * at periods larger than around 68.7 seconds. */ - pin_duration = ns_to_sja1105_ticks(pin_duration / 2); + pin_duration = max_t(u64, ns_to_sja1105_ticks(pin_duration / 2), 1); if (pin_duration > U32_MAX) { rc = -ERANGE; goto out; diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 31cdb11cd78da..1e361a35ebd49 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -311,7 +311,7 @@ static void airoha_fe_pse_ports_init(struct airoha_eth *eth) PSE_QUEUE_RSV_PAGES); /* PPE1 */ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) { - if (q < pse_port_num_queues[FE_PSE_PORT_PPE1]) + if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2) airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, PSE_QUEUE_RSV_PAGES); else @@ -1256,7 +1256,7 @@ static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma) FIELD_PREP(CNTR_CHAN_MASK, i)); /* Tx-fwd transferred count */ airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0); - airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1), + airoha_qdma_wr(qdma, REG_CNTR_CFG((i << 1) + 1), CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK | CNTR_ALL_DSCP_RING_EN_MASK | FIELD_PREP(CNTR_SRC_MASK, 1) | @@ -1933,7 +1933,7 @@ static u16 airoha_dev_select_queue(struct net_device *dev, struct sk_buff *skb, */ channel = netdev_uses_dsa(dev) ? skb_get_queue_mapping(skb) : port->id; channel = channel % AIROHA_NUM_QOS_CHANNELS; - queue = (skb->priority - 1) % AIROHA_NUM_QOS_QUEUES; /* QoS queue */ + queue = skb->priority % AIROHA_NUM_QOS_QUEUES; queue = channel * AIROHA_NUM_QOS_QUEUES + queue; return queue < dev->num_tx_queues ? queue : 0; diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index 5c9dff6bccd1e..a1f9939c21f20 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -1330,7 +1330,7 @@ static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe) { u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); struct airoha_foe_entry *hwe = ppe->foe; - int i, err = 0; + int i; for (i = 0; i < sram_num_entries; i++) { int err; @@ -1338,10 +1338,10 @@ static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe) memset(&hwe[i], 0, sizeof(*hwe)); err = airoha_ppe_foe_commit_sram_entry(ppe, i); if (err) - break; + return err; } - return err; + return 0; } static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth) @@ -1578,7 +1578,8 @@ int airoha_ppe_init(struct airoha_eth *eth) return -ENOMEM; } - ppe->foe_check_time = devm_kzalloc(eth->dev, ppe_num_entries, + ppe->foe_check_time = devm_kzalloc(eth->dev, + ppe_num_entries * sizeof(*ppe->foe_check_time), GFP_KERNEL); if (!ppe->foe_check_time) return -ENOMEM; diff --git a/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c b/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c index 0112c41150bb0..e46a98514486f 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c +++ b/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c @@ -121,8 +121,6 @@ static int airoha_ppe_debugfs_foe_show(struct seq_file *m, void *private, case PPE_PKT_TYPE_IPV4_DSLITE: src_port = &hwe->ipv4.new_tuple.src_port; dest_port = &hwe->ipv4.new_tuple.dest_port; - fallthrough; - case PPE_PKT_TYPE_IPV4_ROUTE: src_addr = &hwe->ipv4.new_tuple.src_ip; dest_addr = &hwe->ipv4.new_tuple.dest_ip; seq_puts(m, " new="); diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 92d149d4f0917..5d05020a6d058 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -752,6 +752,18 @@ static void ena_destroy_all_tx_queues(struct ena_adapter *adapter) } } +static void ena_destroy_xdp_tx_queues(struct ena_adapter *adapter) +{ + u16 ena_qid; + int i; + + for (i = adapter->xdp_first_ring; + i < adapter->xdp_first_ring + adapter->xdp_num_queues; i++) { + ena_qid = ENA_IO_TXQ_IDX(i); + ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); + } +} + static void ena_destroy_all_rx_queues(struct ena_adapter *adapter) { u16 ena_qid; @@ -2078,14 +2090,21 @@ static int create_queues_with_size_backoff(struct ena_adapter *adapter) rc = ena_setup_tx_resources_in_range(adapter, 0, adapter->num_io_queues); - if (rc) + if (rc) { + ena_destroy_xdp_tx_queues(adapter); + ena_free_all_io_tx_resources_in_range(adapter, + adapter->xdp_first_ring, + adapter->xdp_num_queues); goto err_setup_tx; + } rc = ena_create_io_tx_queues_in_range(adapter, 0, adapter->num_io_queues); - if (rc) + if (rc) { + ena_destroy_xdp_tx_queues(adapter); goto err_create_tx_queues; + } rc = ena_setup_all_rx_resources(adapter); if (rc) diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index 19e078479b0d4..5b2640bd31c3a 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -4748,6 +4748,7 @@ int bnx2x_alloc_mem_bp(struct bnx2x *bp) fp = kzalloc_objs(*fp, bp->fp_array_size); if (!fp) goto alloc_err; + bp->fp = fp; for (i = 0; i < bp->fp_array_size; i++) { fp[i].tpa_info = kzalloc_objs(struct bnx2x_agg_info, @@ -4756,8 +4757,6 @@ int bnx2x_alloc_mem_bp(struct bnx2x *bp) goto alloc_err; } - bp->fp = fp; - /* allocate sp objs */ bp->sp_objs = kzalloc_objs(struct bnx2x_sp_objs, bp->fp_array_size); if (!bp->sp_objs) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index c999f9733326a..d44d3ac255aa4 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -1011,6 +1011,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, mapping += bp->rx_dma_offset; rx_buf->data = page; rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; + rx_buf->offset = offset; } else { u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, gfp); @@ -1019,6 +1020,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, rx_buf->data = data; rx_buf->data_ptr = data + bp->rx_offset; + rx_buf->offset = 0; } rx_buf->mapping = mapping; @@ -1040,6 +1042,7 @@ void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; prod_rx_buf->mapping = cons_rx_buf->mapping; + prod_rx_buf->offset = cons_rx_buf->offset; prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)]; @@ -1175,8 +1178,11 @@ static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, struct page *page = data; u16 prod = rxr->rx_prod; struct sk_buff *skb; + void *frag_start; int err; + frag_start = page_address(page) + rxr->rx_buf_ring[cons].offset; + err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); if (unlikely(err)) { bnxt_reuse_rx_data(rxr, cons, data); @@ -1185,13 +1191,13 @@ static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, dma_addr -= bp->rx_dma_offset; dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, rxr->rx_page_size, bp->rx_dir); - skb = napi_build_skb(data_ptr - bp->rx_offset, rxr->rx_page_size); + skb = napi_build_skb(frag_start, rxr->rx_page_size); if (!skb) { page_pool_recycle_direct(rxr->page_pool, page); return NULL; } skb_mark_for_recycle(skb); - skb_reserve(skb, bp->rx_offset); + skb_reserve(skb, data_ptr - (u8 *)frag_start); __skb_put(skb, len); return skb; @@ -10524,7 +10530,7 @@ static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) stats->hw_masks, stats->len / 8, false); } -static void bnxt_accumulate_all_stats(struct bnxt *bp) +static void bnxt_accumulate_ring_stats(struct bnxt *bp) { struct bnxt_stats_mem *ring0_stats; bool ignore_zero = false; @@ -10547,6 +10553,10 @@ static void bnxt_accumulate_all_stats(struct bnxt *bp) ring0_stats->hw_masks, ring0_stats->len / 8, ignore_zero); } +} + +static void bnxt_accumulate_port_stats(struct bnxt *bp) +{ if (bp->flags & BNXT_FLAG_PORT_STATS) { struct bnxt_stats_mem *stats = &bp->port_stats; __le64 *hw_stats = stats->hw_stats; @@ -10569,6 +10579,41 @@ static void bnxt_accumulate_all_stats(struct bnxt *bp) } } +static void bnxt_accumulate_all_stats(struct bnxt *bp) +{ + bnxt_accumulate_ring_stats(bp); + bnxt_accumulate_port_stats(bp); +} + +/* Re-accumulate ring stats from DMA buffers if stale. + * uAPIs for reading sw_stats should call this first. + * + * We promise user space update frequency of bp->stats_coal_ticks but + * the update is a two step process - first device updates the DMA buffer, + * then we have to update from that buffer to driver stats in the service work. + * Worst case we would be 2x off from the desired frequency. + * Sync the stats sooner, if stale. The 20% threshold was chosen arbitrarily. + * + * Ideally we would split the user-configured time into two portions, + * i.e. also lower the DMA period by the 20%. But the DMA timer seems to have + * too coarse granularity to play such tricks. + */ +void bnxt_sync_ring_stats(struct bnxt *bp) +{ + unsigned long stale; + + if (!netif_running(bp->dev) || !bp->stats_coal_ticks) + return; + + spin_lock(&bp->stats_lock); + stale = usecs_to_jiffies(bp->stats_coal_ticks / 5); + if (time_after_eq(jiffies, bp->stats_updated_jiffies + stale)) { + bnxt_accumulate_ring_stats(bp); + bp->stats_updated_jiffies = jiffies; + } + spin_unlock(&bp->stats_lock); +} + static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) { struct hwrm_port_qstats_input *req; @@ -13567,6 +13612,7 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) return; } + bnxt_sync_ring_stats(bp); bnxt_get_ring_stats(bp, stats); bnxt_add_prev_stats(bp, stats); @@ -14748,7 +14794,10 @@ static void bnxt_sp_task(struct work_struct *work) if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { bnxt_hwrm_port_qstats(bp, 0); bnxt_hwrm_port_qstats_ext(bp, 0); + spin_lock(&bp->stats_lock); bnxt_accumulate_all_stats(bp); + bp->stats_updated_jiffies = jiffies; + spin_unlock(&bp->stats_lock); } if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { @@ -15490,6 +15539,7 @@ static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); spin_lock_init(&bp->ntp_fltr_lock); + spin_lock_init(&bp->stats_lock); #if BITS_PER_LONG == 32 spin_lock_init(&bp->db_lock); #endif @@ -16058,6 +16108,7 @@ static void bnxt_get_queue_stats_rx(struct net_device *dev, int i, if (!bp->bnapi) return; + bnxt_sync_ring_stats(bp); cpr = &bp->bnapi[i]->cp_ring; sw = cpr->stats.sw_stats; @@ -16086,6 +16137,7 @@ static void bnxt_get_queue_stats_tx(struct net_device *dev, int i, if (!bp->tx_ring) return; + bnxt_sync_ring_stats(bp); bnapi = bp->tx_ring[bp->tx_ring_map[i]].bnapi; sw = bnapi->cp_ring.stats.sw_stats; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 61c847b36b9ff..fa13cee557798 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -911,6 +911,7 @@ struct bnxt_sw_rx_bd { void *data; u8 *data_ptr; dma_addr_t mapping; + unsigned int offset; }; struct bnxt_sw_rx_agg_bd { @@ -2620,6 +2621,10 @@ struct bnxt { #define BNXT_MIN_STATS_COAL_TICKS 250000 #define BNXT_MAX_STATS_COAL_TICKS 1000000 + /* Protects stats_updated_jiffies and writes to sw_stats */ + spinlock_t stats_lock; + unsigned long stats_updated_jiffies; + struct work_struct sp_task; unsigned long sp_event; #define BNXT_RX_MASK_SP_EVENT 0 @@ -3028,6 +3033,7 @@ void bnxt_reenable_sriov(struct bnxt *bp); void bnxt_close_nic(struct bnxt *, bool, bool); void bnxt_get_ring_drv_stats(struct bnxt *bp, struct bnxt_total_ring_drv_stats *stats); +void bnxt_sync_ring_stats(struct bnxt *bp); bool bnxt_rfs_capable(struct bnxt *bp, bool new_rss_ctx); int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, u32 *reg_buf); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 9b14134d62d25..b98761d55a2b0 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -606,6 +606,7 @@ static void bnxt_get_ethtool_stats(struct net_device *dev, goto skip_ring_stats; } + bnxt_sync_ring_stats(bp); tpa_stats = bnxt_get_num_tpa_ring_stats(bp); for (i = 0; i < bp->cp_nr_rings; i++) { struct bnxt_napi *bnapi = bp->bnapi[i]; diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 7c11cf9167620..ad08c67269be5 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -40,9 +40,8 @@ #include "bcmgenet.h" -/* Default highest priority queue for multi queue support */ -#define GENET_Q1_PRIORITY 0 -#define GENET_Q0_PRIORITY 1 +#define GENET_Q0_WEIGHT 1 +#define GENET_Q1_WEIGHT 4 #define GENET_Q0_RX_BD_CNT \ (TOTAL_DESC - priv->hw_params->rx_queues * priv->hw_params->rx_bds_per_q) @@ -2129,13 +2128,6 @@ static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev) int i; index = skb_get_queue_mapping(skb); - /* Mapping strategy: - * queue_mapping = 0, unclassified, packet xmited through ring 0 - * queue_mapping = 1, goes to ring 1. (highest priority queue) - * queue_mapping = 2, goes to ring 2. - * queue_mapping = 3, goes to ring 3. - * queue_mapping = 4, goes to ring 4. - */ ring = &priv->tx_rings[index]; txq = netdev_get_tx_queue(dev, index); @@ -2881,8 +2873,9 @@ static int bcmgenet_rdma_disable(struct bcmgenet_priv *priv) /* Initialize Tx queues * - * Queues 1-4 are priority-based, each one has 32 descriptors, - * with queue 1 being the highest priority queue. + * Queues 1-4 are the priority queues, each one has 32 descriptors. + * The weighted round-robin arbiter gives them a larger share of TX + * bandwidth than the default queue 0. * * Queue 0 is the default Tx queue with * GENET_Q0_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors. @@ -2900,8 +2893,8 @@ static void bcmgenet_init_tx_queues(struct net_device *dev) unsigned int start = 0, end = GENET_Q0_TX_BD_CNT; u32 i, ring_mask, dma_priority[3] = {0, 0, 0}; - /* Enable strict priority arbiter mode */ - bcmgenet_tdma_writel(priv, DMA_ARBITER_SP, DMA_ARB_CTRL); + /* Enable Weighted Round-Robin arbiter mode */ + bcmgenet_tdma_writel(priv, DMA_ARBITER_WRR, DMA_ARB_CTRL); /* Initialize Tx priority queues */ for (i = 0; i <= priv->hw_params->tx_queues; i++) { @@ -2909,7 +2902,7 @@ static void bcmgenet_init_tx_queues(struct net_device *dev) start = end; end += priv->hw_params->tx_bds_per_q; dma_priority[DMA_PRIO_REG_INDEX(i)] |= - (i ? GENET_Q1_PRIORITY : GENET_Q0_PRIORITY) + (i ? GENET_Q1_WEIGHT : GENET_Q0_WEIGHT) << DMA_PRIO_REG_SHIFT(i); } diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index a12aa21244e83..d394f1f43b685 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -2668,8 +2668,25 @@ static void macb_free_consistent(struct macb *bp) dma_free_coherent(dev, size, bp->queues[0].rx_ring, bp->queues[0].rx_ring_dma); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - kfree(queue->tx_skb); - queue->tx_skb = NULL; + if (queue->tx_skb) { + unsigned int dropped = 0, tail; + + for (tail = queue->tx_tail; tail != queue->tx_head; + tail++) { + if (macb_tx_skb(queue, tail)->skb) + dropped++; + macb_tx_unmap(bp, macb_tx_skb(queue, tail), 0); + } + + queue->stats.tx_dropped += dropped; + bp->dev->stats.tx_dropped += dropped; + + kfree(queue->tx_skb); + queue->tx_skb = NULL; + } + + queue->tx_head = 0; + queue->tx_tail = 0; queue->tx_ring = NULL; queue->rx_ring = NULL; } @@ -4522,6 +4539,13 @@ static int macb_setup_tc(struct net_device *dev, enum tc_setup_type type, } } +static void macb_tx_timeout(struct net_device *dev, unsigned int q) +{ + struct macb *bp = netdev_priv(dev); + + macb_tx_restart(&bp->queues[q]); +} + static const struct net_device_ops macb_netdev_ops = { .ndo_open = macb_open, .ndo_stop = macb_close, @@ -4540,6 +4564,7 @@ static const struct net_device_ops macb_netdev_ops = { .ndo_hwtstamp_set = macb_hwtstamp_set, .ndo_hwtstamp_get = macb_hwtstamp_get, .ndo_setup_tc = macb_setup_tc, + .ndo_tx_timeout = macb_tx_timeout, }; /* Configure peripheral capabilities according to device tree diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c index 75f22f74774cb..06b4424e778ed 100644 --- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c +++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c @@ -1163,18 +1163,14 @@ int setup_cn23xx_octeon_pf_device(struct octeon_device *oct) if (octeon_map_pci_barx(oct, 1, MAX_BAR1_IOREMAP_SIZE)) { dev_err(&oct->pci_dev->dev, "%s CN23XX BAR1 map failed\n", __func__); - octeon_unmap_pci_barx(oct, 0); - return 1; + goto err_unmap_bar0; } if (cn23xx_get_pf_num(oct) != 0) - return 1; + goto err_unmap_bar1; - if (cn23xx_sriov_config(oct)) { - octeon_unmap_pci_barx(oct, 0); - octeon_unmap_pci_barx(oct, 1); - return 1; - } + if (cn23xx_sriov_config(oct)) + goto err_unmap_bar1; octeon_write_csr64(oct, CN23XX_SLI_MAC_CREDIT_CNT, 0x3F802080802080ULL); @@ -1205,6 +1201,12 @@ int setup_cn23xx_octeon_pf_device(struct octeon_device *oct) oct->coproc_clock_rate = 1000000ULL * cn23xx_coprocessor_clock(oct); return 0; + +err_unmap_bar1: + octeon_unmap_pci_barx(oct, 1); +err_unmap_bar0: + octeon_unmap_pci_barx(oct, 0); + return 1; } EXPORT_SYMBOL_GPL(setup_cn23xx_octeon_pf_device); diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index c1a3df2252549..32dd9b25760e3 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -3781,9 +3781,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) static int octeon_enable_sriov(struct octeon_device *oct) { unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced; - struct pci_dev *vfdev; int err; - u32 u; if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) { err = pci_enable_sriov(oct->pci_dev, @@ -3796,23 +3794,6 @@ static int octeon_enable_sriov(struct octeon_device *oct) return err; } oct->sriov_info.sriov_enabled = 1; - - /* init lookup table that maps DPI ring number to VF pci_dev - * struct pointer - */ - u = 0; - vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, - OCTEON_CN23XX_VF_VID, NULL); - while (vfdev) { - if (vfdev->is_virtfn && - (vfdev->physfn == oct->pci_dev)) { - oct->sriov_info.dpiring_to_vfpcidev_lut[u] = - vfdev; - u += oct->sriov_info.rings_per_vf; - } - vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, - OCTEON_CN23XX_VF_VID, vfdev); - } } return num_vfs_alloced; @@ -3820,8 +3801,6 @@ static int octeon_enable_sriov(struct octeon_device *oct) static int lio_pci_sriov_disable(struct octeon_device *oct) { - int u; - if (pci_vfs_assigned(oct->pci_dev)) { dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n"); return -EPERM; @@ -3829,12 +3808,6 @@ static int lio_pci_sriov_disable(struct octeon_device *oct) pci_disable_sriov(oct->pci_dev); - u = 0; - while (u < MAX_POSSIBLE_VFS) { - oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL; - u += oct->sriov_info.rings_per_vf; - } - oct->sriov_info.num_vfs_alloced = 0; dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n", oct->pf_num); diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.h b/drivers/net/ethernet/cavium/liquidio/octeon_device.h index 19344b21f8fb9..858a0fff2cc0b 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_device.h +++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.h @@ -390,9 +390,6 @@ struct octeon_sriov_info { struct lio_trusted_vf trusted_vf; - /*lookup table that maps DPI ring number to VF pci_dev struct pointer*/ - struct pci_dev *dpiring_to_vfpcidev_lut[MAX_POSSIBLE_VFS]; - u64 vf_macaddr[MAX_POSSIBLE_VFS]; u16 vf_vlantci[MAX_POSSIBLE_VFS]; diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c b/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c index ad685f5d0a136..697fcdc41e3cd 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +++ b/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c @@ -26,6 +26,31 @@ #include "octeon_mailbox.h" #include "cn23xx_pf_device.h" +static struct pci_dev *lio_vf_pci_dev_by_qno(struct octeon_device *oct, u32 q_no) +{ + struct pci_dev *vfdev = NULL; + int vfidx; + + if (!oct->sriov_info.rings_per_vf) + return NULL; + + if (q_no % oct->sriov_info.rings_per_vf) + return NULL; + + vfidx = q_no / oct->sriov_info.rings_per_vf; + if (vfidx >= oct->sriov_info.num_vfs_alloced) + return NULL; + + while ((vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, + OCTEON_CN23XX_VF_VID, vfdev))) { + if (pci_physfn(vfdev) && pci_physfn(vfdev) == oct->pci_dev && + pci_iov_vf_id(vfdev) == vfidx) + return vfdev; + } + + return NULL; +} + /** * octeon_mbox_read: * @mbox: Pointer mailbox @@ -237,6 +262,7 @@ static int octeon_mbox_process_cmd(struct octeon_mbox *mbox, struct octeon_mbox_cmd *mbox_cmd) { struct octeon_device *oct = mbox->oct_dev; + struct pci_dev *vfdev; switch (mbox_cmd->msg.s.cmd) { case OCTEON_VF_ACTIVE: @@ -260,7 +286,12 @@ static int octeon_mbox_process_cmd(struct octeon_mbox *mbox, dev_info(&oct->pci_dev->dev, "got a request for FLR from VF that owns DPI ring %u\n", mbox->q_no); - pcie_flr(oct->sriov_info.dpiring_to_vfpcidev_lut[mbox->q_no]); + vfdev = lio_vf_pci_dev_by_qno(oct, mbox->q_no); + if (!vfdev) + break; + + pcie_flr(vfdev); + pci_dev_put(vfdev); break; case OCTEON_PF_CHANGED_VF_MACADDR: diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c index 171750fad44f2..6871127427fad 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c @@ -6737,14 +6737,6 @@ void t4_sge_decode_idma_state(struct adapter *adapter, int state) return; } - if (is_t4(adapter->params.chip)) { - sge_idma_decode = (const char **)t4_decode; - sge_idma_decode_nstates = ARRAY_SIZE(t4_decode); - } else { - sge_idma_decode = (const char **)t5_decode; - sge_idma_decode_nstates = ARRAY_SIZE(t5_decode); - } - if (state < sge_idma_decode_nstates) CH_WARN(adapter, "idma state %s\n", sge_idma_decode[state]); else diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index 52c1cb9cb7e02..fa4e4f4797823 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -2177,7 +2177,7 @@ dpaa2_switch_prechangeupper_sanity_checks(struct net_device *netdev, if (err) { NL_SET_ERR_MSG_MOD(extack, "Cannot join a bridge while VLAN uppers are present"); - return 0; + return err; } netdev_for_each_lower_dev(upper_dev, other_dev, iter) { @@ -2198,6 +2198,7 @@ dpaa2_switch_prechangeupper_sanity_checks(struct net_device *netdev, static int dpaa2_switch_port_prechangeupper(struct net_device *netdev, struct netdev_notifier_changeupper_info *info) { + struct ethsw_port_priv *port_priv; struct netlink_ext_ack *extack; struct net_device *upper_dev; int err; @@ -2216,6 +2217,13 @@ static int dpaa2_switch_port_prechangeupper(struct net_device *netdev, if (!info->linking) dpaa2_switch_port_pre_bridge_leave(netdev); + } else if (is_vlan_dev(upper_dev)) { + port_priv = netdev_priv(netdev); + if (port_priv->fdb->bridge_dev) { + NL_SET_ERR_MSG_MOD(extack, + "Cannot accept VLAN uppers while bridged"); + return -EOPNOTSUPP; + } } return 0; diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index aa8a87124b103..8e3f345dd9aac 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -1783,6 +1783,7 @@ int enetc_xdp_xmit(struct net_device *ndev, int num_frames, { struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct skb_shared_info *shinfo; struct enetc_bdr *tx_ring; int xdp_tx_bd_cnt, i, k; int xdp_tx_frm_cnt = 0; @@ -1798,6 +1799,12 @@ int enetc_xdp_xmit(struct net_device *ndev, int num_frames, prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); for (k = 0; k < num_frames; k++) { + if (xdp_frame_has_frags(frames[k])) { + shinfo = xdp_get_shared_info_from_frame(frames[k]); + if (unlikely((shinfo->nr_frags + 1) > ENETC_MAX_SKB_FRAGS)) + break; + } + xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, xdp_redirect_arr, frames[k]); diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c index 56899f2254aaa..71866d240b40c 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c @@ -326,6 +326,9 @@ static void enetc4_default_rings_allocation(struct enetc_pf *pf) val = enetc4_psicfgr0_val_construct(false, num_tx_bdr, num_rx_bdr); enetc_port_wr(hw, ENETC4_PSICFGR0(0), val); + if (!pf->caps.num_vsi) + return; + num_rx_bdr = pf->caps.num_rx_bdr - num_rx_bdr; rx_rem = num_rx_bdr % pf->caps.num_vsi; num_rx_bdr = num_rx_bdr / pf->caps.num_vsi; diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c index 013273a2de32f..299bab043175b 100644 --- a/drivers/net/ethernet/freescale/fman/fman.c +++ b/drivers/net/ethernet/freescale/fman/fman.c @@ -1995,8 +1995,10 @@ static int fman_init(struct fman *fman) /* Init KeyGen */ fman->keygen = keygen_init(fman->kg_regs); - if (!fman->keygen) + if (!fman->keygen) { + free_init_resources(fman); return -EINVAL; + } err = enable(fman, cfg); if (err != 0) diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c index 3271de5844f84..89215e1ddc2db 100644 --- a/drivers/net/ethernet/freescale/gianfar.c +++ b/drivers/net/ethernet/freescale/gianfar.c @@ -469,10 +469,13 @@ static void free_gfar_dev(struct gfar_private *priv) { int i, j; - for (i = 0; i < priv->num_grps; i++) + for (i = 0; i < MAXGROUPS; i++) for (j = 0; j < GFAR_NUM_IRQS; j++) { - kfree(priv->gfargrp[i].irqinfo[j]); - priv->gfargrp[i].irqinfo[j] = NULL; + if (priv->gfargrp[i].irqinfo[j]) { + irq_dispose_mapping(priv->gfargrp[i].irqinfo[j]->irq); + kfree(priv->gfargrp[i].irqinfo[j]); + priv->gfargrp[i].irqinfo[j] = NULL; + } } free_netdev(priv->ndev); @@ -616,7 +619,7 @@ static phy_interface_t gfar_get_interface(struct net_device *dev) static int gfar_of_init(struct platform_device *ofdev, struct net_device **pdev) { const char *model; - int err = 0, i; + int err = 0, i, j; phy_interface_t interface; struct net_device *dev = NULL; struct gfar_private *priv = NULL; @@ -702,8 +705,11 @@ static int gfar_of_init(struct platform_device *ofdev, struct net_device **pdev) priv->rx_list.count = 0; mutex_init(&priv->rx_queue_access); - for (i = 0; i < MAXGROUPS; i++) + for (i = 0; i < MAXGROUPS; i++) { priv->gfargrp[i].regs = NULL; + for (j = 0; j < GFAR_NUM_IRQS; j++) + priv->gfargrp[i].irqinfo[j] = NULL; + } /* Parse and initialize group specific information */ if (priv->mode == MQ_MG_MODE) { diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c index 7924dce719e29..02cba280d81a8 100644 --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -21,11 +21,13 @@ static void gve_rx_free_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx) { struct device *hdev = &priv->pdev->dev; - int buf_count = rx->dqo.bufq.mask + 1; if (rx->dqo.hdr_bufs.data) { - dma_free_coherent(hdev, priv->header_buf_size * buf_count, - rx->dqo.hdr_bufs.data, rx->dqo.hdr_bufs.addr); + size_t size = + (size_t)priv->header_buf_size * rx->dqo.num_buf_states; + + dma_free_coherent(hdev, size, rx->dqo.hdr_bufs.data, + rx->dqo.hdr_bufs.addr); rx->dqo.hdr_bufs.data = NULL; } } @@ -254,7 +256,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv, /* Allocate header buffers for header-split */ if (cfg->enable_header_split) - if (gve_rx_alloc_hdr_bufs(priv, rx, buffer_queue_slots)) + if (gve_rx_alloc_hdr_bufs(priv, rx, rx->dqo.num_buf_states)) goto err; /* Allocate RX completion queue */ @@ -381,10 +383,13 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx) break; } - if (rx->dqo.hdr_bufs.data) + if (rx->dqo.hdr_bufs.data) { + u16 buf_id = le16_to_cpu(desc->buf_id); + desc->header_buf_addr = cpu_to_le64(rx->dqo.hdr_bufs.addr + - priv->header_buf_size * bufq->tail); + (size_t)priv->header_buf_size * buf_id); + } bufq->tail = (bufq->tail + 1) & bufq->mask; complq->num_free_slots--; @@ -826,10 +831,13 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, int unsplit = 0; if (hdr_len && !hbo) { - rx->ctx.skb_head = gve_rx_copy_data(priv->dev, napi, - rx->dqo.hdr_bufs.data + - desc_idx * priv->header_buf_size, - hdr_len); + size_t offset = + (size_t)buffer_id * priv->header_buf_size; + + rx->ctx.skb_head = + gve_rx_copy_data(priv->dev, napi, + rx->dqo.hdr_bufs.data + offset, + hdr_len); if (unlikely(!rx->ctx.skb_head)) goto error; rx->ctx.skb_tail = rx->ctx.skb_head; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index 9cb7ce9fd311c..442f15476af32 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -811,12 +811,11 @@ static int hns3_get_link_ksettings(struct net_device *netdev, } static int hns3_check_ksettings_param(const struct net_device *netdev, - const struct ethtool_link_ksettings *cmd) + const struct ethtool_link_ksettings *cmd, + u8 media_type) { struct hnae3_handle *handle = hns3_get_handle(netdev); const struct hnae3_ae_ops *ops = hns3_get_ops(handle); - u8 module_type = HNAE3_MODULE_TYPE_UNKNOWN; - u8 media_type = HNAE3_MEDIA_TYPE_UNKNOWN; u32 lane_num; u8 autoneg; u32 speed; @@ -836,9 +835,6 @@ static int hns3_check_ksettings_param(const struct net_device *netdev, return 0; } - if (ops->get_media_type) - ops->get_media_type(handle, &media_type, &module_type); - if (cmd->base.duplex == DUPLEX_HALF && media_type != HNAE3_MEDIA_TYPE_COPPER) { netdev_err(netdev, @@ -863,6 +859,8 @@ static int hns3_set_link_ksettings(struct net_device *netdev, struct hnae3_handle *handle = hns3_get_handle(netdev); struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(handle); const struct hnae3_ae_ops *ops = hns3_get_ops(handle); + u8 module_type = HNAE3_MODULE_TYPE_UNKNOWN; + u8 media_type = HNAE3_MEDIA_TYPE_UNKNOWN; int ret; /* Chip don't support this mode. */ @@ -878,22 +876,23 @@ static int hns3_set_link_ksettings(struct net_device *netdev, cmd->base.autoneg, cmd->base.speed, cmd->base.duplex, cmd->lanes); - /* Only support ksettings_set for netdev with phy attached for now */ - if (netdev->phydev) { - if (cmd->base.speed == SPEED_1000 && - cmd->base.autoneg == AUTONEG_DISABLE) - return -EINVAL; + if (!ops->get_media_type) + return -EOPNOTSUPP; + ops->get_media_type(handle, &media_type, &module_type); - return phy_ethtool_ksettings_set(netdev->phydev, cmd); - } else if (test_bit(HNAE3_DEV_SUPPORT_PHY_IMP_B, ae_dev->caps) && - ops->set_phy_link_ksettings) { - return ops->set_phy_link_ksettings(handle, cmd); + if (media_type == HNAE3_MEDIA_TYPE_COPPER) { + if (!ops->set_phy_link_ksettings) + return -EOPNOTSUPP; + ret = ops->set_phy_link_ksettings(handle, cmd); + if (ret != -ENODEV) + return ret; + /* PHY_INEXISTENT, use MAC-level configuration */ } if (ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2) return -EOPNOTSUPP; - ret = hns3_check_ksettings_param(netdev, cmd); + ret = hns3_check_ksettings_param(netdev, cmd, media_type); if (ret) return ret; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index dd4045c773d43..2a6e4d17452c4 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -1577,6 +1577,11 @@ static int hclge_configure(struct hclge_dev *hdev) hdev->hw.mac.req_autoneg = AUTONEG_ENABLE; hdev->hw.mac.req_duplex = DUPLEX_FULL; + /* When lane_num is 0, the firmware will automatically + * select the appropriate lane_num based on the speed. + */ + hdev->hw.mac.req_lane_num = 0; + hclge_parse_link_mode(hdev, cfg.speed_ability); hdev->hw.mac.max_speed = hclge_get_max_speed(cfg.speed_ability); @@ -2652,8 +2657,11 @@ static int hclge_cfg_mac_speed_dup_h(struct hnae3_handle *handle, int speed, if (ret) return ret; - hdev->hw.mac.req_speed = (u32)speed; - hdev->hw.mac.req_duplex = duplex; + hdev->hw.mac.req_lane_num = lane_num; + if (speed != SPEED_UNKNOWN) + hdev->hw.mac.req_speed = (u32)speed; + if (duplex != DUPLEX_UNKNOWN) + hdev->hw.mac.req_duplex = duplex; return 0; } @@ -2684,6 +2692,7 @@ static int hclge_set_autoneg(struct hnae3_handle *handle, bool enable) { struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; + int ret; if (!hdev->hw.mac.support_autoneg) { if (enable) { @@ -2695,7 +2704,10 @@ static int hclge_set_autoneg(struct hnae3_handle *handle, bool enable) } } - return hclge_set_autoneg_en(hdev, enable); + ret = hclge_set_autoneg_en(hdev, enable); + if (!ret) + hdev->hw.mac.req_autoneg = enable; + return ret; } static int hclge_get_autoneg(struct hnae3_handle *handle) @@ -2957,20 +2969,6 @@ static int hclge_mac_init(struct hclge_dev *hdev) if (!test_bit(HCLGE_STATE_RST_HANDLING, &hdev->state)) hdev->hw.mac.duplex = HCLGE_MAC_FULL; - if (hdev->hw.mac.support_autoneg) { - ret = hclge_set_autoneg_en(hdev, hdev->hw.mac.autoneg); - if (ret) - return ret; - } - - if (!hdev->hw.mac.autoneg) { - ret = hclge_cfg_mac_speed_dup_hw(hdev, hdev->hw.mac.req_speed, - hdev->hw.mac.req_duplex, - hdev->hw.mac.lane_num); - if (ret) - return ret; - } - mac->link = 0; if (mac->user_fec_mode & BIT(HNAE3_FEC_USER_DEF)) { @@ -3358,8 +3356,8 @@ static int hclge_get_phy_link_ksettings(struct hnae3_handle *handle, } static int -hclge_set_phy_link_ksettings(struct hnae3_handle *handle, - const struct ethtool_link_ksettings *cmd) +hclge_ethtool_ksettings_set(struct hnae3_handle *handle, + const struct ethtool_link_ksettings *cmd) { struct hclge_desc desc[HCLGE_PHY_LINK_SETTING_BD_NUM]; struct hclge_vport *vport = hclge_get_vport(handle); @@ -3400,10 +3398,34 @@ hclge_set_phy_link_ksettings(struct hnae3_handle *handle, return ret; } - hdev->hw.mac.req_autoneg = cmd->base.autoneg; - hdev->hw.mac.req_speed = cmd->base.speed; - hdev->hw.mac.req_duplex = cmd->base.duplex; linkmode_copy(hdev->hw.mac.advertising, cmd->link_modes.advertising); + return 0; +} + +static int +hclge_set_phy_link_ksettings(struct hnae3_handle *handle, + const struct ethtool_link_ksettings *cmd) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + int ret = -ENODEV; + + if (hnae3_dev_phy_imp_supported(hdev)) { + ret = hclge_ethtool_ksettings_set(handle, cmd); + } else if (handle->netdev->phydev) { + if (cmd->base.speed == SPEED_1000 && + cmd->base.autoneg == AUTONEG_DISABLE) + return -EINVAL; + ret = phy_ethtool_ksettings_set(handle->netdev->phydev, cmd); + } + if (ret) + return ret; + + hdev->hw.mac.req_autoneg = cmd->base.autoneg; + if (cmd->base.speed != SPEED_UNKNOWN) + hdev->hw.mac.req_speed = cmd->base.speed; + if (cmd->base.duplex != DUPLEX_UNKNOWN) + hdev->hw.mac.req_duplex = cmd->base.duplex; return 0; } @@ -11722,6 +11744,27 @@ static int hclge_set_wol(struct hnae3_handle *handle, return ret; } +static int hclge_set_autoneg_speed_dup(struct hclge_dev *hdev) +{ + int ret; + + if (hdev->hw.mac.support_autoneg) { + ret = hclge_set_autoneg_en(hdev, hdev->hw.mac.req_autoneg); + if (ret) + return ret; + } + + if (!hdev->hw.mac.req_autoneg) { + ret = hclge_cfg_mac_speed_dup_hw(hdev, hdev->hw.mac.req_speed, + hdev->hw.mac.req_duplex, + hdev->hw.mac.req_lane_num); + if (ret) + return ret; + } + + return 0; +} + static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) { struct pci_dev *pdev = ae_dev->pdev; @@ -11883,6 +11926,20 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) if (ret) goto err_ptp_uninit; + if (hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_COPPER) { + hdev->hw.mac.req_autoneg = hdev->hw.mac.autoneg; + if (hdev->hw.mac.autoneg == AUTONEG_DISABLE && + hdev->hw.mac.speed != SPEED_UNKNOWN) + hdev->hw.mac.req_speed = hdev->hw.mac.speed; + } + + ret = hclge_set_autoneg_speed_dup(hdev); + if (ret) { + dev_err(&pdev->dev, + "failed to set autoneg speed duplex, ret = %d\n", ret); + goto err_ptp_uninit; + } + INIT_KFIFO(hdev->mac_tnl_log); hclge_dcb_ops_set(hdev); @@ -12213,6 +12270,13 @@ static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev) return ret; } + ret = hclge_set_autoneg_speed_dup(hdev); + if (ret) { + dev_err(&pdev->dev, + "failed to set autoneg speed duplex, ret = %d\n", ret); + return ret; + } + ret = hclge_tp_port_init(hdev); if (ret) { dev_err(&pdev->dev, "failed to init tp port, ret = %d\n", diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index 032b472d23685..4ca6458625a9b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -287,6 +287,7 @@ struct hclge_mac { u8 support_autoneg; u8 speed_type; /* 0: sfp speed, 1: active speed */ u8 lane_num; + u8 req_lane_num; u32 speed; u32 req_speed; u32 max_speed; diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index 4e503b3d0d2d3..e1047915e101e 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -3035,6 +3035,12 @@ static int emac_probe(struct platform_device *ofdev) if (err) goto err_gone; + dev->emacp = devm_platform_ioremap_resource(ofdev, 0); + if (IS_ERR(dev->emacp)) { + err = PTR_ERR(dev->emacp); + goto err_gone; + } + /* Setup error IRQ handler */ dev->emac_irq = platform_get_irq(ofdev, 0); err = devm_request_irq(&ofdev->dev, dev->emac_irq, emac_irq, 0, "EMAC", @@ -3047,13 +3053,6 @@ static int emac_probe(struct platform_device *ofdev) ndev->irq = dev->emac_irq; - dev->emacp = devm_platform_ioremap_resource(ofdev, 0); - if (IS_ERR(dev->emacp)) { - dev_err(&ofdev->dev, "can't map device registers"); - err = PTR_ERR(dev->emacp); - goto err_gone; - } - /* Wait for dependent devices */ err = emac_wait_deps(dev); if (err) diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index dea208db1be5b..aa90e0ce8acaf 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -1594,6 +1594,9 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) phy_reg &= ~I217_PLL_CLOCK_GATE_MASK; if (speed == SPEED_100 || speed == SPEED_10) phy_reg |= 0x3E8; + else if (hw->mac.type == e1000_pch_mtp || + hw->mac.type == e1000_pch_ptp) + phy_reg |= 0x1D5; else phy_reg |= 0xFA; e1e_wphy_locked(hw, I217_PLL_CLOCK_GATE_REG, phy_reg); diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 7ce0cc8ab8f4c..7612ddd066fe5 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "e1000.h" #define CREATE_TRACE_POINTS @@ -58,6 +59,17 @@ static const struct e1000_info *e1000_info_tbl[] = { [board_pch_ptp] = &e1000_pch_ptp_info, }; +static const struct dmi_system_id disable_k1_list[] = { + { + .ident = "Dell Pro 16 Plus PB16250", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell Pro 16 Plus PB16250"), + }, + }, + {} +}; + struct e1000_reg_info { u32 ofs; char *name; @@ -7670,7 +7682,8 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* init PTP hardware clock */ e1000e_ptp_init(adapter); - if (hw->mac.type >= e1000_pch_mtp) + /* disable K1 by default on known problematic systems */ + if (hw->mac.type >= e1000_pch_mtp && dmi_check_system(disable_k1_list)) adapter->flags2 |= FLAG2_DISABLE_K1; /* reset the hardware with the new settings */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_debug.h b/drivers/net/ethernet/intel/i40e/i40e_debug.h index e9871dfb32bd4..01fd70db90866 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_debug.h +++ b/drivers/net/ethernet/intel/i40e/i40e_debug.h @@ -42,7 +42,7 @@ struct device *i40e_hw_to_dev(struct i40e_hw *hw); #define i40e_debug(h, m, s, ...) \ do { \ if (((m) & (h)->debug_mask)) \ - dev_info(i40e_hw_to_dev(hw), s, ##__VA_ARGS__); \ + dev_info(i40e_hw_to_dev(h), s, ##__VA_ARGS__); \ } while (0) #endif /* _I40E_DEBUG_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index b617a6bff8913..98960d3cbdd1d 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -3882,7 +3882,6 @@ ice_set_fc(struct ice_port_info *pi, u8 *aq_failures, bool ena_auto_link_update) if (!pi || !aq_failures) return -EINVAL; - *aq_failures = 0; hw = pi->hw; pcaps = kzalloc_obj(*pcaps); diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c index 0704e92ab0430..5f6f29142e301 100644 --- a/drivers/net/ethernet/intel/ice/ice_dpll.c +++ b/drivers/net/ethernet/intel/ice/ice_dpll.c @@ -4249,9 +4249,13 @@ ice_dpll_init_pins_info(struct ice_pf *pf, enum ice_dpll_pin_type pin_type) static void ice_dpll_deinit_info(struct ice_pf *pf) { kfree(pf->dplls.inputs); + pf->dplls.inputs = NULL; kfree(pf->dplls.outputs); + pf->dplls.outputs = NULL; kfree(pf->dplls.eec.input_prio); + pf->dplls.eec.input_prio = NULL; kfree(pf->dplls.pps.input_prio); + pf->dplls.pps.input_prio = NULL; } /** @@ -4343,12 +4347,16 @@ static int ice_dpll_init_info(struct ice_pf *pf, bool cgu) alloc_size = sizeof(*de->input_prio) * d->num_inputs; de->input_prio = kzalloc(alloc_size, GFP_KERNEL); - if (!de->input_prio) - return -ENOMEM; + if (!de->input_prio) { + ret = -ENOMEM; + goto deinit_info; + } dp->input_prio = kzalloc(alloc_size, GFP_KERNEL); - if (!dp->input_prio) - return -ENOMEM; + if (!dp->input_prio) { + ret = -ENOMEM; + goto deinit_info; + } ret = ice_dpll_init_pins_info(pf, ICE_DPLL_PIN_TYPE_INPUT); if (ret) @@ -4373,12 +4381,12 @@ static int ice_dpll_init_info(struct ice_pf *pf, bool cgu) ret = ice_get_cgu_rclk_pin_info(&pf->hw, &d->base_rclk_idx, &pf->dplls.rclk.num_parents); if (ret) - return ret; + goto deinit_info; for (i = 0; i < pf->dplls.rclk.num_parents; i++) pf->dplls.rclk.parent_idx[i] = d->base_rclk_idx + i; ret = ice_dpll_init_pins_info(pf, ICE_DPLL_PIN_TYPE_RCLK_INPUT); if (ret) - return ret; + goto deinit_info; de->mode = DPLL_MODE_AUTOMATIC; dp->mode = DPLL_MODE_AUTOMATIC; diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c index 2e4f0969035f7..c30e27bbfe6e2 100644 --- a/drivers/net/ethernet/intel/ice/ice_eswitch.c +++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c @@ -117,8 +117,6 @@ static int ice_eswitch_setup_repr(struct ice_pf *pf, struct ice_repr *repr) if (!repr->dst) return -ENOMEM; - netif_keep_dst(uplink_vsi->netdev); - dst = repr->dst; dst->u.port_info.port_id = vsi->vsi_num; dst->u.port_info.lower_dev = uplink_vsi->netdev; @@ -312,6 +310,8 @@ static int ice_eswitch_enable_switchdev(struct ice_pf *pf) if (ice_eswitch_br_offloads_init(pf)) goto err_br_offloads; + netif_keep_dst(uplink_vsi->netdev); + pf->eswitch.is_running = true; return 0; diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index f28416a707d77..cf6f3bd29f840 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -3508,7 +3508,7 @@ ice_set_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause) struct ice_vsi *vsi = np->vsi; struct ice_hw *hw = &pf->hw; struct ice_port_info *pi; - u8 aq_failures; + u8 aq_failures = 0; bool link_up; u32 is_an; int err; @@ -3579,18 +3579,22 @@ ice_set_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause) /* Set the FC mode and only restart AN if link is up */ err = ice_set_fc(pi, &aq_failures, link_up); - if (aq_failures & ICE_SET_FC_AQ_FAIL_GET) { + switch (aq_failures) { + case ICE_SET_FC_AQ_FAIL_GET: netdev_info(netdev, "Set fc failed on the get_phy_capabilities call with err %d aq_err %s\n", err, libie_aq_str(hw->adminq.sq_last_status)); err = -EAGAIN; - } else if (aq_failures & ICE_SET_FC_AQ_FAIL_SET) { + break; + case ICE_SET_FC_AQ_FAIL_SET: netdev_info(netdev, "Set fc failed on the set_phy_config call with err %d aq_err %s\n", err, libie_aq_str(hw->adminq.sq_last_status)); err = -EAGAIN; - } else if (aq_failures & ICE_SET_FC_AQ_FAIL_UPDATE) { + break; + case ICE_SET_FC_AQ_FAIL_UPDATE: netdev_info(netdev, "Set fc failed on the get_link_info call with err %d aq_err %s\n", err, libie_aq_str(hw->adminq.sq_last_status)); err = -EAGAIN; + break; } return err; diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index e2fbe111f8496..e2fd2dab03e36 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -4789,16 +4789,14 @@ static void ice_init_wakeup(struct ice_pf *pf) device_set_wakeup_enable(ice_pf_to_dev(pf), false); } -static int ice_init_link(struct ice_pf *pf) +static void ice_init_link(struct ice_pf *pf) { struct device *dev = ice_pf_to_dev(pf); int err; err = ice_init_link_events(pf->hw.port_info); - if (err) { + if (err) dev_err(dev, "ice_init_link_events failed: %d\n", err); - return err; - } /* not a fatal error if this fails */ err = ice_init_nvm_phy_type(pf->hw.port_info); @@ -4838,8 +4836,6 @@ static int ice_init_link(struct ice_pf *pf) } else { set_bit(ICE_FLAG_NO_MEDIA, pf->flags); } - - return err; } static int ice_init_pf_sw(struct ice_pf *pf) @@ -4982,13 +4978,11 @@ static int ice_init(struct ice_pf *pf) ice_init_wakeup(pf); - err = ice_init_link(pf); - if (err) - goto err_init_link; + ice_init_link(pf); err = ice_send_version(pf); if (err) - goto err_init_link; + goto err_deinit_pf_sw; ice_verify_cacheline_size(pf); @@ -5007,7 +5001,7 @@ static int ice_init(struct ice_pf *pf) return 0; -err_init_link: +err_deinit_pf_sw: ice_deinit_pf_sw(pf); err_init_pf_sw: ice_dealloc_vsis(pf); diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c index b1f46707dcc00..27e4acb1620f0 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c @@ -801,7 +801,7 @@ void ice_reset_all_vfs(struct ice_pf *pf) * setup only when VF creates its first FDIR rule. */ if (vf->ctrl_vsi_idx != ICE_NO_VSI) - ice_vf_ctrl_invalidate_vsi(vf); + ice_vf_ctrl_vsi_release(vf); ice_vf_pre_vsi_rebuild(vf); if (ice_vf_rebuild_vsi(vf)) { diff --git a/drivers/net/ethernet/intel/idpf/virtchnl2.h b/drivers/net/ethernet/intel/idpf/virtchnl2.h index 02ae447cc24ac..39fea65c075c7 100644 --- a/drivers/net/ethernet/intel/idpf/virtchnl2.h +++ b/drivers/net/ethernet/intel/idpf/virtchnl2.h @@ -1572,13 +1572,15 @@ VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_get_vport_tx_tstamp_latches); * struct virtchnl2_ptp_get_dev_clk_time - Associated with message * VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME. * @dev_time_ns: Device clock time value in nanoseconds + * @pad: Padding for future extensions * * PF/VF sends this message to receive the time from the main timer. */ struct virtchnl2_ptp_get_dev_clk_time { __le64 dev_time_ns; + u8 pad[8]; }; -VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_ptp_get_dev_clk_time); +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_get_dev_clk_time); /** * struct virtchnl2_ptp_get_cross_time: Associated with message @@ -1586,26 +1588,30 @@ VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_ptp_get_dev_clk_time); * @sys_time_ns: System counter value expressed in nanoseconds, read * synchronously with device time * @dev_time_ns: Device clock time value expressed in nanoseconds + * @pad: Padding for future extensions * * PF/VF sends this message to receive the cross time. */ struct virtchnl2_ptp_get_cross_time { __le64 sys_time_ns; __le64 dev_time_ns; + u8 pad[8]; }; -VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_get_cross_time); +VIRTCHNL2_CHECK_STRUCT_LEN(24, virtchnl2_ptp_get_cross_time); /** * struct virtchnl2_ptp_set_dev_clk_time: Associated with message * VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME. * @dev_time_ns: Device time value expressed in nanoseconds to set + * @pad: Padding for future extensions * * PF/VF sends this message to set the time of the main timer. */ struct virtchnl2_ptp_set_dev_clk_time { __le64 dev_time_ns; + u8 pad[8]; }; -VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_ptp_set_dev_clk_time); +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_set_dev_clk_time); /** * struct virtchnl2_ptp_adj_dev_clk_fine: Associated with message diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 8ac16808023cc..c470d2354ce8f 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -2649,7 +2649,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) } if (igc_fpe_is_pmac_enabled(adapter) && - igc_fpe_handle_mpacket(adapter, rx_desc, size, pktbuf)) { + igc_fpe_handle_mpacket(adapter, rx_desc, size, pktbuf + pkt_offset)) { /* Advance the ring next-to-clean */ igc_is_non_eop(rx_ring, rx_desc); cleaned_count++; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 2646ee6f295f0..9ec250c262842 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -3958,7 +3958,8 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, } /* initialize XPS */ - if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, ring->state)) { + if (!ring_is_xdp(ring) && + !test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, ring->state)) { struct ixgbe_q_vector *q_vector = ring->q_vector; if (q_vector) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index 61f2ef67defdd..eb5bf3b6bbb52 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -2798,6 +2798,7 @@ struct ixgbe_hic_hdr2_rsp { }; union ixgbe_hic_hdr2 { + u32 buf[1]; struct ixgbe_hic_hdr2_req req; struct ixgbe_hic_hdr2_rsp rsp; }; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c index 76d2fa3ef5182..4a0ccbf448a2a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c @@ -1228,7 +1228,7 @@ static int ixgbe_update_flash_X550(struct ixgbe_hw *hw) buffer.req.buf_lenl = FW_SHADOW_RAM_DUMP_LEN; buffer.req.checksum = FW_DEFAULT_CHECKSUM; - status = ixgbe_host_interface_command(hw, &buffer, sizeof(buffer), + status = ixgbe_host_interface_command(hw, buffer.buf, sizeof(buffer), IXGBE_HI_COMMAND_TIMEOUT, false); return status; } diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 0c061fb0ed072..744d6585a949d 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -5900,6 +5900,9 @@ static int mvneta_resume(struct device *device) rtnl_unlock(); mvneta_set_rx_mode(dev); + if (!pp->neta_armada3700) + on_each_cpu(mvneta_percpu_enable, pp, true); + return 0; } #endif diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c index fe8c4ffcd8f73..5dfdfbabfaddc 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c @@ -3542,15 +3542,18 @@ static int npc_defrag_alloc_free_slots(struct rvu *rvu, alloc_cnt2 = 0; rc = __npc_subbank_alloc(rvu, sb, - NPC_MCAM_KEY_X2, sb->b0b, + f->key_type, sb->b0b, sb->b0t, NPC_MCAM_LOWER_PRIO, false, cnt, save, cnt, true, &alloc_cnt1); - if (alloc_cnt1 < cnt) { + /* X4 entries only occupy bank 0 (b0b..b0t); see npc_subbank_idx_2_mcam_idx(). + * X2 uses both halves of the subbank, so spill into bank 1 if needed. + */ + if (alloc_cnt1 < cnt && f->key_type == NPC_MCAM_KEY_X2) { rc = __npc_subbank_alloc(rvu, sb, - NPC_MCAM_KEY_X2, sb->b1b, + f->key_type, sb->b1b, sb->b1t, NPC_MCAM_LOWER_PRIO, false, cnt - alloc_cnt1, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index dc42c81c09425..44fdd6ba7307c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -283,31 +283,30 @@ M(NPC_GET_FIELD_HASH_INFO, 0x6013, npc_get_field_hash_info, M(NPC_GET_FIELD_STATUS, 0x6014, npc_get_field_status, \ npc_get_field_status_req, \ npc_get_field_status_rsp) \ -M(NPC_CN20K_MCAM_GET_FREE_COUNT, 0x6015, npc_cn20k_get_fcnt, \ - msg_req, npc_cn20k_get_fcnt_rsp) \ -M(NPC_CN20K_GET_KEX_CFG, 0x6016, npc_cn20k_get_kex_cfg, \ +M(NPC_MCAM_DEFRAG, 0x6016, npc_defrag, \ + msg_req, \ + msg_rsp) \ +M(NPC_CN20K_GET_KEX_CFG, 0x6017, npc_cn20k_get_kex_cfg, \ msg_req, npc_cn20k_get_kex_cfg_rsp) \ -M(NPC_CN20K_MCAM_WRITE_ENTRY, 0x6017, npc_cn20k_mcam_write_entry, \ - npc_cn20k_mcam_write_entry_req, msg_rsp) \ -M(NPC_CN20K_MCAM_ALLOC_AND_WRITE_ENTRY, 0x6018, \ -npc_cn20k_mcam_alloc_and_write_entry, \ +M(NPC_CN20K_MCAM_GET_FREE_COUNT, 0x6018, npc_cn20k_get_fcnt, \ + msg_req, npc_cn20k_get_fcnt_rsp) \ +M(NPC_CN20K_MCAM_WRITE_ENTRY, 0x6019, npc_cn20k_mcam_write_entry, \ + npc_cn20k_mcam_write_entry_req, msg_rsp) \ +M(NPC_CN20K_MCAM_ALLOC_AND_WRITE_ENTRY, 0x601a, npc_cn20k_mcam_alloc_and_write_entry, \ npc_cn20k_mcam_alloc_and_write_entry_req, \ npc_mcam_alloc_and_write_entry_rsp) \ -M(NPC_CN20K_MCAM_READ_ENTRY, 0x6019, npc_cn20k_mcam_read_entry, \ +M(NPC_CN20K_MCAM_READ_ENTRY, 0x601b, npc_cn20k_mcam_read_entry, \ npc_mcam_read_entry_req, \ npc_cn20k_mcam_read_entry_rsp) \ -M(NPC_CN20K_MCAM_READ_BASE_RULE, 0x601a, npc_cn20k_read_base_steer_rule, \ - msg_req, npc_cn20k_mcam_read_base_rule_rsp) \ -M(NPC_MCAM_DEFRAG, 0x601b, npc_defrag, \ - msg_req, \ - msg_rsp) \ -M(NPC_MCAM_GET_NUM_KWS, 0x601c, npc_get_num_kws, \ +M(NPC_CN20K_MCAM_READ_BASE_RULE, 0x601c, npc_cn20k_read_base_steer_rule, \ + msg_req, npc_cn20k_mcam_read_base_rule_rsp) \ +M(NPC_MCAM_GET_NUM_KWS, 0x601d, npc_get_num_kws, \ npc_get_num_kws_req, \ npc_get_num_kws_rsp) \ -M(NPC_MCAM_GET_DFT_RL_IDXS, 0x601d, npc_get_dft_rl_idxs, \ +M(NPC_MCAM_GET_DFT_RL_IDXS, 0x601e, npc_get_dft_rl_idxs, \ msg_req, \ npc_get_dft_rl_idxs_rsp)\ -M(NPC_MCAM_GET_NPC_PFL_INFO, 0x601e, npc_get_pfl_info, \ +M(NPC_MCAM_GET_NPC_PFL_INFO, 0x601f, npc_get_pfl_info, \ msg_req, \ npc_get_pfl_info_rsp) \ /* NIX mbox IDs (range 0x8000 - 0xFFFF) */ \ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c index c1775bd01c2b4..a07e0b3d8d000 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c @@ -120,13 +120,13 @@ void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDX(id); stats->pkt_untaged_cnt = mcs_reg_read(mcs, reg); - reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(id); - stats->pkt_ctl_cnt = mcs_reg_read(mcs, reg); - if (mcs->hw->mcs_blks > 1) { reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(id); stats->pkt_notag_cnt = mcs_reg_read(mcs, reg); + return; } + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(id); + stats->pkt_ctl_cnt = mcs_reg_read(mcs, reg); } void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c index d2163da28d18f..fa4ea1258d29d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c @@ -178,6 +178,15 @@ int rvu_mbox_handler_lmtst_tbl_setup(struct rvu *rvu, * pcifunc (will be the one who is calling this mailbox). */ if (req->base_pcifunc) { + /* A VF is untrusted and must not redirect its LMTLINE to + * another PF's region, so confine VF callers to their own PF. + */ + if (is_vf(req->hdr.pcifunc) && + (!is_pf_func_valid(rvu, req->base_pcifunc) || + rvu_get_pf(rvu->pdev, req->hdr.pcifunc) != + rvu_get_pf(rvu->pdev, req->base_pcifunc))) + return -EPERM; + /* Calculating the LMT table index equivalent to primary * pcifunc. */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index fa461489acdd6..3456313d3b3c0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -482,10 +482,11 @@ static int rvu_dbg_mcs_rx_secy_stats_display(struct seq_file *filp, void *unused seq_printf(filp, "secy%d: Tagged ctrl pkts: %lld\n", secy_id, stats.pkt_tagged_ctl_cnt); seq_printf(filp, "secy%d: Untaged pkts: %lld\n", secy_id, stats.pkt_untaged_cnt); - seq_printf(filp, "secy%d: Ctrl pkts: %lld\n", secy_id, stats.pkt_ctl_cnt); if (mcs->hw->mcs_blks > 1) seq_printf(filp, "secy%d: pkts notag: %lld\n", secy_id, stats.pkt_notag_cnt); + else + seq_printf(filp, "secy%d: Ctrl pkts: %lld\n", secy_id, stats.pkt_ctl_cnt); } mutex_unlock(&mcs->stats_lock); return 0; @@ -2809,6 +2810,14 @@ static void rvu_dbg_npa_init(struct rvu *rvu) &rvu_dbg_npa_ndc_hits_miss_fops); } +/* Per-lmac CGX debugfs files need both RVU and CGX handle; inode->i_private + * points here so seq_file ops avoid pci_get_device(PCI_DEVID_OCTEONTX2_RVU_AF). + */ +struct rvu_cgx_lmac_dbgfs_ctx { + struct rvu *rvu; + void *cgxd; +}; + #define PRINT_CGX_CUML_NIXRX_STATUS(idx, name) \ ({ \ u64 cnt; \ @@ -2831,18 +2840,14 @@ static void rvu_dbg_npa_init(struct rvu *rvu) static int cgx_print_stats(struct seq_file *s, int lmac_id) { + struct rvu_cgx_lmac_dbgfs_ctx *dctx = s->private; struct cgx_link_user_info linfo; + struct rvu *rvu = dctx->rvu; struct mac_ops *mac_ops; - void *cgxd = s->private; + void *cgxd = dctx->cgxd; u64 ucast, mcast, bcast; int stat = 0, err = 0; u64 tx_stat, rx_stat; - struct rvu *rvu; - - rvu = pci_get_drvdata(pci_get_device(PCI_VENDOR_ID_CAVIUM, - PCI_DEVID_OCTEONTX2_RVU_AF, NULL)); - if (!rvu) - return -ENODEV; mac_ops = get_mac_ops(cgxd); /* There can be no CGX devices at all */ @@ -2949,20 +2954,16 @@ RVU_DEBUG_SEQ_FOPS(cgx_stat, cgx_stat_display, NULL); static int cgx_print_dmac_flt(struct seq_file *s, int lmac_id) { + struct rvu_cgx_lmac_dbgfs_ctx *dctx = s->private; + struct rvu *rvu = dctx->rvu; struct pci_dev *pdev = NULL; - void *cgxd = s->private; + void *cgxd = dctx->cgxd; char *bcast, *mcast; u16 index, domain; u8 dmac[ETH_ALEN]; - struct rvu *rvu; u64 cfg, mac; int pf; - rvu = pci_get_drvdata(pci_get_device(PCI_VENDOR_ID_CAVIUM, - PCI_DEVID_OCTEONTX2_RVU_AF, NULL)); - if (!rvu) - return -ENODEV; - pf = cgxlmac_to_pf(rvu, cgx_get_cgxid(cgxd), lmac_id); domain = 2; @@ -3009,17 +3010,13 @@ RVU_DEBUG_SEQ_FOPS(cgx_dmac_flt, cgx_dmac_flt_display, NULL); static int cgx_print_fwdata(struct seq_file *s, int lmac_id) { + struct rvu_cgx_lmac_dbgfs_ctx *dctx = s->private; struct cgx_lmac_fwdata_s *fwdata; - void *cgxd = s->private; + struct rvu *rvu = dctx->rvu; + void *cgxd = dctx->cgxd; struct phy_s *phy; - struct rvu *rvu; int cgx_id, i; - rvu = pci_get_drvdata(pci_get_device(PCI_VENDOR_ID_CAVIUM, - PCI_DEVID_OCTEONTX2_RVU_AF, NULL)); - if (!rvu) - return -ENODEV; - if (!rvu->fwdata) return -EAGAIN; @@ -3100,6 +3097,7 @@ RVU_DEBUG_SEQ_FOPS(cgx_fwdata, cgx_fwdata_display, NULL); static void rvu_dbg_cgx_init(struct rvu *rvu) { + struct rvu_cgx_lmac_dbgfs_ctx *ctx; struct mac_ops *mac_ops; unsigned long lmac_bmap; int i, lmac_id; @@ -3126,6 +3124,13 @@ static void rvu_dbg_cgx_init(struct rvu *rvu) rvu->rvu_dbg.cgx = debugfs_create_dir(dname, rvu->rvu_dbg.cgx_root); + ctx = devm_kzalloc(rvu->dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + continue; + + ctx->rvu = rvu; + ctx->cgxd = cgx; + for_each_set_bit(lmac_id, &lmac_bmap, rvu->hw->lmac_per_cgx) { /* lmac debugfs dir */ sprintf(dname, "lmac%d", lmac_id); @@ -3133,13 +3138,13 @@ static void rvu_dbg_cgx_init(struct rvu *rvu) debugfs_create_dir(dname, rvu->rvu_dbg.cgx); debugfs_create_file_aux_num("stats", 0600, rvu->rvu_dbg.lmac, - cgx, lmac_id, &rvu_dbg_cgx_stat_fops); + ctx, lmac_id, &rvu_dbg_cgx_stat_fops); debugfs_create_file_aux_num("mac_filter", 0600, - rvu->rvu_dbg.lmac, cgx, lmac_id, + rvu->rvu_dbg.lmac, ctx, lmac_id, &rvu_dbg_cgx_dmac_flt_fops); - debugfs_create_file("fwdata", 0600, - rvu->rvu_dbg.lmac, cgx, - &rvu_dbg_cgx_fwdata_fops); + debugfs_create_file_aux_num("fwdata", 0600, + rvu->rvu_dbg.lmac, ctx, + lmac_id, &rvu_dbg_cgx_fwdata_fops); } } } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index 6494a9ee2f0dd..3b47ecb44d51b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -1510,7 +1510,9 @@ static int rvu_af_dl_nix_maxlf_validate(struct devlink *devlink, u32 id, struct rvu_devlink *rvu_dl = devlink_priv(devlink); struct rvu *rvu = rvu_dl->rvu; u16 max_nix0_lf, max_nix1_lf; - struct npc_mcam *mcam; + struct rvu_block *block; + int blkaddr = 0; + int free_lfs; u64 cfg; cfg = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_CONST2); @@ -1518,14 +1520,23 @@ static int rvu_af_dl_nix_maxlf_validate(struct devlink *devlink, u32 id, cfg = rvu_read64(rvu, BLKADDR_NIX1, NIX_AF_CONST2); max_nix1_lf = cfg & 0xFFF; - /* Do not allow user to modify maximum NIX LFs while mcam entries - * have already been assigned. + /* Do not allow user to modify maximum NIX LFs while NIX LFs + * have already been assigned. Note that modifying NIX LFs count + * can be done only before any LF attach requests from PFs and VFs + * and not later or concurrently. */ - mcam = &rvu->hw->mcam; - if (mcam->bmap_fcnt < mcam->bmap_entries) { - NL_SET_ERR_MSG_MOD(extack, - "mcam entries have already been assigned, can't resize"); - return -EPERM; + blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); + while (blkaddr) { + block = &rvu->hw->block[blkaddr]; + + free_lfs = rvu_rsrc_free_count(&block->lf); + if (free_lfs != block->lf.max) { + NL_SET_ERR_MSG_MOD(extack, + "NIX LFs already assigned, can't resize"); + return -EPERM; + } + + blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); } if (max_nix0_lf && val.vu16 > max_nix0_lf) { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index f977734ae712c..219fc44ab68d4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -527,19 +527,24 @@ static int nix_setup_bpids(struct rvu *rvu, struct nix_hw *hw, int blkaddr) bp->fn_map = devm_kcalloc(rvu->dev, bp->bpids.max, sizeof(u16), GFP_KERNEL); if (!bp->fn_map) - return -ENOMEM; + goto free_bpids; bp->intf_map = devm_kcalloc(rvu->dev, bp->bpids.max, sizeof(u8), GFP_KERNEL); if (!bp->intf_map) - return -ENOMEM; + goto free_bpids; bp->ref_cnt = devm_kcalloc(rvu->dev, bp->bpids.max, sizeof(u8), GFP_KERNEL); if (!bp->ref_cnt) - return -ENOMEM; + goto free_bpids; return 0; + +free_bpids: + rvu_free_bitmap(&bp->bpids); + bp->bpids.bmap = NULL; + return -ENOMEM; } void rvu_nix_flr_free_bpids(struct rvu *rvu, u16 pcifunc) @@ -4548,7 +4553,7 @@ int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req, rvu_npc_install_allmulti_entry(rvu, pcifunc, nixlf, pfvf->rx_chan_base); } else { - if (!nix_rx_multicast) + if (!nix_rx_multicast && !is_vf(pcifunc)) rvu_npc_enable_allmulti_entry(rvu, pcifunc, nixlf, false); } @@ -4558,7 +4563,7 @@ int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req, pfvf->rx_chan_base, pfvf->rx_chan_cnt); else - if (!nix_rx_multicast) + if (!nix_rx_multicast && !is_vf(pcifunc)) rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false); return 0; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index d301a3f0f87a8..4994385a822b7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -2181,7 +2181,7 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) /* Alloc memory for MCAM entry to counter mapping and for tracking * counter's reference count. */ - mcam->entry2cntr_map = kcalloc(mcam->bmap_entries, sizeof(u16), + mcam->entry2cntr_map = kcalloc(mcam->total_entries, sizeof(u16), GFP_KERNEL); if (!mcam->entry2cntr_map) goto free_cntr_map; @@ -2197,10 +2197,11 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) if (!mcam->entry2target_pffunc) goto free_cntr_refcnt; - for (index = 0; index < mcam->bmap_entries; index++) { + for (index = 0; index < mcam->bmap_entries; index++) mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; + + for (index = 0; index < mcam->total_entries; index++) mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; - } for (cntr = 0; cntr < mcam->counters.max; cntr++) mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; @@ -3531,7 +3532,7 @@ static int __npc_mcam_free_counter(struct rvu *rvu, struct msg_rsp *rsp) { struct npc_mcam *mcam = &rvu->hw->mcam; - u16 index, entry = 0; + u16 index; int blkaddr, err; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); @@ -3547,20 +3548,16 @@ static int __npc_mcam_free_counter(struct rvu *rvu, mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; rvu_free_rsrc(&mcam->counters, req->cntr); - /* Disable all MCAM entry's stats which are using this counter */ - while (entry < mcam->bmap_entries) { + /* Disable all MCAM entry's stats which are using this counter. + * Scan the full MCAM index range: AF-reserved rules (e.g. CPT pass-2) + */ + for (index = 0; index < mcam->total_entries; index++) { if (!mcam->cntr_refcnt[req->cntr]) break; - - index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); - if (index >= mcam->bmap_entries) - break; - entry = index + 1; if (mcam->entry2cntr_map[index] != req->cntr) continue; - - npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, - index, req->cntr); + npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, index, + req->cntr); } return 0; @@ -3631,7 +3628,7 @@ int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) { struct npc_mcam *mcam = &rvu->hw->mcam; - u16 index, entry = 0; + u16 index; int blkaddr, rc; /* Counter is not supported for CN20K */ @@ -3658,20 +3655,13 @@ int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, } /* Disable all MCAM entry's stats which are using this counter */ - while (entry < mcam->bmap_entries) { + for (index = 0; index < mcam->total_entries; index++) { if (!mcam->cntr_refcnt[req->cntr]) break; - - index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); - if (index >= mcam->bmap_entries) - break; - entry = index + 1; - if (mcam->entry2cntr_map[index] != req->cntr) continue; - - npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, - index, req->cntr); + npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, index, + req->cntr); } exit: mutex_unlock(&mcam->lock); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c index 34f1e066707bd..3d4d3ab5183b8 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c @@ -2215,7 +2215,7 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, return err; } - dev_err(rvu->dev, + dev_dbg(rvu->dev, "%s: Installed single drop on non hit rule at %d, cntr=%d\n", __func__, mcam_idx, req.cntr); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c index 2cc1bdfd9b2e8..9524d38f15822 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c @@ -182,6 +182,7 @@ static void cn10k_mcs_free_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir, clear_req->id = hw_rsrc_id; clear_req->type = type; clear_req->dir = dir; + clear_req->all = all; req = otx2_mbox_alloc_msg_mcs_free_resources(mbox); if (!req) @@ -1776,11 +1777,16 @@ int cn10k_mcs_init(struct otx2_nic *pfvf) void cn10k_mcs_free(struct otx2_nic *pfvf) { + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) return; - cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, 0, true); - cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, 0, true); + if (!list_empty(&cfg->txsc_list)) { + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, 0, true); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, 0, true); + } + kfree(pfvf->macsec_cfg); pfvf->macsec_cfg = NULL; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index f9fbf0c176482..617a0db97e804 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -1119,9 +1119,16 @@ int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af) { struct otx2_hw *hw = &pf->hw; struct msg_req *req; + u64 mbox_int_mask; char *irq_name; int err; + mbox_int_mask = !is_cn20k(pf->pdev) ? BIT_ULL(0) : + BIT_ULL(0) | BIT_ULL(1); + + /* Clear stale mailbox interrupt state before installing the handler. */ + otx2_write64(pf, RVU_PF_INT, mbox_int_mask); + /* Register mailbox interrupt handler */ if (!is_cn20k(pf->pdev)) { irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE]; @@ -1147,17 +1154,8 @@ int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af) return err; } - /* Enable mailbox interrupt for msgs coming from AF. - * First clear to avoid spurious interrupts, if any. - */ - if (!is_cn20k(pf->pdev)) { - otx2_write64(pf, RVU_PF_INT, BIT_ULL(0)); - otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0)); - } else { - otx2_write64(pf, RVU_PF_INT, BIT_ULL(0) | BIT_ULL(1)); - otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0) | - BIT_ULL(1)); - } + /* Enable mailbox interrupt for msgs coming from AF. */ + otx2_write64(pf, RVU_PF_INT_ENA_W1S, mbox_int_mask); if (!probe_af) return 0; @@ -1571,14 +1569,15 @@ static void otx2_free_sq_res(struct otx2_nic *pf) otx2_sq_free_sqbs(pf); for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) { sq = &qset->sq[qidx]; - /* Skip freeing Qos queues if they are not initialized */ - if (!sq->sqe) - continue; - qmem_free(pf->dev, sq->sqe); - qmem_free(pf->dev, sq->sqe_ring); - qmem_free(pf->dev, sq->cpt_resp); - qmem_free(pf->dev, sq->tso_hdrs); - kfree(sq->sg); + /* sq->sqe is not initialized for unused QoS queues */ + if (sq->sqe) { + qmem_free(pf->dev, sq->sqe); + qmem_free(pf->dev, sq->sqe_ring); + qmem_free(pf->dev, sq->cpt_resp); + qmem_free(pf->dev, sq->tso_hdrs); + qmem_free(pf->dev, sq->timestamps); + kfree(sq->sg); + } kfree(sq->sqb_ptrs); } } @@ -1713,13 +1712,12 @@ int otx2_init_hw_resources(struct otx2_nic *pf) return err; err_free_nix_queues: - otx2_free_sq_res(pf); otx2_free_cq_res(pf); otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false); err_free_txsch: otx2_txschq_stop(pf); err_free_sq_ptrs: - otx2_sq_free_sqbs(pf); + otx2_free_sq_res(pf); err_free_rq_ptrs: otx2_free_aura_ptr(pf, AURA_NIX_RQ); otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true); @@ -2520,10 +2518,42 @@ EXPORT_SYMBOL(otx2_config_hwtstamp_set); static int otx2_do_set_vf_mac(struct otx2_nic *pf, int vf, const u8 *mac) { + struct npc_get_field_status_req *freq; + struct npc_get_field_status_rsp *frsp; struct npc_install_flow_req *req; int err; mutex_lock(&pf->mbox.lock); + + /* Skip installing the DMAC filter if the hardware parser profile + * does not support DMAC extraction. + */ + freq = otx2_mbox_alloc_msg_npc_get_field_status(&pf->mbox); + if (!freq) { + err = -ENOMEM; + goto out; + } + + freq->field = NPC_DMAC; + err = otx2_sync_mbox_msg(&pf->mbox); + if (err) + goto out; + + frsp = (struct npc_get_field_status_rsp *)otx2_mbox_get_rsp + (&pf->mbox.mbox, 0, &freq->hdr); + if (IS_ERR(frsp)) { + err = PTR_ERR(frsp); + goto out; + } + + if (!frsp->enable) { + netdev_warn(pf->netdev, + "VF %d MAC filter not installed: DMAC extraction not supported by parser profile\n", + vf); + err = -EOPNOTSUPP; + goto out; + } + req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox); if (!req) { err = -ENOMEM; @@ -2562,13 +2592,12 @@ static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) if (!is_valid_ether_addr(mac)) return -EINVAL; - config = &pf->vf_configs[vf]; - ether_addr_copy(config->mac, mac); - ret = otx2_do_set_vf_mac(pf, vf, mac); - if (ret == 0) - dev_info(&pdev->dev, - "Load/Reload VF driver\n"); + if (ret == 0) { + config = &pf->vf_configs[vf]; + ether_addr_copy(config->mac, mac); + dev_info(&pdev->dev, "Load/Reload VF driver\n"); + } return ret; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index f4fdbfba86676..b022f52c68450 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -251,9 +251,17 @@ static int otx2vf_register_mbox_intr(struct otx2_nic *vf, bool probe_pf) { struct otx2_hw *hw = &vf->hw; struct msg_req *req; + u64 mbox_int_mask; char *irq_name; int err; + mbox_int_mask = !is_cn20k(vf->pdev) ? BIT_ULL(0) : + BIT_ULL(0) | BIT_ULL(1) | + BIT_ULL(2) | BIT_ULL(3); + + /* Clear stale mailbox interrupt state before installing the handler. */ + otx2_write64(vf, RVU_VF_INT, mbox_int_mask); + /* Register mailbox interrupt handler */ irq_name = &hw->irq_name[RVU_VF_INT_VEC_MBOX * NAME_SIZE]; snprintf(irq_name, NAME_SIZE, "RVUVF%d AFVF Mbox", ((vf->pcifunc & @@ -274,18 +282,8 @@ static int otx2vf_register_mbox_intr(struct otx2_nic *vf, bool probe_pf) return err; } - /* Enable mailbox interrupt for msgs coming from PF. - * First clear to avoid spurious interrupts, if any. - */ - if (!is_cn20k(vf->pdev)) { - otx2_write64(vf, RVU_VF_INT, BIT_ULL(0)); - otx2_write64(vf, RVU_VF_INT_ENA_W1S, BIT_ULL(0)); - } else { - otx2_write64(vf, RVU_VF_INT, BIT_ULL(0) | BIT_ULL(1) | - BIT_ULL(2) | BIT_ULL(3)); - otx2_write64(vf, RVU_VF_INT_ENA_W1S, BIT_ULL(0) | - BIT_ULL(1) | BIT_ULL(2) | BIT_ULL(3)); - } + /* Enable mailbox interrupt for msgs coming from PF. */ + otx2_write64(vf, RVU_VF_INT_ENA_W1S, mbox_int_mask); if (!probe_pf) return 0; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index 41e19e9ad28d4..a82e7a8029851 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -373,7 +373,7 @@ static int prestera_port_sfp_bind(struct prestera_port *port) struct device_node *ports, *node; struct fwnode_handle *fwnode; struct phylink *phy_link; - int err; + int err = 0; if (!sw->np) return 0; diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 7d771168b9901..5d291e50a47bb 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -4960,6 +4960,11 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) mac_ops = &rt5350_phylink_ops; + if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_2P5GPHY) && + id == MTK_GMAC2_ID) + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + mac->phylink_config.supported_interfaces); + phylink = phylink_create(&mac->phylink_config, of_fwnode_handle(mac->of_node), phy_mode, mac_ops); @@ -4970,11 +4975,6 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink = phylink; - if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_2P5GPHY) && - id == MTK_GMAC2_ID) - __set_bit(PHY_INTERFACE_MODE_INTERNAL, - mac->phylink_config.supported_interfaces); - SET_NETDEV_DEV(eth->netdev[id], eth->dev); eth->netdev[id]->watchdog_timeo = 5 * HZ; eth->netdev[id]->netdev_ops = &mtk_netdev_ops; diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index 18279e2a7022e..8451dc3fd00aa 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -918,7 +918,7 @@ struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index) mib = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*mib), &ppe->mib_phys, GFP_KERNEL); if (!mib) - return NULL; + goto err_free_l2_flows; ppe->mib_table = mib; @@ -926,7 +926,7 @@ struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index) GFP_KERNEL); if (!acct) - return NULL; + goto err_free_l2_flows; ppe->acct_table = acct; } diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c index 781c691473e14..519c364e87d19 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c @@ -310,9 +310,9 @@ wed_amsdu_show(struct seq_file *s, void *data) WED_AMSDU_ENG_MAX_QGPP_CNT), DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_CUR_ENTRY), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(2), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_MAX_BUF_MERGED), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(2), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_MAX_MSDU_MERGED), DUMP_STR("WED AMDSU ENG2 INFO"), @@ -414,7 +414,7 @@ wed_amsdu_show(struct seq_file *s, void *data) WED_AMSDU_ENG_CUR_ENTRY), DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(7), WED_AMSDU_ENG_MAX_BUF_MERGED), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(4), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(7), WED_AMSDU_ENG_MAX_MSDU_MERGED), DUMP_STR("WED AMDSU ENG8 INFO"), diff --git a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c index fa6b216034169..0d38183c6ba70 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c @@ -367,8 +367,12 @@ mtk_wed_mcu_load_firmware(struct mtk_wed_wo *wo) /* wo firmware reset */ wo_w32(MTK_WO_MCU_CFG_LS_WF_MCCR_CLR_ADDR, 0xc00); - val = wo_r32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR) | - MTK_WO_MCU_CFG_LS_WF_WM_WA_WM_CPU_RSTB_MASK; + val = wo_r32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR); + + if (!mtk_wed_is_v3_or_greater(wo->hw) && wo->hw->index) + val |= MTK_WO_MCU_CFG_LS_WF_WM_WA_WA_CPU_RSTB_MASK; + else + val |= MTK_WO_MCU_CFG_LS_WF_WM_WA_WM_CPU_RSTB_MASK; wo_w32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR, val); out: release_firmware(fw); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index d39fe9c4a87cd..19e50f0d55af3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -41,7 +41,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) += en_dcbnl.o en/port_buffer.o mlx5_core-$(CONFIG_PCI_HYPERV_INTERFACE) += en/hv_vhca_stats.o mlx5_core-$(CONFIG_MLX5_ESWITCH) += lag/mp.o lag/port_sel.o lib/geneve.o lib/port_tun.o \ en_rep.o en/rep/bond.o en/mod_hdr.o \ - en/mapping.o lag/mpesw.o + en/mapping.o lag/mpesw.o lag/shared_fdb.o mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en_tc.o en/rep/tc.o en/rep/neigh.o \ lib/fs_chains.o en/tc_tun.o \ esw/indir_table.o en/tc_tun_encap.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 2270e2e550dd9..d507289096c20 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -987,6 +987,18 @@ struct mlx5e_priv { struct ethtool_fec_hist_range *fec_ranges; }; +static inline u16 mlx5e_stats_nch_read(const struct mlx5e_priv *priv) +{ + /* Pairs with smp_store_release in mlx5e_stats_nch_write(). */ + return smp_load_acquire(&priv->stats_nch); +} + +static inline void mlx5e_stats_nch_write(struct mlx5e_priv *priv, u16 n) +{ + /* Pairs with smp_load_acquire in mlx5e_stats_nch_read(). */ + smp_store_release(&priv->stats_nch, n); +} + struct mlx5e_dev { struct net_device *netdev; struct devlink_port dl_port; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c index 195863b2c013e..631f802105d5b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c @@ -33,9 +33,10 @@ mlx5e_hv_vhca_fill_ring_stats(struct mlx5e_priv *priv, int ch, static void mlx5e_hv_vhca_fill_stats(struct mlx5e_priv *priv, void *data, int buf_len) { + u16 nch = mlx5e_stats_nch_read(priv); int ch, i = 0; - for (ch = 0; ch < priv->stats_nch; ch++) { + for (ch = 0; ch < nch; ch++) { void *buf = data + i; if (WARN_ON_ONCE(buf + @@ -49,9 +50,16 @@ static void mlx5e_hv_vhca_fill_stats(struct mlx5e_priv *priv, void *data, } static int mlx5e_hv_vhca_stats_buf_size(struct mlx5e_priv *priv) +{ + u16 nch = mlx5e_stats_nch_read(priv); + + return sizeof(struct mlx5e_hv_vhca_per_ring_stats) * nch; +} + +static int mlx5e_hv_vhca_stats_buf_max_size(struct mlx5e_priv *priv) { return (sizeof(struct mlx5e_hv_vhca_per_ring_stats) * - priv->stats_nch); + max(priv->max_nch, priv->stats_nch)); } static void mlx5e_hv_vhca_stats_work(struct work_struct *work) @@ -67,7 +75,7 @@ static void mlx5e_hv_vhca_stats_work(struct work_struct *work) sagent = container_of(dwork, struct mlx5e_hv_vhca_stats_agent, work); priv = container_of(sagent, struct mlx5e_priv, stats_agent); buf_len = mlx5e_hv_vhca_stats_buf_size(priv); - agent = sagent->agent; + agent = READ_ONCE(sagent->agent); buf = sagent->buf; memset(buf, 0, buf_len); @@ -100,7 +108,7 @@ static void mlx5e_hv_vhca_stats_control(struct mlx5_hv_vhca_agent *agent, sagent = &priv->stats_agent; block->version = MLX5_HV_VHCA_STATS_VERSION; - block->rings = priv->stats_nch; + block->rings = mlx5e_stats_nch_read(priv); if (!block->command) { cancel_delayed_work_sync(&priv->stats_agent.work); @@ -122,18 +130,21 @@ static void mlx5e_hv_vhca_stats_cleanup(struct mlx5_hv_vhca_agent *agent) void mlx5e_hv_vhca_stats_create(struct mlx5e_priv *priv) { - int buf_len = mlx5e_hv_vhca_stats_buf_size(priv); + int buf_len = mlx5e_hv_vhca_stats_buf_max_size(priv); struct mlx5_hv_vhca_agent *agent; priv->stats_agent.buf = kvzalloc(buf_len, GFP_KERNEL); if (!priv->stats_agent.buf) return; + INIT_DELAYED_WORK(&priv->stats_agent.work, mlx5e_hv_vhca_stats_work); + agent = mlx5_hv_vhca_agent_create(priv->mdev->hv_vhca, MLX5_HV_VHCA_AGENT_STATS, mlx5e_hv_vhca_stats_control, NULL, mlx5e_hv_vhca_stats_cleanup, - priv); + priv, + &priv->stats_agent.agent); if (IS_ERR_OR_NULL(agent)) { if (IS_ERR(agent)) @@ -142,18 +153,20 @@ void mlx5e_hv_vhca_stats_create(struct mlx5e_priv *priv) agent); kvfree(priv->stats_agent.buf); - return; + priv->stats_agent.buf = NULL; } - - priv->stats_agent.agent = agent; - INIT_DELAYED_WORK(&priv->stats_agent.work, mlx5e_hv_vhca_stats_work); } void mlx5e_hv_vhca_stats_destroy(struct mlx5e_priv *priv) { - if (IS_ERR_OR_NULL(priv->stats_agent.agent)) + struct mlx5_hv_vhca_agent *agent; + + agent = READ_ONCE(priv->stats_agent.agent); + if (IS_ERR_OR_NULL(agent)) return; - mlx5_hv_vhca_agent_destroy(priv->stats_agent.agent); + mlx5_hv_vhca_agent_destroy(agent); + WRITE_ONCE(priv->stats_agent.agent, NULL); kvfree(priv->stats_agent.buf); + priv->stats_agent.buf = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 71b3a059c9647..daff53ba7d09f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -714,34 +714,43 @@ static int mlx5e_macsec_add_rxsc(struct macsec_context *ctx) } sc_xarray_element->rx_sc = rx_sc; - err = xa_alloc(&macsec->sc_xarray, &sc_xarray_element->fs_id, sc_xarray_element, - XA_LIMIT(1, MLX5_MACEC_RX_FS_ID_MAX), GFP_KERNEL); - if (err) { - if (err == -EBUSY) - netdev_err(ctx->netdev, - "MACsec offload: unable to create entry for RX SC (%d Rx SCs already allocated)\n", - MLX5_MACEC_RX_FS_ID_MAX); - goto destroy_sc_xarray_elemenet; - } rx_sc->md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL); if (!rx_sc->md_dst) { err = -ENOMEM; - goto erase_xa_alloc; + goto destroy_sc_xarray_elemenet; } rx_sc->sci = ctx_rx_sc->sci; rx_sc->active = ctx_rx_sc->active; - list_add_rcu(&rx_sc->rx_sc_list_element, rx_sc_list); - rx_sc->sc_xarray_element = sc_xarray_element; rx_sc->md_dst->u.macsec_info.sci = rx_sc->sci; + + /* + * Publish the fully-initialised SC last: xa_alloc() makes + * sc_xarray_element->rx_sc (and rx_sc->md_dst) reachable from the RX + * datapath via xa_load(). Doing it only after md_dst is allocated and + * initialised pairs with the rcu_read_lock()/xa_load() in + * mlx5e_macsec_offload_handle_rx_skb(), so a reader can never observe + * a non-NULL md_dst with uninitialised contents. + */ + err = xa_alloc(&macsec->sc_xarray, &sc_xarray_element->fs_id, sc_xarray_element, + XA_LIMIT(1, MLX5_MACEC_RX_FS_ID_MAX), GFP_KERNEL); + if (err) { + if (err == -EBUSY) + netdev_err(ctx->netdev, + "MACsec offload: unable to create entry for RX SC (%d Rx SCs already allocated)\n", + MLX5_MACEC_RX_FS_ID_MAX); + goto destroy_md_dst; + } + + list_add_rcu(&rx_sc->rx_sc_list_element, rx_sc_list); mutex_unlock(&macsec->lock); return 0; -erase_xa_alloc: - xa_erase(&macsec->sc_xarray, sc_xarray_element->fs_id); +destroy_md_dst: + dst_release(&rx_sc->md_dst->dst); destroy_sc_xarray_elemenet: kfree(sc_xarray_element); destroy_rx_sc: @@ -829,7 +838,7 @@ static void macsec_del_rxsc_ctx(struct mlx5e_macsec *macsec, struct mlx5e_macsec */ list_del_rcu(&rx_sc->rx_sc_list_element); xa_erase(&macsec->sc_xarray, rx_sc->sc_xarray_element->fs_id); - metadata_dst_free(rx_sc->md_dst); + dst_release(&rx_sc->md_dst->dst); kfree(rx_sc->sc_xarray_element); kfree_rcu_mightsleep(rx_sc); } @@ -1695,10 +1704,10 @@ void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev, rcu_read_lock(); sc_xarray_element = xa_load(&macsec->sc_xarray, fs_id); - rx_sc = sc_xarray_element->rx_sc; - if (rx_sc) { - dst_hold(&rx_sc->md_dst->dst); - skb_dst_set(skb, &rx_sc->md_dst->dst); + rx_sc = sc_xarray_element ? sc_xarray_element->rx_sc : NULL; + if (rx_sc && rx_sc->md_dst) { + if (dst_hold_safe(&rx_sc->md_dst->dst)) + skb_dst_set(skb, &rx_sc->md_dst->dst); } rcu_read_unlock(); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 8f2b3abe00921..94e5352a246cd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -2773,7 +2773,7 @@ static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu) GFP_KERNEL, cpu_to_node(cpu)); if (!priv->channel_stats[ix]) return -ENOMEM; - priv->stats_nch++; + mlx5e_stats_nch_write(priv, priv->stats_nch + 1); return 0; } @@ -4043,9 +4043,10 @@ static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) { + u16 nch = mlx5e_stats_nch_read(priv); int i; - for (i = 0; i < priv->stats_nch; i++) { + for (i = 0; i < nch; i++) { struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; @@ -5489,7 +5490,7 @@ static void mlx5e_get_queue_stats_rx(struct net_device *dev, int i, struct mlx5e_rq_stats *xskrq_stats; struct mlx5e_rq_stats *rq_stats; - if (mlx5e_is_uplink_rep(priv) || !priv->stats_nch) + if (mlx5e_is_uplink_rep(priv) || !mlx5e_stats_nch_read(priv)) return; channel_stats = priv->channel_stats[i]; @@ -5508,7 +5509,7 @@ static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i, struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_sq_stats *sq_stats; - if (!priv->stats_nch) + if (!mlx5e_stats_nch_read(priv)) return; /* no special case needed for ptp htb etc since txq2sq_stats is kept up @@ -5525,6 +5526,7 @@ static void mlx5e_get_base_stats(struct net_device *dev, struct netdev_queue_stats_tx *tx) { struct mlx5e_priv *priv = netdev_priv(dev); + u16 nch = mlx5e_stats_nch_read(priv); struct mlx5e_ptp *ptp_channel; int i, tc; @@ -5533,7 +5535,7 @@ static void mlx5e_get_base_stats(struct net_device *dev, rx->bytes = 0; rx->alloc_fail = 0; - for (i = priv->channels.params.num_channels; i < priv->stats_nch; i++) { + for (i = priv->channels.params.num_channels; i < nch; i++) { struct netdev_queue_stats_rx rx_i = {0}; mlx5e_get_queue_stats_rx(dev, i, &rx_i); @@ -5558,7 +5560,7 @@ static void mlx5e_get_base_stats(struct net_device *dev, tx->packets = 0; tx->bytes = 0; - for (i = 0; i < priv->stats_nch; i++) { + for (i = 0; i < nch; i++) { struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; /* handle two cases: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 1a3ecf073913c..8632b73179cbc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -516,6 +516,7 @@ static void mlx5e_stats_update_stats_rq_page_pool(struct mlx5e_channel *c) static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw) { struct mlx5e_sw_stats *s = &priv->stats.sw; + u16 nch = mlx5e_stats_nch_read(priv); int i; memset(s, 0, sizeof(*s)); @@ -523,7 +524,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw) for (i = 0; i < priv->channels.num; i++) /* for active channels only */ mlx5e_stats_update_stats_rq_page_pool(priv->channels.c[i]); - for (i = 0; i < priv->stats_nch; i++) { + for (i = 0; i < nch; i++) { struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; @@ -2615,7 +2616,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(ptp) { return; } static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(channels) { - int max_nch = priv->stats_nch; + int max_nch = mlx5e_stats_nch_read(priv); return (NUM_RQ_STATS * max_nch) + (NUM_CH_STATS * max_nch) + @@ -2628,8 +2629,8 @@ static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(channels) static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(channels) { + int max_nch = mlx5e_stats_nch_read(priv); bool is_xsk = priv->xsk.ever_used; - int max_nch = priv->stats_nch; int i, j, tc; for (i = 0; i < max_nch; i++) @@ -2661,8 +2662,8 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(channels) static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(channels) { + int max_nch = mlx5e_stats_nch_read(priv); bool is_xsk = priv->xsk.ever_used; - int max_nch = priv->stats_nch; int i, j, tc; for (i = 0; i < max_nch; i++) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index a9001d1c902ff..8b5a421a22a37 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -5526,6 +5526,9 @@ void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw) mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) { i = mlx5_lag_get_dev_seq(peer_esw->dev); + if (i < 0) + continue; + list_for_each_entry_safe(flow, tmp, &esw->offloads.peer_flows[i], peer[i]) mlx5e_tc_del_fdb_peers_flow(flow); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index e9cf7c592ce9e..f248c66caa848 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -316,6 +316,7 @@ struct mlx5_esw_offload { DECLARE_HASHTABLE(termtbl_tbl, 8); struct mutex termtbl_mutex; /* protects termtbl hash */ struct xarray vhca_map; + struct mutex reps_lock; /* protects representor load/unload/register */ const struct mlx5_eswitch_rep_ops *rep_ops[NUM_REP_TYPES]; u8 inline_mode; atomic64_t num_flows; @@ -898,6 +899,8 @@ int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw, void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw, struct mlx5_eswitch *slave_esw); int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw); +bool mlx5_eswitch_is_peer(struct mlx5_eswitch *esw, + struct mlx5_eswitch *peer_esw); bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev, bool from_fdb); void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev); @@ -913,13 +916,6 @@ static inline int mlx5_eswitch_num_vfs(struct mlx5_eswitch *esw) return 0; } -static inline int mlx5_eswitch_get_npeers(struct mlx5_eswitch *esw) -{ - if (mlx5_esw_allowed(esw)) - return esw->num_peers; - return 0; -} - static inline struct mlx5_flow_table * mlx5_eswitch_get_slow_fdb(struct mlx5_eswitch *esw) { @@ -949,6 +945,8 @@ mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_handle * mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num, struct mlx5_flow_table *lag_ft); +void mlx5_esw_reps_block(struct mlx5_eswitch *esw); +void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw); #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } @@ -993,8 +991,6 @@ static inline void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw, struct mlx5_eswitch *slave_esw) {} -static inline int mlx5_eswitch_get_npeers(struct mlx5_eswitch *esw) { return 0; } - static inline int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw) { @@ -1026,6 +1022,9 @@ mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev) return true; } +static inline void mlx5_esw_reps_block(struct mlx5_eswitch *esw) {} +static inline void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw) {} + static inline bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index a078d06f45673..fbd59225b8637 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -2410,6 +2410,16 @@ static int esw_create_restore_table(struct mlx5_eswitch *esw) return err; } +void mlx5_esw_reps_block(struct mlx5_eswitch *esw) +{ + mutex_lock(&esw->offloads.reps_lock); +} + +void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw) +{ + mutex_unlock(&esw->offloads.reps_lock); +} + static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode) { mlx5_devcom_comp_lock(esw->dev->priv.hca_devcom_comp); @@ -2642,6 +2652,7 @@ static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) mlx5_esw_for_each_rep(esw, i, rep) mlx5_esw_offloads_rep_cleanup(esw, rep); xa_destroy(&esw->offloads.vport_reps); + mutex_destroy(&esw->offloads.reps_lock); } static int esw_offloads_init_reps(struct mlx5_eswitch *esw) @@ -2651,6 +2662,7 @@ static int esw_offloads_init_reps(struct mlx5_eswitch *esw) int err; xa_init(&esw->offloads.vport_reps); + mutex_init(&esw->offloads.reps_lock); mlx5_esw_for_each_vport(esw, i, vport) { err = mlx5_esw_offloads_rep_add(esw, vport); @@ -3225,6 +3237,18 @@ static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, return 0; } +bool mlx5_eswitch_is_peer(struct mlx5_eswitch *esw, + struct mlx5_eswitch *peer_esw) +{ + u16 peer_esw_i; + + if (!mlx5_esw_allowed(esw) || !mlx5_esw_allowed(peer_esw)) + return false; + + peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); + return !!xa_load(&esw->paired, peer_esw_i); +} + static int mlx5_esw_offloads_devcom_event(int event, void *my_data, void *event_data) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 0a6003fe60e9f..674bed721e637 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -135,10 +135,11 @@ void mlx5i_cleanup(struct mlx5e_priv *priv) static void mlx5i_grp_sw_update_stats(struct mlx5e_priv *priv) { + u16 nch = mlx5e_stats_nch_read(priv); struct rtnl_link_stats64 s = {}; int i, j; - for (i = 0; i < priv->stats_nch; i++) { + for (i = 0; i < nch; i++) { struct mlx5e_channel_stats *channel_stats; struct mlx5e_rq_stats *rq_stats; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index f8e70ac5a85b7..3decb49e9f197 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -242,7 +242,7 @@ static void mlx5_ldev_free(struct kref *ref) unregister_netdevice_notifier_net(net, &ldev->nb); } - mlx5_ldev_for_each(i, 0, ldev) { + mlx5_lag_for_each(i, 0, ldev, MLX5_LAG_FILTER_ALL) { pf = mlx5_lag_pf(ldev, i); if (pf->port_change_nb.nb.notifier_call) { struct mlx5_nb *nb = &pf->port_change_nb; @@ -370,6 +370,22 @@ int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq) return -ENOENT; } +/* Return the appropriate iterator filter for a device in LAG: + * - SD shared FDB active: iterate only the device's SD group + * - SD group exists but shared FDB not active: iterate all devices + * - No SD: iterate ports only + */ +static u32 mlx5_lag_get_filter(struct mlx5_lag *ldev, struct mlx5_core_dev *dev) +{ + struct lag_func *pf = mlx5_lag_pf_by_dev(ldev, dev); + + if (pf && pf->sd_fdb_active) + return pf->group_id; + if (pf && pf->group_id) + return MLX5_LAG_FILTER_ALL; + return MLX5_LAG_FILTER_PORTS; +} + /* Reverse of mlx5_lag_get_dev_index_by_seq: given a device, return its * sequence number in the LAG. Master is always 0, others numbered * sequentially starting from 1. @@ -379,11 +395,13 @@ int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev) struct mlx5_lag *ldev = mlx5_lag_dev(dev); int master_idx, i, num = 1; struct lag_func *pf; + u32 filter; if (!ldev) return -ENOENT; - master_idx = mlx5_lag_get_master_idx(ldev); + filter = mlx5_lag_get_filter(ldev, dev); + master_idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, 0, filter); if (master_idx < 0) return -ENOENT; @@ -391,7 +409,7 @@ int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev) if (pf && pf->dev == dev) return 0; - mlx5_ldev_for_each(i, 0, ldev) { + mlx5_lag_for_each(i, 0, ldev, filter) { if (i == master_idx) continue; pf = mlx5_lag_pf(ldev, i); @@ -403,6 +421,69 @@ int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev) } EXPORT_SYMBOL(mlx5_lag_get_dev_seq); +/* seq 0 = master, then all remaining devices */ +static int mlx5_lag_get_dev_index_by_seq_all(struct mlx5_lag *ldev, int seq) +{ + int master_idx, i, num = 0; + + master_idx = mlx5_lag_get_master_idx(ldev); + + if (master_idx >= 0) { + if (seq == 0) + return master_idx; + num++; + } + + mlx5_lag_for_each(i, 0, ldev, MLX5_LAG_FILTER_ALL) { + if (i == master_idx) + continue; + if (num == seq) + return i; + num++; + } + return -ENOENT; +} + +/* From group POV, port-marked entry is the lag master */ +static int mlx5_lag_get_dev_index_by_seq_group(struct mlx5_lag *ldev, int seq, + u32 group_id) +{ + int i, num = 0; + + mlx5_lag_for_each(i, 0, ldev, group_id) { + if (xa_get_mark(&ldev->pfs, i, MLX5_LAG_XA_MARK_PORT)) { + if (seq == 0) + return i; + num++; + break; + } + } + + mlx5_lag_for_each(i, 0, ldev, group_id) { + if (xa_get_mark(&ldev->pfs, i, MLX5_LAG_XA_MARK_PORT)) + continue; + if (num == seq) + return i; + num++; + } + return -ENOENT; +} + +int mlx5_lag_get_dev_index_by_seq_filter(struct mlx5_lag *ldev, int seq, + u32 filter) +{ + if (!ldev) + return -ENOENT; + + if (!filter || filter == MLX5_LAG_FILTER_PORTS) + return mlx5_lag_get_dev_index_by_seq(ldev, seq); + + if (filter == MLX5_LAG_FILTER_ALL) + return mlx5_lag_get_dev_index_by_seq_all(ldev, seq); + + return mlx5_lag_get_dev_index_by_seq_group(ldev, seq, filter); +} + /* Devcom events for LAG master marking */ #define LAG_DEVCOM_PAIR (0) #define LAG_DEVCOM_UNPAIR (1) @@ -512,6 +593,14 @@ static bool __mlx5_lag_is_sriov(struct mlx5_lag *ldev) return ldev->mode == MLX5_LAG_MODE_SRIOV; } +static bool __mlx5_lag_is_sd_active(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev) +{ + struct lag_func *pf = mlx5_lag_pf_by_dev(ldev, dev); + + return pf && pf->sd_fdb_active; +} + /* Create a mapping between steering slots and active ports. * As we have ldev->buckets slots per port first assume the native * mapping should be used. @@ -817,43 +906,6 @@ char *mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags) } } -static int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev) -{ - int master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); - struct mlx5_eswitch *master_esw; - struct mlx5_core_dev *dev0; - int i, j; - int err; - - if (master_idx < 0) - return -EINVAL; - - dev0 = mlx5_lag_pf(ldev, master_idx)->dev; - master_esw = dev0->priv.eswitch; - mlx5_ldev_for_each(i, 0, ldev) { - struct mlx5_eswitch *slave_esw; - - if (i == master_idx) - continue; - - slave_esw = mlx5_lag_pf(ldev, i)->dev->priv.eswitch; - - err = mlx5_eswitch_offloads_single_fdb_add_one(master_esw, - slave_esw, ldev->ports); - if (err) - goto err; - } - return 0; -err: - mlx5_ldev_for_each_reverse(j, i, 0, ldev) { - if (j == master_idx) - continue; - mlx5_eswitch_offloads_single_fdb_del_one(master_esw, - mlx5_lag_pf(ldev, j)->dev->priv.eswitch); - } - return err; -} - static int mlx5_create_lag(struct mlx5_lag *ldev, struct lag_tracker *tracker, enum mlx5_lag_mode mode, @@ -964,27 +1016,19 @@ int mlx5_deactivate_lag(struct mlx5_lag *ldev) u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {}; bool roce_lag = __mlx5_lag_is_roce(ldev); unsigned long flags = ldev->mode_flags; - struct mlx5_eswitch *master_esw; struct mlx5_core_dev *dev0; int err; - int i; if (master_idx < 0) return -EINVAL; dev0 = mlx5_lag_pf(ldev, master_idx)->dev; - master_esw = dev0->priv.eswitch; ldev->mode = MLX5_LAG_MODE_NONE; ldev->mode_flags = 0; mlx5_lag_mp_reset(ldev); if (test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags)) { - mlx5_ldev_for_each(i, 0, ldev) { - if (i == master_idx) - continue; - mlx5_eswitch_offloads_single_fdb_del_one(master_esw, - mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - } + mlx5_lag_destroy_single_fdb(ldev); clear_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags); } @@ -1063,34 +1107,159 @@ bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) return true; } -void mlx5_lag_add_devices(struct mlx5_lag *ldev) +static void mlx5_lag_assert_locked_transition(struct mlx5_lag *ldev, u32 filter) { + struct mlx5_devcom_comp_dev *devcom = NULL; struct lag_func *pf; int i; - mlx5_ldev_for_each(i, 0, ldev) { + lockdep_assert_held(&ldev->lock); + + i = mlx5_get_next_lag_func(ldev, 0, filter); + if (i < MLX5_MAX_PORTS) { + pf = mlx5_lag_pf(ldev, i); + if (filter == MLX5_LAG_FILTER_PORTS || + filter == MLX5_LAG_FILTER_ALL) + devcom = pf->dev->priv.hca_devcom_comp; + else + devcom = mlx5_sd_get_devcom(pf->dev); + } + mlx5_devcom_comp_assert_locked(devcom); +} + +static void mlx5_lag_drop_lock_for_reps(struct mlx5_lag *ldev, u32 filter) +{ + mlx5_lag_assert_locked_transition(ldev, filter); + + /* Keep PF membership stable while ldev->lock is dropped. Device add + * and remove paths observe mode_changes_in_progress and retry. + */ + ldev->mode_changes_in_progress++; + mutex_unlock(&ldev->lock); +} + +static void mlx5_lag_retake_lock_after_reps(struct mlx5_lag *ldev) +{ + mutex_lock(&ldev->lock); + ldev->mode_changes_in_progress--; +} + +void mlx5_lag_rescan_dev_locked(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev, + bool enable) +{ + if (dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) + return; + + if (enable) + dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + else + dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + + /* Auxiliary bus probe/remove can register or unregister representor + * callbacks and take reps_lock. Drop ldev->lock so the only ordering + * remains reps_lock -> ldev->lock from representor callbacks. + */ + mlx5_lag_drop_lock_for_reps(ldev, mlx5_lag_get_filter(ldev, dev)); + mlx5_rescan_drivers_locked(dev); + mlx5_lag_retake_lock_after_reps(ldev); +} + +static void mlx5_lag_rescan_devices_locked_filter(struct mlx5_lag *ldev, + bool enable, u32 filter) +{ + struct mlx5_core_dev *devs[MLX5_MAX_PORTS]; + struct lag_func *pf; + int num_devs = 0; + int i; + + mlx5_lag_assert_locked_transition(ldev, filter); + + mlx5_lag_for_each(i, 0, ldev, filter) { pf = mlx5_lag_pf(ldev, i); if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) continue; - pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(pf->dev); + if (enable) + pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + else + pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + devs[num_devs++] = pf->dev; } + + mlx5_lag_drop_lock_for_reps(ldev, filter); + for (i = 0; i < num_devs; i++) + mlx5_rescan_drivers_locked(devs[i]); + mlx5_lag_retake_lock_after_reps(ldev); +} + +void mlx5_lag_add_devices_filter(struct mlx5_lag *ldev, u32 filter) +{ + mlx5_lag_rescan_devices_locked_filter(ldev, true, filter); +} + +void mlx5_lag_add_devices(struct mlx5_lag *ldev) +{ + mlx5_lag_add_devices_filter(ldev, MLX5_LAG_FILTER_PORTS); +} + +void mlx5_lag_remove_devices_filter(struct mlx5_lag *ldev, u32 filter) +{ + mlx5_lag_rescan_devices_locked_filter(ldev, false, filter); } void mlx5_lag_remove_devices(struct mlx5_lag *ldev) +{ + mlx5_lag_remove_devices_filter(ldev, MLX5_LAG_FILTER_PORTS); +} + +static int mlx5_lag_reload_ib_reps_unlocked(struct mlx5_lag *ldev, u32 flags, + u32 filter, bool cont_on_fail) { struct lag_func *pf; + int ret; int i; - mlx5_ldev_for_each(i, 0, ldev) { + mlx5_lag_for_each(i, 0, ldev, filter) { pf = mlx5_lag_pf(ldev, i); - if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) - continue; - - pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(pf->dev); + if (!(pf->dev->priv.flags & flags)) { + struct mlx5_eswitch *esw; + + esw = pf->dev->priv.eswitch; + mlx5_esw_reps_block(esw); + ret = mlx5_eswitch_reload_ib_reps(esw); + mlx5_esw_reps_unblock(esw); + if (ret && !cont_on_fail) + return ret; + } } + + return 0; +} + +static int mlx5_lag_reload_ib_reps(struct mlx5_lag *ldev, u32 flags, + u32 filter, bool cont_on_fail) +{ + int ret; + + /* The HCA devcom component lock serializes LAG mode transitions while + * ldev->lock is dropped here. Dropping ldev->lock is required because + * the reload takes the per-E-Switch reps_lock, and representor + * load/unload callbacks can re-enter LAG netdev add/remove and take + * ldev->lock. Keep the ordering reps_lock -> ldev->lock. + */ + mlx5_lag_drop_lock_for_reps(ldev, filter); + ret = mlx5_lag_reload_ib_reps_unlocked(ldev, flags, filter, + cont_on_fail); + mlx5_lag_retake_lock_after_reps(ldev); + + return ret; +} + +int mlx5_lag_reload_ib_reps_from_locked(struct mlx5_lag *ldev, u32 flags, + u32 filter, bool cont_on_fail) +{ + return mlx5_lag_reload_ib_reps(ldev, flags, filter, cont_on_fail); } void mlx5_disable_lag(struct mlx5_lag *ldev) @@ -1105,16 +1274,16 @@ void mlx5_disable_lag(struct mlx5_lag *ldev) if (idx < 0) return; + if (shared_fdb) { + mlx5_lag_shared_fdb_destroy(ldev, 0); + return; + } + dev0 = mlx5_lag_pf(ldev, idx)->dev; roce_lag = __mlx5_lag_is_roce(ldev); - if (shared_fdb) { - mlx5_lag_remove_devices(ldev); - } else if (roce_lag) { - if (!(dev0->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) { - dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - } + if (roce_lag) { + mlx5_lag_rescan_dev_locked(ldev, dev0, false); mlx5_ldev_for_each(i, 0, ldev) { if (i == idx) continue; @@ -1126,49 +1295,8 @@ void mlx5_disable_lag(struct mlx5_lag *ldev) if (err) return; - if (shared_fdb || roce_lag) + if (roce_lag) mlx5_lag_add_devices(ldev); - - if (shared_fdb) - mlx5_ldev_for_each(i, 0, ldev) - if (!(mlx5_lag_pf(ldev, i)->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); -} - -bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev) -{ - struct mlx5_core_dev *dev; - bool ret = false; - int idx; - int i; - - idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); - if (idx < 0) - return false; - - mlx5_ldev_for_each(i, 0, ldev) { - if (i == idx) - continue; - dev = mlx5_lag_pf(ldev, i)->dev; - if (is_mdev_switchdev_mode(dev) && - mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) && - MLX5_CAP_GEN(dev, lag_native_fdb_selection) && - MLX5_CAP_ESW(dev, root_ft_on_other_esw) && - mlx5_eswitch_get_npeers(dev->priv.eswitch) == - MLX5_CAP_GEN(dev, num_lag_ports) - 1) - continue; - return false; - } - - dev = mlx5_lag_pf(ldev, idx)->dev; - if (is_mdev_switchdev_mode(dev) && - mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) && - mlx5_esw_offloads_devcom_is_ready(dev->priv.eswitch) && - MLX5_CAP_ESW(dev, esw_shared_ingress_acl) && - mlx5_eswitch_get_npeers(dev->priv.eswitch) == MLX5_CAP_GEN(dev, num_lag_ports) - 1) - ret = true; - - return ret; } static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev) @@ -1383,58 +1511,38 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) roce_lag = mlx5_lag_is_roce_lag(ldev); - if (shared_fdb || roce_lag) - mlx5_lag_remove_devices(ldev); - - err = mlx5_activate_lag(ldev, &tracker, - roce_lag ? MLX5_LAG_MODE_ROCE : - MLX5_LAG_MODE_SRIOV, - shared_fdb); - if (err) { - if (shared_fdb || roce_lag) - mlx5_lag_add_devices(ldev); - if (shared_fdb) { - mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - } - - return; - } - - if (roce_lag) { - struct mlx5_core_dev *dev; - - dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - mlx5_ldev_for_each(i, 0, ldev) { - if (i == idx) - continue; - dev = mlx5_lag_pf(ldev, i)->dev; - if (mlx5_get_roce_state(dev)) - mlx5_nic_vport_enable_roce(dev); - } - } else if (shared_fdb) { - int i; - - dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - - mlx5_ldev_for_each(i, 0, ldev) { - err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - if (err) - break; - } + if (shared_fdb) { + err = mlx5_lag_shared_fdb_create(ldev, &tracker, + MLX5_LAG_MODE_SRIOV, + 0); + if (err) + return; + } else { + if (roce_lag) + mlx5_lag_remove_devices(ldev); + err = mlx5_activate_lag(ldev, &tracker, + roce_lag ? MLX5_LAG_MODE_ROCE : + MLX5_LAG_MODE_SRIOV, + false); if (err) { - dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - mlx5_deactivate_lag(ldev); - mlx5_lag_add_devices(ldev); - mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - mlx5_core_err(dev0, "Failed to enable lag\n"); + if (roce_lag) + mlx5_lag_add_devices(ldev); return; } + + if (roce_lag) { + struct mlx5_core_dev *dev; + + mlx5_lag_rescan_dev_locked(ldev, dev0, true); + mlx5_ldev_for_each(i, 0, ldev) { + if (i == idx) + continue; + dev = mlx5_lag_pf(ldev, i)->dev; + if (mlx5_get_roce_state(dev)) + mlx5_nic_vport_enable_roce(dev); + } + } } if (tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) { ndev = mlx5_lag_active_backup_get_netdev(dev0); @@ -1446,7 +1554,8 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) ndev); dev_put(ndev); } - mlx5_lag_set_vports_agg_speed(ldev); + if (!shared_fdb) + mlx5_lag_set_vports_agg_speed(ldev); } else if (mlx5_lag_should_modify_lag(ldev, do_bond)) { mlx5_modify_lag(ldev, &tracker); mlx5_lag_set_vports_agg_speed(ldev); @@ -1467,7 +1576,7 @@ struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev) int i; mutex_lock(&ldev->lock); - i = mlx5_get_next_ldev_func(ldev, 0); + i = mlx5_get_next_lag_func(ldev, 0, MLX5_LAG_FILTER_PORTS); if (i < MLX5_MAX_PORTS) { pf = mlx5_lag_pf(ldev, i); devcom = pf->dev->priv.hca_devcom_comp; @@ -1950,8 +2059,9 @@ static void mlx5_ldev_remove_netdev(struct mlx5_lag *ldev, spin_unlock_irqrestore(&lag_lock, flags); } -static int mlx5_ldev_add_mdev(struct mlx5_lag *ldev, - struct mlx5_core_dev *dev) +int mlx5_ldev_add_mdev(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev, + u32 group_id) { struct lag_func *pf; u32 idx; @@ -1970,8 +2080,14 @@ static int mlx5_ldev_add_mdev(struct mlx5_lag *ldev, pf->idx = idx; pf->dev = dev; + pf->group_id = group_id; dev->priv.lag = ldev; + if (group_id) + return 0; + + xa_set_mark(&ldev->pfs, idx, MLX5_LAG_XA_MARK_PORT); + MLX5_NB_INIT(&pf->port_change_nb, mlx5_lag_mpesw_port_change_event, PORT_CHANGE); mlx5_eq_notifier_register(dev, &pf->port_change_nb); @@ -1979,13 +2095,13 @@ static int mlx5_ldev_add_mdev(struct mlx5_lag *ldev, return 0; } -static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, - struct mlx5_core_dev *dev) +void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev) { struct lag_func *pf; int i; - mlx5_ldev_for_each(i, 0, ldev) { + mlx5_lag_for_each(i, 0, ldev, MLX5_LAG_FILTER_ALL) { pf = mlx5_lag_pf(ldev, i); if (pf->dev == dev) break; @@ -2020,7 +2136,7 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) mlx5_core_err(dev, "Failed to alloc lag dev\n"); return 0; } - err = mlx5_ldev_add_mdev(ldev, dev); + err = mlx5_ldev_add_mdev(ldev, dev, 0); if (err) { mlx5_core_err(dev, "Failed to add mdev to lag dev\n"); mlx5_ldev_put(ldev); @@ -2035,7 +2151,7 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) return -EAGAIN; } mlx5_ldev_get(ldev); - err = mlx5_ldev_add_mdev(ldev, dev); + err = mlx5_ldev_add_mdev(ldev, dev, 0); if (err) { mlx5_ldev_put(ldev); mutex_unlock(&ldev->lock); @@ -2172,27 +2288,47 @@ void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, mlx5_queue_bond_work(ldev, 0); } -int mlx5_get_pre_ldev_func(struct mlx5_lag *ldev, int start_idx, int end_idx) +int mlx5_get_pre_lag_func(struct mlx5_lag *ldev, int start_idx, int end_idx, + u32 filter) { struct lag_func *pf; int i; for (i = start_idx; i >= end_idx; i--) { pf = xa_load(&ldev->pfs, i); - if (pf && pf->dev) + if (!pf || !pf->dev) + continue; + if (filter == MLX5_LAG_FILTER_PORTS) { + if (xa_get_mark(&ldev->pfs, i, MLX5_LAG_XA_MARK_PORT)) + return i; + } else if (filter == MLX5_LAG_FILTER_ALL || + filter == pf->group_id) { return i; + } } return -1; } -int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx) +int mlx5_get_next_lag_func(struct mlx5_lag *ldev, int start_idx, u32 filter) { struct lag_func *pf; unsigned long idx; - xa_for_each_start(&ldev->pfs, idx, pf, start_idx) - if (pf->dev) + if (filter == MLX5_LAG_FILTER_PORTS) { + xa_for_each_marked_start(&ldev->pfs, idx, pf, + MLX5_LAG_XA_MARK_PORT, start_idx) + if (pf->dev) + return idx; + return MLX5_MAX_PORTS; + } + + xa_for_each_start(&ldev->pfs, idx, pf, start_idx) { + if (!pf->dev) + continue; + if (filter == MLX5_LAG_FILTER_ALL || + filter == pf->group_id) return idx; + } return MLX5_MAX_PORTS; } @@ -2219,7 +2355,8 @@ bool mlx5_lag_is_active(struct mlx5_core_dev *dev) spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); - res = ldev && __mlx5_lag_is_active(ldev); + res = ldev && (__mlx5_lag_is_active(ldev) || + __mlx5_lag_is_sd_active(ldev, dev)); spin_unlock_irqrestore(&lag_lock, flags); return res; @@ -2252,10 +2389,17 @@ bool mlx5_lag_is_master(struct mlx5_core_dev *dev) spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); - idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); - if (ldev && __mlx5_lag_is_active(ldev) && idx >= 0) { - pf = mlx5_lag_pf(ldev, idx); - res = pf && dev == pf->dev; + if (ldev) { + u32 filter; + + filter = mlx5_lag_get_filter(ldev, dev); + idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1, + filter); + if ((__mlx5_lag_is_active(ldev) || + __mlx5_lag_is_sd_active(ldev, dev)) && idx >= 0) { + pf = mlx5_lag_pf(ldev, idx); + res = pf && dev == pf->dev; + } } spin_unlock_irqrestore(&lag_lock, flags); @@ -2282,11 +2426,16 @@ bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; unsigned long flags; - bool res; + bool res = false; spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); - res = ldev && test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags); + if (ldev) { + res = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, + &ldev->mode_flags); + if (__mlx5_lag_is_sd(ldev, dev) && !__mlx5_lag_is_active(ldev)) + res = __mlx5_lag_is_sd_active(ldev, dev); + } spin_unlock_irqrestore(&lag_lock, flags); return res; @@ -2387,7 +2536,7 @@ struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int if (*i == MLX5_MAX_PORTS) goto unlock; - mlx5_ldev_for_each(idx, *i, ldev) { + mlx5_lag_for_each(idx, *i, ldev, mlx5_lag_get_filter(ldev, dev)) { pf = mlx5_lag_pf(ldev, idx); if (pf->dev != dev) break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h index 6c911374f409b..82b82bdb16dd3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h @@ -15,6 +15,13 @@ * Note: XA_MARK_0 is reserved by XA_FLAGS_ALLOC for free-slot tracking. */ #define MLX5_LAG_XA_MARK_MASTER XA_MARK_1 +/* XArray mark for port-level entries (excludes SD secondaries) */ +#define MLX5_LAG_XA_MARK_PORT XA_MARK_2 + +/* Like xa_for_each_marked but starting from a given index */ +#define xa_for_each_marked_start(xa, index, entry, filter, start) \ + for (index = start, entry = xa_find(xa, &index, ULONG_MAX, filter); \ + entry; entry = xa_find_after(xa, &index, ULONG_MAX, filter)) #include "mlx5_core.h" #include "mp.h" @@ -50,6 +57,8 @@ struct lag_func { bool has_drop; unsigned int idx; /* xarray index assigned by LAG */ struct mlx5_nb port_change_nb; + u32 group_id; /* SD group ID, 0 = not SD */ + bool sd_fdb_active; /* set on all SD group members */ }; /* Used for collection of netdev event info. */ @@ -125,6 +134,28 @@ mlx5_lag_pf_by_dev_idx(struct mlx5_lag *ldev, int dev_idx) return NULL; } +/* Find lag_func by mlx5_core_dev pointer */ +static inline struct lag_func * +mlx5_lag_pf_by_dev(struct mlx5_lag *ldev, struct mlx5_core_dev *dev) +{ + struct lag_func *pf; + unsigned long idx; + + xa_for_each(&ldev->pfs, idx, pf) { + if (pf->dev == dev) + return pf; + } + return NULL; +} + +static inline bool +__mlx5_lag_is_sd(struct mlx5_lag *ldev, struct mlx5_core_dev *dev) +{ + struct lag_func *pf = mlx5_lag_pf_by_dev(ldev, dev); + + return pf && pf->group_id != 0; +} + static inline bool __mlx5_lag_is_active(struct mlx5_lag *ldev) { @@ -137,7 +168,39 @@ mlx5_lag_is_ready(struct mlx5_lag *ldev) return test_bit(MLX5_LAG_FLAG_NDEVS_READY, &ldev->state_flags); } +#ifdef CONFIG_MLX5_ESWITCH +int mlx5_lag_shared_fdb_create(struct mlx5_lag *ldev, + struct lag_tracker *tracker, + enum mlx5_lag_mode mode, + u32 group_id); +void mlx5_lag_shared_fdb_destroy(struct mlx5_lag *ldev, u32 group_id); +int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev); +void mlx5_lag_destroy_single_fdb(struct mlx5_lag *ldev); bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev); +bool mlx5_lag_shared_fdb_supported_filter(struct mlx5_lag *ldev, u32 filter); +#else +static inline int mlx5_lag_shared_fdb_create(struct mlx5_lag *ldev, + struct lag_tracker *tracker, + enum mlx5_lag_mode mode, + u32 group_id) +{ + return -EOPNOTSUPP; +} + +static inline void mlx5_lag_shared_fdb_destroy(struct mlx5_lag *ldev, + u32 group_id) {} + +static inline int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev) +{ + return -EOPNOTSUPP; +} + +static inline void mlx5_lag_destroy_single_fdb(struct mlx5_lag *ldev) {} +static inline bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev) +{ + return false; +} +#endif bool mlx5_lag_check_prereq(struct mlx5_lag *ldev); int mlx5_lag_demux_init(struct mlx5_core_dev *dev, struct mlx5_flow_table_attr *ft_attr); @@ -162,8 +225,13 @@ void mlx5_ldev_add_debugfs(struct mlx5_core_dev *dev); void mlx5_ldev_remove_debugfs(struct dentry *dbg); void mlx5_disable_lag(struct mlx5_lag *ldev); void mlx5_lag_remove_devices(struct mlx5_lag *ldev); +void mlx5_lag_remove_devices_filter(struct mlx5_lag *ldev, u32 filter); int mlx5_deactivate_lag(struct mlx5_lag *ldev); void mlx5_lag_add_devices(struct mlx5_lag *ldev); +void mlx5_lag_rescan_dev_locked(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev, + bool enable); +void mlx5_lag_add_devices_filter(struct mlx5_lag *ldev, u32 filter); struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev); #ifdef CONFIG_MLX5_ESWITCH @@ -185,18 +253,40 @@ static inline bool mlx5_lag_is_supported(struct mlx5_core_dev *dev) return true; } -#define mlx5_ldev_for_each(i, start_index, ldev) \ - for (int tmp = start_index; tmp = mlx5_get_next_ldev_func(ldev, tmp), \ +/* Iterator filter constants for mlx5_lag_for_each() */ +#define MLX5_LAG_FILTER_PORTS 0 /* iterate ports only (XA_MARK_PORT) */ +#define MLX5_LAG_FILTER_ALL U32_MAX /* iterate ALL devices */ +/* any other value = iterate devices with that specific group_id */ + +#define mlx5_lag_for_each(i, start_index, ldev, filter) \ + for (int tmp = start_index; \ + tmp = mlx5_get_next_lag_func(ldev, tmp, filter), \ i = tmp, tmp < MLX5_MAX_PORTS; tmp++) -#define mlx5_ldev_for_each_reverse(i, start_index, end_index, ldev) \ +#define mlx5_lag_for_each_reverse(i, start_index, end_index, ldev, filter) \ for (int tmp = start_index, tmp1 = end_index; \ - tmp = mlx5_get_pre_ldev_func(ldev, tmp, tmp1), \ + tmp = mlx5_get_pre_lag_func(ldev, tmp, tmp1, filter), \ i = tmp, tmp >= tmp1; tmp--) -int mlx5_get_pre_ldev_func(struct mlx5_lag *ldev, int start_idx, int end_idx); -int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx); +/* Convenience wrappers - keeps existing behavior */ +#define mlx5_ldev_for_each(i, start_index, ldev) \ + mlx5_lag_for_each(i, start_index, ldev, MLX5_LAG_FILTER_PORTS) + +#define mlx5_ldev_for_each_reverse(i, start_index, end_index, ldev) \ + mlx5_lag_for_each_reverse(i, start_index, end_index, ldev, \ + MLX5_LAG_FILTER_PORTS) + +int mlx5_get_pre_lag_func(struct mlx5_lag *ldev, int start_idx, int end_idx, + u32 filter); +int mlx5_get_next_lag_func(struct mlx5_lag *ldev, int start_idx, u32 filter); int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq); +int mlx5_lag_get_dev_index_by_seq_filter(struct mlx5_lag *ldev, int seq, + u32 filter); int mlx5_lag_num_devs(struct mlx5_lag *ldev); int mlx5_lag_num_netdevs(struct mlx5_lag *ldev); +int mlx5_lag_reload_ib_reps_from_locked(struct mlx5_lag *ldev, u32 flags, + u32 filter, bool cont_on_fail); +int mlx5_ldev_add_mdev(struct mlx5_lag *ldev, struct mlx5_core_dev *dev, + u32 group_id); +void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, struct mlx5_core_dev *dev); #endif /* __MLX5_LAG_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index 5eea12a6887ac..97334b563c242 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -70,7 +70,6 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_core_dev *dev0; int err; - int i; if (ldev->mode == MLX5_LAG_MODE_MPESW) return 0; @@ -86,50 +85,29 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table) || !MLX5_CAP_GEN(dev0, create_lag_when_not_master_up) || !mlx5_lag_check_prereq(ldev) || - !mlx5_lag_shared_fdb_supported(ldev)) + !mlx5_lag_shared_fdb_supported_filter(ldev, MLX5_LAG_FILTER_ALL)) return -EOPNOTSUPP; err = mlx5_mpesw_metadata_set(ldev); if (err) return err; - mlx5_lag_remove_devices(ldev); - - err = mlx5_activate_lag(ldev, NULL, MLX5_LAG_MODE_MPESW, true); + err = mlx5_lag_shared_fdb_create(ldev, NULL, MLX5_LAG_MODE_MPESW, + MLX5_LAG_FILTER_ALL); if (err) { mlx5_core_warn(dev0, "Failed to create LAG in MPESW mode (%d)\n", err); - goto err_add_devices; - } - - dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - mlx5_ldev_for_each(i, 0, ldev) { - err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - if (err) - goto err_rescan_drivers; + mlx5_mpesw_metadata_cleanup(ldev); + return err; } - mlx5_lag_set_vports_agg_speed(ldev); - return 0; - -err_rescan_drivers: - dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - mlx5_deactivate_lag(ldev); -err_add_devices: - mlx5_lag_add_devices(ldev); - mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - mlx5_mpesw_metadata_cleanup(ldev); - return err; } void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) { if (ldev->mode == MLX5_LAG_MODE_MPESW) { mlx5_mpesw_metadata_cleanup(ldev); - mlx5_disable_lag(ldev); + mlx5_lag_shared_fdb_destroy(ldev, MLX5_LAG_FILTER_ALL); } } @@ -140,8 +118,10 @@ static void mlx5_mpesw_work(struct work_struct *work) struct mlx5_lag *ldev = mpesww->lag; devcom = mlx5_lag_get_devcom_comp(ldev); - if (!devcom) - return; + if (!devcom) { + mpesww->result = -ENODEV; + goto complete; + } mlx5_devcom_comp_lock(devcom); mutex_lock(&ldev->lock); @@ -157,6 +137,7 @@ static void mlx5_mpesw_work(struct work_struct *work) unlock: mutex_unlock(&ldev->lock); mlx5_devcom_comp_unlock(devcom); +complete: complete(&mpesww->comp); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c new file mode 100644 index 0000000000000..b72e8659a9686 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include +#include +#include +#include "mlx5_core.h" +#include "lag.h" +#include "eswitch.h" + +bool mlx5_lag_shared_fdb_supported_filter(struct mlx5_lag *ldev, u32 filter) +{ + int idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1, + filter); + struct mlx5_core_dev *dev0, *dev; + bool ret = false; + int i; + + if (idx < 0) + return false; + + dev0 = mlx5_lag_pf(ldev, idx)->dev; + mlx5_lag_for_each(i, 0, ldev, filter) { + if (i == idx) + continue; + dev = mlx5_lag_pf(ldev, i)->dev; + if (is_mdev_switchdev_mode(dev) && + mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) && + MLX5_CAP_GEN(dev, lag_native_fdb_selection) && + MLX5_CAP_ESW(dev, root_ft_on_other_esw) && + mlx5_eswitch_is_peer(dev0->priv.eswitch, dev->priv.eswitch)) + continue; + return false; + } + + if (is_mdev_switchdev_mode(dev0) && + mlx5_eswitch_vport_match_metadata_enabled(dev0->priv.eswitch) && + mlx5_esw_offloads_devcom_is_ready(dev0->priv.eswitch) && + MLX5_CAP_ESW(dev0, esw_shared_ingress_acl)) + ret = true; + + return ret; +} + +bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev) +{ + return mlx5_lag_shared_fdb_supported_filter(ldev, + MLX5_LAG_FILTER_PORTS); +} + +static int mlx5_lag_create_single_fdb_filter(struct mlx5_lag *ldev, u32 filter) +{ + int master_idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1, + filter); + struct mlx5_eswitch *master_esw; + struct mlx5_core_dev *dev0; + int i, j; + int err; + + if (master_idx < 0) + return -EINVAL; + + dev0 = mlx5_lag_pf(ldev, master_idx)->dev; + master_esw = dev0->priv.eswitch; + mlx5_lag_for_each(i, 0, ldev, filter) { + struct mlx5_eswitch *slave_esw; + + if (i == master_idx) + continue; + + slave_esw = mlx5_lag_pf(ldev, i)->dev->priv.eswitch; + + err = mlx5_eswitch_offloads_single_fdb_add_one(master_esw, + slave_esw, + ldev->ports); + if (err) + goto err; + } + return 0; +err: + mlx5_lag_for_each_reverse(j, i - 1, 0, ldev, filter) { + struct mlx5_eswitch *slave_esw; + + if (j == master_idx) + continue; + slave_esw = mlx5_lag_pf(ldev, j)->dev->priv.eswitch; + mlx5_eswitch_offloads_single_fdb_del_one(master_esw, slave_esw); + } + return err; +} + +static void mlx5_lag_destroy_single_fdb_filter(struct mlx5_lag *ldev, + u32 filter) +{ + int master_idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1, + filter); + struct mlx5_eswitch *master_esw; + struct mlx5_eswitch *peer_esw; + int i; + + if (master_idx < 0) + return; + + master_esw = mlx5_lag_pf(ldev, master_idx)->dev->priv.eswitch; + mlx5_lag_for_each(i, 0, ldev, filter) { + if (i == master_idx) + continue; + + peer_esw = mlx5_lag_pf(ldev, i)->dev->priv.eswitch; + mlx5_eswitch_offloads_single_fdb_del_one(master_esw, peer_esw); + } +} + +int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev) +{ + return mlx5_lag_create_single_fdb_filter(ldev, MLX5_LAG_FILTER_ALL); +} + +void mlx5_lag_destroy_single_fdb(struct mlx5_lag *ldev) +{ + mlx5_lag_destroy_single_fdb_filter(ldev, MLX5_LAG_FILTER_ALL); +} + +/** + * mlx5_lag_shared_fdb_create - Create shared FDB LAG + * @ldev: LAG device + * @tracker: LAG tracker (NULL for SD) + * @mode: LAG mode (unused for SD) + * @group_id: SD group ID; 0 (MLX5_LAG_FILTER_PORTS) for ports LAG; + * MLX5_LAG_FILTER_ALL for all-device (mpesw) LAG + * + * When group_id is 0 (MLX5_LAG_FILTER_PORTS) or MLX5_LAG_FILTER_ALL, + * activates a FW LAG with shared FDB. + * When group_id is a specific SD group ID, creates a software-only shared + * FDB scoped to that group (no FW LAG commands). + * + * Return: 0 on success, negative error code on failure. + */ +int mlx5_lag_shared_fdb_create(struct mlx5_lag *ldev, + struct lag_tracker *tracker, + enum mlx5_lag_mode mode, + u32 group_id) +{ + u32 filter = group_id ? group_id : MLX5_LAG_FILTER_PORTS; + int idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1, + filter); + struct mlx5_core_dev *dev0; + struct lag_func *pf; + int err; + int i; + + if (idx < 0) + return -EINVAL; + + dev0 = mlx5_lag_pf(ldev, idx)->dev; + + mlx5_lag_remove_devices_filter(ldev, filter); + + if (filter == MLX5_LAG_FILTER_PORTS || filter == MLX5_LAG_FILTER_ALL) { + err = mlx5_activate_lag(ldev, tracker, mode, true); + if (err) { + mlx5_core_warn(dev0, + "Failed to create LAG in shared FDB mode (%d)\n", + err); + goto err_add_devices; + } + } else { + err = mlx5_lag_create_single_fdb_filter(ldev, group_id); + if (err) { + mlx5_core_warn(dev0, + "Failed to create SD shared FDB (%d)\n", + err); + goto err_add_devices; + } + mlx5_lag_for_each(i, 0, ldev, filter) { + pf = mlx5_lag_pf(ldev, i); + pf->sd_fdb_active = true; + } + BLOCKING_INIT_NOTIFIER_HEAD(&dev0->priv.lag_nh); + } + + mlx5_lag_rescan_dev_locked(ldev, dev0, true); + err = mlx5_lag_reload_ib_reps_from_locked(ldev, 0, filter, false); + if (err) { + mlx5_core_err(dev0, "Failed to enable lag\n"); + goto err_rescan_drivers; + } + + if (filter == MLX5_LAG_FILTER_PORTS || filter == MLX5_LAG_FILTER_ALL) + mlx5_lag_set_vports_agg_speed(ldev); + return 0; + +err_rescan_drivers: + mlx5_lag_rescan_dev_locked(ldev, dev0, false); + if (filter == MLX5_LAG_FILTER_PORTS || filter == MLX5_LAG_FILTER_ALL) { + mlx5_deactivate_lag(ldev); + } else { + mlx5_lag_for_each(i, 0, ldev, filter) { + pf = mlx5_lag_pf(ldev, i); + pf->sd_fdb_active = false; + } + mlx5_lag_destroy_single_fdb_filter(ldev, group_id); + } +err_add_devices: + mlx5_lag_add_devices_filter(ldev, filter); + mlx5_lag_reload_ib_reps_from_locked(ldev, 0, filter, true); + return err; +} + +void mlx5_lag_shared_fdb_destroy(struct mlx5_lag *ldev, u32 group_id) +{ + u32 filter = group_id ? group_id : MLX5_LAG_FILTER_PORTS; + struct lag_func *pf; + int err; + int i; + + mlx5_lag_remove_devices_filter(ldev, filter); + + if (filter == MLX5_LAG_FILTER_PORTS || filter == MLX5_LAG_FILTER_ALL) { + err = mlx5_deactivate_lag(ldev); + if (err) + return; + } else { + mlx5_lag_for_each(i, 0, ldev, filter) { + pf = mlx5_lag_pf(ldev, i); + pf->sd_fdb_active = false; + } + mlx5_lag_destroy_single_fdb_filter(ldev, group_id); + } + + mlx5_lag_add_devices_filter(ldev, filter); + mlx5_lag_reload_ib_reps_from_locked(ldev, + MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV, + filter, true); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index 4b5ac2db55ce5..d40c53193ea80 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -3,6 +3,7 @@ #include #include +#include #include "lib/devcom.h" #include "lib/mlx5.h" #include "mlx5_core.h" @@ -438,3 +439,10 @@ int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom) return 0; return down_write_trylock(&devcom->comp->sem); } + +void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom) +{ + if (!devcom) + return; + lockdep_assert_held_write(&devcom->comp->sem); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index 91e5ae529d5c6..316052a85ca5c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -75,5 +75,6 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom); void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom); int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom); +void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom); #endif /* __LIB_MLX5_DEVCOM_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.c index d6dc7bce855e0..305752dab7bdf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.c @@ -190,7 +190,7 @@ mlx5_hv_vhca_control_agent_create(struct mlx5_hv_vhca *hv_vhca) return mlx5_hv_vhca_agent_create(hv_vhca, MLX5_HV_VHCA_AGENT_CONTROL, NULL, mlx5_hv_vhca_control_agent_invalidate, - NULL, NULL); + NULL, NULL, NULL); } static void mlx5_hv_vhca_control_agent_destroy(struct mlx5_hv_vhca_agent *agent) @@ -256,7 +256,8 @@ mlx5_hv_vhca_agent_create(struct mlx5_hv_vhca *hv_vhca, void (*invalidate)(struct mlx5_hv_vhca_agent*, u64 block_mask), void (*cleaup)(struct mlx5_hv_vhca_agent *agent), - void *priv) + void *priv, + struct mlx5_hv_vhca_agent **ctx_update) { struct mlx5_hv_vhca_agent *agent; @@ -284,6 +285,9 @@ mlx5_hv_vhca_agent_create(struct mlx5_hv_vhca *hv_vhca, agent->invalidate = invalidate; agent->cleanup = cleaup; + if (ctx_update) + WRITE_ONCE(*ctx_update, agent); + mutex_lock(&hv_vhca->agents_lock); hv_vhca->agents[type] = agent; mutex_unlock(&hv_vhca->agents_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.h index f240ffe5116c5..8b3974cf0ee49 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.h @@ -43,7 +43,8 @@ mlx5_hv_vhca_agent_create(struct mlx5_hv_vhca *hv_vhca, void (*invalidate)(struct mlx5_hv_vhca_agent*, u64 block_mask), void (*cleanup)(struct mlx5_hv_vhca_agent *agent), - void *context); + void *context, + struct mlx5_hv_vhca_agent **ctx_update); void mlx5_hv_vhca_agent_destroy(struct mlx5_hv_vhca_agent *agent); int mlx5_hv_vhca_agent_write(struct mlx5_hv_vhca_agent *agent, @@ -84,7 +85,8 @@ mlx5_hv_vhca_agent_create(struct mlx5_hv_vhca *hv_vhca, void (*invalidate)(struct mlx5_hv_vhca_agent*, u64 block_mask), void (*cleanup)(struct mlx5_hv_vhca_agent *agent), - void *context) + void *context, + struct mlx5_hv_vhca_agent **ctx_update) { return NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c index 4571c56ec3c9b..97f6097d4c705 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c @@ -176,7 +176,8 @@ void mlx5_tun_entropy_refcount_dec(struct mlx5_tun_entropy *tun_entropy, int reformat_type) { mutex_lock(&tun_entropy->lock); - if (reformat_type == MLX5_REFORMAT_TYPE_L2_TO_VXLAN) + if (reformat_type == MLX5_REFORMAT_TYPE_L2_TO_VXLAN || + reformat_type == MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL) tun_entropy->num_enabling_entries--; else if (reformat_type == MLX5_REFORMAT_TYPE_L2_TO_NVGRE && --tun_entropy->num_disabling_entries == 0) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c index 6e199161b0083..bbd77ae11e848 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c @@ -57,6 +57,16 @@ static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev) return sd->primary ? dev : sd->primary_dev; } +struct mlx5_devcom_comp_dev *mlx5_sd_get_devcom(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + if (!sd) + return NULL; + + return sd->devcom; +} + struct mlx5_core_dev * mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h index 9bfd5b9756b50..2ab259095d7ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h @@ -21,6 +21,16 @@ void mlx5_sd_put_adev(struct auxiliary_device *actual_adev, int mlx5_sd_init(struct mlx5_core_dev *dev); void mlx5_sd_cleanup(struct mlx5_core_dev *dev); +#ifdef CONFIG_MLX5_CORE_EN +struct mlx5_devcom_comp_dev *mlx5_sd_get_devcom(struct mlx5_core_dev *dev); +#else +static inline struct mlx5_devcom_comp_dev * +mlx5_sd_get_devcom(struct mlx5_core_dev *dev) +{ + return NULL; +} +#endif + #define mlx5_sd_for_each_dev_from_to(i, primary, ix_from, to, pos) \ for (i = ix_from; \ (pos = mlx5_sd_primary_get_peer(primary, i)) && pos != (to); i++) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c index 997be91f0a135..7cedc348790db 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c @@ -175,6 +175,7 @@ int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index) if (refcount_dec_and_test(&idx_data->usecount)) { xa_erase(&st->idx_xa, st_index); + kfree(idx_data); /* We leave PCI config space as was before, no mkey will refer to it */ } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 74827e8ca1255..37af619e5e045 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -527,7 +527,6 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) { struct mlx5_profile *prof = &dev->profile; void *set_hca_cap; - int max_uc_list; int err; err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); @@ -610,10 +609,13 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) MLX5_SET(cmd_hca_cap, set_hca_cap, roce, mlx5_is_roce_on(dev)); - max_uc_list = max_uc_list_get_devlink_param(dev); - if (max_uc_list > 0) - MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_current_uc_list, - ilog2(max_uc_list)); + if (MLX5_CAP_GEN_MAX(dev, log_max_current_uc_list)) { + int max_uc_list = max_uc_list_get_devlink_param(dev); + + if (max_uc_list > 0) + MLX5_SET(cmd_hca_cap, set_hca_cap, + log_max_current_uc_list, ilog2(max_uc_list)); + } /* enable absolute native port num */ if (MLX5_CAP_GEN_MAX(dev, abs_native_port_num)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c index 6dcd9c2a78aa8..e60435eaebc56 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c @@ -205,6 +205,7 @@ static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher) ret = mlx5hws_matcher_resize_set_target(old_matcher, new_matcher); if (ret) { mlx5hws_err(ctx, "Rehash error: failed setting resize target\n"); + mlx5hws_matcher_destroy(new_matcher); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 3a65420fa1ad1..fed708c173328 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -4360,7 +4360,7 @@ static int mlxsw_sp_port_lag_join(struct mlxsw_sp_port *mlxsw_sp_port, lag_id = lag->lag_id; err = mlxsw_sp_port_lag_index_get(mlxsw_sp, lag_id, &port_index); if (err) - return err; + goto err_lag_uppers_bridge_join; err = mlxsw_sp_lag_uppers_bridge_join(mlxsw_sp_port, lag_dev, extack); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 7bd87d0547d8a..3d6fdbab05e01 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -1018,6 +1018,7 @@ static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, err_tree_replace: for (i--; i >= 0; i--) { + vr = &mlxsw_sp->router->vrs[i]; if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) continue; mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 0c6812fcf1851..283d25fae79e7 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -526,15 +526,10 @@ int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership) goto free_message; } - err = fbnic_mbx_map_tlv_msg(fbd, msg); - if (err) - goto free_message; - /* Initialize heartbeat, set last response to 1 second in the past * so that we will trigger a timeout if the firmware doesn't respond */ fbd->last_heartbeat_response = req_time - HZ; - fbd->last_heartbeat_request = req_time; /* Set prev_firmware_time to 0 to avoid triggering firmware crash @@ -542,6 +537,10 @@ int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership) */ fbd->prev_firmware_time = 0; + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + /* Set heartbeat detection based on if we are taking ownership */ fbd->fw_heartbeat_enabled = take_ownership; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index 4dea2bb58d2fc..44c046b253126 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -262,8 +262,11 @@ static int fbnic_set_mac(struct net_device *netdev, void *p) eth_hw_addr_set(netdev, addr->sa_data); - if (netif_running(netdev)) + if (netif_running(netdev)) { + netif_addr_lock_bh(netdev); __fbnic_set_rx_mode(fbn->fbd, &netdev->uc, &netdev->mc); + netif_addr_unlock_bh(netdev); + } return 0; } @@ -308,8 +311,10 @@ void fbnic_clear_rx_mode(struct fbnic_dev *fbd) /* Write updates to hardware */ fbnic_write_macda(fbd); + netif_addr_lock_bh(netdev); __dev_uc_unsync(netdev, NULL); __dev_mc_unsync(netdev, NULL); + netif_addr_unlock_bh(netdev); } static int fbnic_hwtstamp_get(struct net_device *netdev, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 7e85b480203c8..8b9bc9e8ea56c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -135,7 +135,9 @@ void fbnic_up(struct fbnic_net *fbn) fbnic_rss_reinit_hw(fbn->fbd, fbn); + netif_addr_lock_bh(fbn->netdev); __fbnic_set_rx_mode(fbn->fbd, &fbn->netdev->uc, &fbn->netdev->mc); + netif_addr_unlock_bh(fbn->netdev); /* Enable Tx/Rx processing */ fbnic_napi_enable(fbn); @@ -180,7 +182,9 @@ static int fbnic_fw_config_after_crash(struct fbnic_dev *fbd) } fbnic_rpc_reset_valid_entries(fbd); + netif_addr_lock_bh(fbd->netdev); __fbnic_set_rx_mode(fbd, &fbd->netdev->uc, &fbd->netdev->mc); + netif_addr_unlock_bh(fbd->netdev); return 0; } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index fe95b6f696463..bc0f38b6a2b2b 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -244,7 +244,9 @@ void fbnic_bmc_rpc_check(struct fbnic_dev *fbd) if (fbd->fw_cap.need_bmc_tcam_reinit) { fbnic_bmc_rpc_init(fbd); + netif_addr_lock_bh(fbd->netdev); __fbnic_set_rx_mode(fbd, &fbd->netdev->uc, &fbd->netdev->mc); + netif_addr_unlock_bh(fbd->netdev); fbd->fw_cap.need_bmc_tcam_reinit = false; } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 9cd85a0d0c3a7..401f8b8ae1cae 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -194,16 +194,18 @@ static bool fbnic_tx_tstamp(struct sk_buff *skb) static bool fbnic_tx_lso(struct fbnic_ring *ring, struct sk_buff *skb, - struct skb_shared_info *shinfo, __le64 *meta, - unsigned int *l2len, unsigned int *i3len) + __le64 *meta, unsigned int *l2len, unsigned int *i3len) { unsigned int l3_type, l4_type, l4len, hdrlen; + struct skb_shared_info *shinfo; unsigned char *l4hdr; __be16 payload_len; if (unlikely(skb_cow_head(skb, 0))) return true; + shinfo = skb_shinfo(skb); + if (shinfo->gso_type & SKB_GSO_PARTIAL) { l3_type = FBNIC_TWD_L3_TYPE_OTHER; } else if (!skb->encapsulation) { @@ -258,7 +260,6 @@ fbnic_tx_lso(struct fbnic_ring *ring, struct sk_buff *skb, static bool fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) { - struct skb_shared_info *shinfo = skb_shinfo(skb); unsigned int l2len, i3len; if (fbnic_tx_tstamp(skb)) @@ -273,8 +274,8 @@ fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_CSUM_OFFSET_MASK, skb->csum_offset / 2)); - if (shinfo->gso_size) { - if (fbnic_tx_lso(ring, skb, shinfo, meta, &l2len, &i3len)) + if (skb_is_gso(skb)) { + if (fbnic_tx_lso(ring, skb, meta, &l2len, &i3len)) return true; } else { *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_CSO); diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index ffac22883e497..95257c2bc451f 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -3542,8 +3542,8 @@ static int lan743x_hardware_init(struct lan743x_adapter *adapter, adapter->max_tx_channels = PCI11X1X_MAX_TX_CHANNELS; adapter->used_tx_channels = PCI11X1X_USED_TX_CHANNELS; adapter->max_vector_count = PCI11X1X_MAX_VECTOR_COUNT; - pci11x1x_strap_get_status(adapter); spin_lock_init(&adapter->eth_syslock_spinlock); + pci11x1x_strap_get_status(adapter); mutex_init(&adapter->sgmii_rw_lock); pci11x1x_set_rfe_rd_fifo_threshold(adapter); sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c index 72e3b189bac5a..eb28df80b2818 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c @@ -601,7 +601,6 @@ static void lan966x_vcap_admin_free(struct vcap_admin *admin) kfree(admin->cache.keystream); kfree(admin->cache.maskstream); kfree(admin->cache.actionstream); - mutex_destroy(&admin->lock); kfree(admin); } @@ -615,7 +614,7 @@ lan966x_vcap_admin_alloc(struct lan966x *lan966x, struct vcap_control *ctrl, if (!admin) return ERR_PTR(-ENOMEM); - mutex_init(&admin->lock); + admin->vctrl = ctrl; INIT_LIST_HEAD(&admin->list); INIT_LIST_HEAD(&admin->rules); INIT_LIST_HEAD(&admin->enabled); @@ -721,6 +720,7 @@ int lan966x_vcap_init(struct lan966x *lan966x) ctrl->ops = &lan966x_vcap_ops; INIT_LIST_HEAD(&ctrl->list); + mutex_init(&ctrl->lock); for (int i = 0; i < ARRAY_SIZE(lan966x_vcap_inst_cfg); ++i) { cfg = &lan966x_vcap_inst_cfg[i]; @@ -780,5 +780,6 @@ void lan966x_vcap_deinit(struct lan966x *lan966x) lan966x_vcap_admin_free(admin); } + mutex_destroy(&ctrl->lock); kfree(ctrl); } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c index 644458108dd20..dac4dd833127a 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c @@ -765,11 +765,13 @@ int sparx5_register_notifier_blocks(struct sparx5 *s5) sparx5_owq = alloc_ordered_workqueue("sparx5_order", 0); if (!sparx5_owq) { err = -ENOMEM; - goto err_switchdev_blocking_nb; + goto err_alloc_workqueue; } return 0; +err_alloc_workqueue: + unregister_switchdev_blocking_notifier(&s5->switchdev_blocking_nb); err_switchdev_blocking_nb: unregister_switchdev_notifier(&s5->switchdev_nb); err_switchdev_nb: diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c index 95b93e46a41dc..cf332de6bf736 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c @@ -1930,7 +1930,6 @@ static void sparx5_vcap_admin_free(struct vcap_admin *admin) { if (!admin) return; - mutex_destroy(&admin->lock); kfree(admin->cache.keystream); kfree(admin->cache.maskstream); kfree(admin->cache.actionstream); @@ -1950,7 +1949,7 @@ sparx5_vcap_admin_alloc(struct sparx5 *sparx5, struct vcap_control *ctrl, INIT_LIST_HEAD(&admin->list); INIT_LIST_HEAD(&admin->rules); INIT_LIST_HEAD(&admin->enabled); - mutex_init(&admin->lock); + admin->vctrl = ctrl; admin->vtype = cfg->vtype; admin->vinst = cfg->vinst; admin->ingress = cfg->ingress; @@ -2059,6 +2058,7 @@ int sparx5_vcap_init(struct sparx5 *sparx5) ctrl->ops = &sparx5_vcap_ops; INIT_LIST_HEAD(&ctrl->list); + mutex_init(&ctrl->lock); for (idx = 0; idx < ARRAY_SIZE(sparx5_vcap_inst_cfg); ++idx) { cfg = &consts->vcaps_cfg[idx]; admin = sparx5_vcap_admin_alloc(sparx5, ctrl, cfg); @@ -2097,5 +2097,6 @@ void sparx5_vcap_deinit(struct sparx5 *sparx5) list_del(&admin->list); sparx5_vcap_admin_free(admin); } + mutex_destroy(&ctrl->lock); kfree(ctrl); } diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api.c b/drivers/net/ethernet/microchip/vcap/vcap_api.c index 0fdb5e363badc..ff86cde11a32c 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api.c @@ -934,6 +934,16 @@ static bool vcap_rule_exists(struct vcap_control *vctrl, u32 id) return false; } +void vcap_lock(struct vcap_admin *admin) +{ + mutex_lock(&admin->vctrl->lock); +} + +void vcap_unlock(struct vcap_admin *admin) +{ + mutex_unlock(&admin->vctrl->lock); +} + /* Find a rule with a provided rule id return a locked vcap */ static struct vcap_rule_internal * vcap_get_locked_rule(struct vcap_control *vctrl, u32 id) @@ -943,11 +953,11 @@ vcap_get_locked_rule(struct vcap_control *vctrl, u32 id) /* Look for the rule id in all vcaps */ list_for_each_entry(admin, &vctrl->list, list) { - mutex_lock(&admin->lock); + vcap_lock(admin); list_for_each_entry(ri, &admin->rules, list) if (ri->data.id == id) return ri; - mutex_unlock(&admin->lock); + vcap_unlock(admin); } return NULL; } @@ -961,14 +971,14 @@ int vcap_lookup_rule_by_cookie(struct vcap_control *vctrl, u64 cookie) /* Look for the rule id in all vcaps */ list_for_each_entry(admin, &vctrl->list, list) { - mutex_lock(&admin->lock); + vcap_lock(admin); list_for_each_entry(ri, &admin->rules, list) { if (ri->data.cookie == cookie) { id = ri->data.id; break; } } - mutex_unlock(&admin->lock); + vcap_unlock(admin); if (id) return id; } @@ -985,11 +995,11 @@ int vcap_admin_rule_count(struct vcap_admin *admin, int cid) int count = 0; list_for_each_entry(elem, &admin->rules, list) { - mutex_lock(&admin->lock); + vcap_lock(admin); if (elem->data.vcap_chain_id >= min_cid && elem->data.vcap_chain_id < max_cid) ++count; - mutex_unlock(&admin->lock); + vcap_unlock(admin); } return count; } @@ -2266,7 +2276,7 @@ int vcap_add_rule(struct vcap_rule *rule) if (ret) return ret; /* Insert the new rule in the list of vcap rules */ - mutex_lock(&ri->admin->lock); + vcap_lock(ri->admin); vcap_rule_set_state(ri); ret = vcap_insert_rule(ri, &move); @@ -2302,7 +2312,7 @@ int vcap_add_rule(struct vcap_rule *rule) goto out; } out: - mutex_unlock(&ri->admin->lock); + vcap_unlock(ri->admin); return ret; } EXPORT_SYMBOL_GPL(vcap_add_rule); @@ -2330,7 +2340,7 @@ struct vcap_rule *vcap_alloc_rule(struct vcap_control *vctrl, if (vctrl->vcaps[admin->vtype].rows == 0) return ERR_PTR(-EINVAL); - mutex_lock(&admin->lock); + vcap_lock(admin); /* Check if a rule with this id already exists */ if (vcap_rule_exists(vctrl, id)) { err = -EINVAL; @@ -2369,13 +2379,13 @@ struct vcap_rule *vcap_alloc_rule(struct vcap_control *vctrl, goto out_free; } - mutex_unlock(&admin->lock); + vcap_unlock(admin); return (struct vcap_rule *)ri; out_free: kfree(ri); out_unlock: - mutex_unlock(&admin->lock); + vcap_unlock(admin); return ERR_PTR(err); } @@ -2446,7 +2456,7 @@ struct vcap_rule *vcap_get_rule(struct vcap_control *vctrl, u32 id) return ERR_PTR(-ENOENT); rule = vcap_decode_rule(elem); - mutex_unlock(&elem->admin->lock); + vcap_unlock(elem->admin); return rule; } EXPORT_SYMBOL_GPL(vcap_get_rule); @@ -2483,7 +2493,7 @@ int vcap_mod_rule(struct vcap_rule *rule) err = vcap_write_counter(ri, &ctr); out: - mutex_unlock(&ri->admin->lock); + vcap_unlock(ri->admin); return err; } EXPORT_SYMBOL_GPL(vcap_mod_rule); @@ -2570,7 +2580,7 @@ int vcap_del_rule(struct vcap_control *vctrl, struct net_device *ndev, u32 id) admin->last_used_addr = elem->addr; } - mutex_unlock(&admin->lock); + vcap_unlock(admin); return err; } EXPORT_SYMBOL_GPL(vcap_del_rule); @@ -2585,7 +2595,7 @@ int vcap_del_rules(struct vcap_control *vctrl, struct vcap_admin *admin) if (ret) return ret; - mutex_lock(&admin->lock); + vcap_lock(admin); list_for_each_entry_safe(ri, next_ri, &admin->rules, list) { vctrl->ops->init(ri->ndev, admin, ri->addr, ri->size); list_del(&ri->list); @@ -2598,7 +2608,7 @@ int vcap_del_rules(struct vcap_control *vctrl, struct vcap_admin *admin) list_del(&eport->list); kfree(eport); } - mutex_unlock(&admin->lock); + vcap_unlock(admin); return 0; } @@ -3016,7 +3026,7 @@ static int vcap_enable_rules(struct vcap_control *vctrl, continue; /* Found the admin, now find the offloadable rules */ - mutex_lock(&admin->lock); + vcap_lock(admin); list_for_each_entry(ri, &admin->rules, list) { /* Is the rule in the lookup defined by the chain */ if (!(ri->data.vcap_chain_id >= chain && @@ -3034,7 +3044,7 @@ static int vcap_enable_rules(struct vcap_control *vctrl, if (err) break; } - mutex_unlock(&admin->lock); + vcap_unlock(admin); if (err) break; } @@ -3074,7 +3084,7 @@ static int vcap_disable_rules(struct vcap_control *vctrl, continue; /* Found the admin, now find the rules on the chain */ - mutex_lock(&admin->lock); + vcap_lock(admin); list_for_each_entry(ri, &admin->rules, list) { if (ri->data.vcap_chain_id != chain) continue; @@ -3089,7 +3099,7 @@ static int vcap_disable_rules(struct vcap_control *vctrl, if (err) break; } - mutex_unlock(&admin->lock); + vcap_unlock(admin); if (err) break; } @@ -3133,9 +3143,9 @@ static int vcap_enable(struct vcap_control *vctrl, struct net_device *ndev, eport->cookie = cookie; eport->src_cid = src_cid; eport->dst_cid = dst_cid; - mutex_lock(&admin->lock); + vcap_lock(admin); list_add_tail(&eport->list, &admin->enabled); - mutex_unlock(&admin->lock); + vcap_unlock(admin); if (vcap_path_exist(vctrl, ndev, src_cid)) { /* Enable chained lookups */ @@ -3185,9 +3195,9 @@ static int vcap_disable(struct vcap_control *vctrl, struct net_device *ndev, dst_cid = vcap_get_next_chain(vctrl, ndev, dst_cid); } - mutex_lock(&found->lock); + vcap_lock(found); list_del(&eport->list); - mutex_unlock(&found->lock); + vcap_unlock(found); kfree(eport); return 0; } @@ -3270,9 +3280,9 @@ int vcap_rule_set_counter(struct vcap_rule *rule, struct vcap_counter *ctr) return -EINVAL; } - mutex_lock(&ri->admin->lock); + vcap_lock(ri->admin); err = vcap_write_counter(ri, ctr); - mutex_unlock(&ri->admin->lock); + vcap_unlock(ri->admin); return err; } @@ -3291,9 +3301,9 @@ int vcap_rule_get_counter(struct vcap_rule *rule, struct vcap_counter *ctr) return -EINVAL; } - mutex_lock(&ri->admin->lock); + vcap_lock(ri->admin); err = vcap_read_counter(ri, ctr); - mutex_unlock(&ri->admin->lock); + vcap_unlock(ri->admin); return err; } @@ -3395,7 +3405,7 @@ int vcap_get_rule_count_by_cookie(struct vcap_control *vctrl, /* Iterate all rules in each VCAP instance */ list_for_each_entry(admin, &vctrl->list, list) { - mutex_lock(&admin->lock); + vcap_lock(admin); list_for_each_entry(ri, &admin->rules, list) { if (ri->data.cookie != cookie) continue; @@ -3412,12 +3422,12 @@ int vcap_get_rule_count_by_cookie(struct vcap_control *vctrl, if (err) goto unlock; } - mutex_unlock(&admin->lock); + vcap_unlock(admin); } return err; unlock: - mutex_unlock(&admin->lock); + vcap_unlock(admin); return err; } EXPORT_SYMBOL_GPL(vcap_get_rule_count_by_cookie); diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api.h b/drivers/net/ethernet/microchip/vcap/vcap_api.h index 6069ad95c27e9..05b4b02e59ef5 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_api.h @@ -164,7 +164,7 @@ struct vcap_admin { struct list_head list; /* for insertion in vcap_control */ struct list_head rules; /* list of rules */ struct list_head enabled; /* list of enabled ports */ - struct mutex lock; /* control access to rules */ + struct vcap_control *vctrl; /* the control instance owning this vcap */ enum vcap_type vtype; /* type of vcap */ int vinst; /* instance number within the same type */ int first_cid; /* first chain id in this vcap */ @@ -275,6 +275,7 @@ struct vcap_control { const struct vcap_info *vcaps; /* client supplied vcap models */ const struct vcap_statistics *stats; /* client supplied vcap stats */ struct list_head list; /* list of vcap instances */ + struct mutex lock; /* serialize access to all vcap instances */ }; #endif /* __VCAP_API__ */ diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c index 59bfbda29bb3d..e0c65c7ab23e1 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c @@ -410,9 +410,9 @@ static int vcap_debugfs_show(struct seq_file *m, void *unused) }; int ret; - mutex_lock(&info->admin->lock); + vcap_lock(info->admin); ret = vcap_show_admin(info->vctrl, info->admin, &out); - mutex_unlock(&info->admin->lock); + vcap_unlock(info->admin); return ret; } DEFINE_SHOW_ATTRIBUTE(vcap_debugfs); @@ -427,9 +427,9 @@ static int vcap_raw_debugfs_show(struct seq_file *m, void *unused) }; int ret; - mutex_lock(&info->admin->lock); + vcap_lock(info->admin); ret = vcap_show_admin_raw(info->vctrl, info->admin, &out); - mutex_unlock(&info->admin->lock); + vcap_unlock(info->admin); return ret; } DEFINE_SHOW_ATTRIBUTE(vcap_raw_debugfs); diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c index 9c9d380421255..ac2a3b8c4f321 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c @@ -243,10 +243,11 @@ static void vcap_test_api_init(struct vcap_admin *admin) { /* Initialize the shared objects */ INIT_LIST_HEAD(&test_vctrl.list); + mutex_init(&test_vctrl.lock); INIT_LIST_HEAD(&admin->list); INIT_LIST_HEAD(&admin->rules); INIT_LIST_HEAD(&admin->enabled); - mutex_init(&admin->lock); + admin->vctrl = &test_vctrl; list_add_tail(&admin->list, &test_vctrl.list); memset(test_updateaddr, 0, sizeof(test_updateaddr)); test_updateaddridx = 0; diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_kunit.c b/drivers/net/ethernet/microchip/vcap/vcap_api_kunit.c index ce26ccbdccdf9..83de384d3e3bd 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_kunit.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_kunit.c @@ -233,10 +233,11 @@ static void vcap_test_api_init(struct vcap_admin *admin) { /* Initialize the shared objects */ INIT_LIST_HEAD(&test_vctrl.list); + mutex_init(&test_vctrl.lock); INIT_LIST_HEAD(&admin->list); INIT_LIST_HEAD(&admin->rules); INIT_LIST_HEAD(&admin->enabled); - mutex_init(&admin->lock); + admin->vctrl = &test_vctrl; list_add_tail(&admin->list, &test_vctrl.list); memset(test_updateaddr, 0, sizeof(test_updateaddr)); test_updateaddridx = 0; diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_private.h b/drivers/net/ethernet/microchip/vcap/vcap_api_private.h index 844bdf6b5f450..b4057fbe3d188 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_private.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_private.h @@ -50,6 +50,9 @@ struct vcap_stream_iter { /* Check that the control has a valid set of callbacks */ int vcap_api_check(struct vcap_control *ctrl); +/* Serialize access to the vcap instances of a control */ +void vcap_lock(struct vcap_admin *admin); +void vcap_unlock(struct vcap_admin *admin); /* Erase the VCAP cache area used or encoding and decoding */ void vcap_erase_cache(struct vcap_rule_internal *ri); diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index d8e816882f02c..ac71ca8450bf4 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -1192,6 +1192,8 @@ int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, if (!queue) return -ENOMEM; + queue->id = INVALID_QUEUE_ID; + gmi = &queue->mem_info; err = mana_gd_alloc_memory(gc, spec->queue_size, gmi); if (err) { diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c index 7697c9b52ed34..d59dec89a8077 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c +++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c @@ -237,7 +237,8 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog, bpf_prog_put(old_prog); if (prog) - ndev->max_mtu = MANA_XDP_MTU_MAX; + ndev->max_mtu = min_t(unsigned int, MANA_XDP_MTU_MAX, + gc->adapter_mtu - ETH_HLEN); else ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index c9b1df1ed1098..352398ae0376e 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -1215,12 +1215,24 @@ static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver, *max_num_vports = resp.max_num_vports; if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) { - if (resp.adapter_mtu < ETH_MIN_MTU + ETH_HLEN) { + if (resp.adapter_mtu == 0) { + /* + * Some older PF firmware versions report an + * adapter_mtu of 0. MANA hardware always supports the + * standard Ethernet MTU, so fall back to ETH_FRAME_LEN. + * Jumbo frames will not be available in this case. + */ + dev_info(dev, + "PF reported adapter_mtu of 0, falling back to %u (jumbo frames disabled)\n", + ETH_FRAME_LEN); + gc->adapter_mtu = ETH_FRAME_LEN; + } else if (resp.adapter_mtu < ETH_MIN_MTU + ETH_HLEN) { dev_err(dev, "Adapter MTU too small: %u\n", resp.adapter_mtu); return -EPROTO; + } else { + gc->adapter_mtu = resp.adapter_mtu; } - gc->adapter_mtu = resp.adapter_mtu; } else { gc->adapter_mtu = ETH_FRAME_LEN; } @@ -2044,12 +2056,16 @@ static void mana_rx_skb(void *buf_va, bool from_pool, } static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, - dma_addr_t *da, bool *from_pool) + dma_addr_t *da, bool *from_pool, + struct page **pp_page, u32 *dma_sync_offset) { struct page *page; u32 offset; void *va; + *from_pool = false; + *pp_page = NULL; + *dma_sync_offset = 0; /* Don't use fragments for jumbo frames or XDP where it's 1 fragment * per page. @@ -2087,31 +2103,47 @@ static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, va = page_to_virt(page) + offset; *da = page_pool_get_dma_addr(page) + offset + rxq->headroom; *from_pool = true; + *pp_page = page; + *dma_sync_offset = offset + rxq->headroom; return va; } /* Allocate frag for rx buffer, and save the old buf */ static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, - struct mana_recv_buf_oob *rxoob, void **old_buf, - bool *old_fp) + struct mana_recv_buf_oob *rxoob, u32 pktlen, + void **old_buf, bool *old_fp) { + struct page *pp_page; + u32 dma_sync_offset; bool from_pool; dma_addr_t da; void *va; - va = mana_get_rxfrag(rxq, dev, &da, &from_pool); + va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, + &dma_sync_offset); if (!va) return; - if (!rxoob->from_pool || rxq->frag_count == 1) + if (!rxoob->from_pool || rxq->frag_count == 1) { dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize, DMA_FROM_DEVICE); + } else { + /* The page pool maps the whole page and only syncs for device + * automatically (PP_FLAG_DMA_SYNC_DEV). Sync the received bytes + * for the CPU before they are read: this is required if DMA + * is incoherent or bounce buffers are used. + */ + page_pool_dma_sync_for_cpu(rxq->page_pool, rxoob->pp_page, + rxoob->dma_sync_offset, pktlen); + } *old_buf = rxoob->buf_va; *old_fp = rxoob->from_pool; rxoob->buf_va = va; rxoob->sgl[0].address = da; rxoob->from_pool = from_pool; + rxoob->pp_page = pp_page; + rxoob->dma_sync_offset = dma_sync_offset; } static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, @@ -2170,12 +2202,26 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, rxbuf_oob = &rxq->rx_oobs[curr]; WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); - mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp); + if (unlikely(pktlen > rxq->datasize)) { + /* Increase it even if mana_rx_skb() isn't called. */ + rxq->rx_cq.work_done++; - /* Unsuccessful refill will have old_buf == NULL. - * In this case, mana_rx_skb() will drop the packet. - */ - mana_rx_skb(old_buf, old_fp, oob, rxq, i); + ++ndev->stats.rx_dropped; + netdev_warn_once(ndev, + "Dropped oversized RX packet: len=%u, datasize=%u\n", + pktlen, rxq->datasize); + + /* Reuse the RX buffer since rxbuf_oob is unchanged. */ + } else { + + mana_refill_rx_oob(dev, rxq, rxbuf_oob, pktlen, + &old_buf, &old_fp); + + /* Unsuccessful refill will have old_buf == NULL. + * In this case, mana_rx_skb() will drop the packet. + */ + mana_rx_skb(old_buf, old_fp, oob, rxq, i); + } mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); @@ -2334,7 +2380,8 @@ static void mana_destroy_txq(struct mana_port_context *apc) netif_napi_del_locked(napi); apc->tx_qp[i].txq.napi_initialized = false; } - mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object); + if (apc->tx_qp[i].tx_object != INVALID_MANA_HANDLE) + mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object); mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq); @@ -2566,6 +2613,8 @@ static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, struct mana_rxq *rxq, struct device *dev) { struct mana_port_context *mpc = netdev_priv(rxq->ndev); + struct page *pp_page = NULL; + u32 dma_sync_offset = 0; bool from_pool = false; dma_addr_t da; void *va; @@ -2573,13 +2622,16 @@ static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, if (mpc->rxbufs_pre) va = mana_get_rxbuf_pre(rxq, &da); else - va = mana_get_rxfrag(rxq, dev, &da, &from_pool); + va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, + &dma_sync_offset); if (!va) return -ENOMEM; rx_oob->buf_va = va; rx_oob->from_pool = from_pool; + rx_oob->pp_page = pp_page; + rx_oob->dma_sync_offset = dma_sync_offset; rx_oob->sgl[0].address = da; rx_oob->sgl[0].size = rxq->datasize; diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c index de5e29230b3c8..c464086982632 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c @@ -166,18 +166,23 @@ static void mucse_mbx_inc_pf_ack(struct mucse_hw *hw) * * Return: 0 on success, negative errno on failure **/ -static int mucse_read_mbx_pf(struct mucse_hw *hw, u32 *msg, u16 size) +static int mucse_read_mbx_pf(struct mucse_hw *hw, __le32 *msg, u16 size) { - const int size_in_words = size / sizeof(u32); + const int size_in_words = size / sizeof(__le32); struct mucse_mbx_info *mbx = &hw->mbx; + int off = MUCSE_MBX_FWPF_SHM; int err; err = mucse_obtain_mbx_lock_pf(hw); if (err) return err; + /* memcpy_fromio() is unsuitable: the mailbox uses 32-bit MMIO + * registers, not byte-addressable RAM. readl() guarantees + * the required 32-bit access width. + */ for (int i = 0; i < size_in_words; i++) - msg[i] = mbx_data_rd32(mbx, MUCSE_MBX_FWPF_SHM + 4 * i); + msg[i] = cpu_to_le32(mbx_data_rd32(mbx, off + 4 * i)); /* Hw needs write data_reg at last */ mbx_data_wr32(mbx, MUCSE_MBX_FWPF_SHM, 0); /* flush reqs as we have read this request data */ @@ -236,7 +241,7 @@ static int mucse_poll_for_msg(struct mucse_hw *hw) * Return: 0 if it successfully received a message notification and * copied it into the receive buffer, negative errno on failure **/ -int mucse_poll_and_read_mbx(struct mucse_hw *hw, u32 *msg, u16 size) +int mucse_poll_and_read_mbx(struct mucse_hw *hw, __le32 *msg, u16 size) { int err; @@ -290,9 +295,9 @@ static void mucse_mbx_inc_pf_req(struct mucse_hw *hw) * Return: 0 if it successfully copied message into the buffer, * negative errno on failure **/ -static int mucse_write_mbx_pf(struct mucse_hw *hw, u32 *msg, u16 size) +static int mucse_write_mbx_pf(struct mucse_hw *hw, const __le32 *msg, u16 size) { - const int size_in_words = size / sizeof(u32); + const int size_in_words = size / sizeof(__le32); struct mucse_mbx_info *mbx = &hw->mbx; int err; @@ -300,8 +305,12 @@ static int mucse_write_mbx_pf(struct mucse_hw *hw, u32 *msg, u16 size) if (err) return err; + /* memcpy_toio() would decompose into arbitrary-width accesses; + * the mailbox requires 32-bit MMIO writes via writel(). + */ for (int i = 0; i < size_in_words; i++) - mbx_data_wr32(mbx, MUCSE_MBX_FWPF_SHM + i * 4, msg[i]); + mbx_data_wr32(mbx, MUCSE_MBX_FWPF_SHM + i * 4, + le32_to_cpu(msg[i])); /* flush acks as we are overwriting the message buffer */ hw->mbx.fw_ack = mucse_mbx_get_fwack(mbx); @@ -360,7 +369,8 @@ static int mucse_poll_for_ack(struct mucse_hw *hw) * Return: 0 if it successfully copied message into the buffer and * received an ack to that message within delay * timeout_cnt period **/ -int mucse_write_and_wait_ack_mbx(struct mucse_hw *hw, u32 *msg, u16 size) +int mucse_write_and_wait_ack_mbx(struct mucse_hw *hw, const __le32 *msg, + u16 size) { int err; diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h index e6fcc8d1d3ca7..75b88b18b04d3 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h @@ -14,7 +14,8 @@ #define MUCSE_MBX_REQ BIT(0) /* Request a req to mailbox */ #define MUCSE_MBX_PFU BIT(3) /* PF owns the mailbox buffer */ -int mucse_write_and_wait_ack_mbx(struct mucse_hw *hw, u32 *msg, u16 size); +int mucse_write_and_wait_ack_mbx(struct mucse_hw *hw, + const __le32 *msg, u16 size); void mucse_init_mbx_params_pf(struct mucse_hw *hw); -int mucse_poll_and_read_mbx(struct mucse_hw *hw, u32 *msg, u16 size); +int mucse_poll_and_read_mbx(struct mucse_hw *hw, __le32 *msg, u16 size); #endif /* _RNPGBE_MBX_H */ diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c index 8c8bd5e8e1db1..5ba74997beacf 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c @@ -20,32 +20,32 @@ * Return: 0 on success, negative errno on failure **/ static int mucse_fw_send_cmd_wait_resp(struct mucse_hw *hw, - struct mbx_fw_cmd_req *req, - struct mbx_fw_cmd_reply *reply) + union mbx_fw_cmd_req_u *req, + union mbx_fw_cmd_reply_u *reply) { - int len = le16_to_cpu(req->datalen); + int len = le16_to_cpu(req->r.datalen); int retry_cnt = 3; int err; mutex_lock(&hw->mbx.lock); - err = mucse_write_and_wait_ack_mbx(hw, (u32 *)req, len); + err = mucse_write_and_wait_ack_mbx(hw, req->dwords, len); if (err) goto out; do { - err = mucse_poll_and_read_mbx(hw, (u32 *)reply, - sizeof(*reply)); + err = mucse_poll_and_read_mbx(hw, reply->dwords, + sizeof(reply->r)); if (err) goto out; /* mucse_write_and_wait_ack_mbx return 0 means fw has * received request, wait for the expect opcode * reply with 'retry_cnt' times. */ - } while (--retry_cnt >= 0 && reply->opcode != req->opcode); + } while (--retry_cnt >= 0 && reply->r.opcode != req->r.opcode); out: mutex_unlock(&hw->mbx.lock); if (!err && retry_cnt < 0) return -ETIMEDOUT; - if (!err && reply->error_code) + if (!err && reply->r.error_code) return -EIO; return err; @@ -61,17 +61,19 @@ static int mucse_fw_send_cmd_wait_resp(struct mucse_hw *hw, **/ static int mucse_mbx_get_info(struct mucse_hw *hw) { - struct mbx_fw_cmd_req req = { - .datalen = cpu_to_le16(MUCSE_MBX_REQ_HDR_LEN), - .opcode = cpu_to_le16(GET_HW_INFO), + union mbx_fw_cmd_req_u req = { + .r = { + .datalen = cpu_to_le16(MUCSE_MBX_REQ_HDR_LEN), + .opcode = cpu_to_le16(GET_HW_INFO), + }, }; - struct mbx_fw_cmd_reply reply = {}; + union mbx_fw_cmd_reply_u reply = {}; int err; err = mucse_fw_send_cmd_wait_resp(hw, &req, &reply); if (!err) hw->pfvfnum = FIELD_GET(GENMASK_U16(7, 0), - le16_to_cpu(reply.hw_info.pfnum)); + le16_to_cpu(reply.r.hw_info.pfnum)); return err; } @@ -111,21 +113,23 @@ int mucse_mbx_sync_fw(struct mucse_hw *hw) **/ int mucse_mbx_powerup(struct mucse_hw *hw, bool is_powerup) { - struct mbx_fw_cmd_req req = { - .datalen = cpu_to_le16(sizeof(req.powerup) + - MUCSE_MBX_REQ_HDR_LEN), - .opcode = cpu_to_le16(POWER_UP), - .powerup = { - /* fw needs this to reply correct cmd */ - .version = cpu_to_le32(GENMASK_U32(31, 0)), - .status = cpu_to_le32(is_powerup ? 1 : 0), + union mbx_fw_cmd_req_u req = { + .r = { + .datalen = cpu_to_le16(sizeof(req.r.powerup) + + MUCSE_MBX_REQ_HDR_LEN), + .opcode = cpu_to_le16(POWER_UP), + .powerup = { + /* fw needs this to reply correct cmd */ + .version = cpu_to_le32(GENMASK_U32(31, 0)), + .status = cpu_to_le32(is_powerup ? 1 : 0), + }, }, }; int len, err; - len = le16_to_cpu(req.datalen); + len = le16_to_cpu(req.r.datalen); mutex_lock(&hw->mbx.lock); - err = mucse_write_and_wait_ack_mbx(hw, (u32 *)&req, len); + err = mucse_write_and_wait_ack_mbx(hw, req.dwords, len); mutex_unlock(&hw->mbx.lock); return err; @@ -142,11 +146,13 @@ int mucse_mbx_powerup(struct mucse_hw *hw, bool is_powerup) **/ int mucse_mbx_reset_hw(struct mucse_hw *hw) { - struct mbx_fw_cmd_req req = { - .datalen = cpu_to_le16(MUCSE_MBX_REQ_HDR_LEN), - .opcode = cpu_to_le16(RESET_HW), + union mbx_fw_cmd_req_u req = { + .r = { + .datalen = cpu_to_le16(MUCSE_MBX_REQ_HDR_LEN), + .opcode = cpu_to_le16(RESET_HW), + }, }; - struct mbx_fw_cmd_reply reply = {}; + union mbx_fw_cmd_reply_u reply = {}; return mucse_fw_send_cmd_wait_resp(hw, &req, &reply); } @@ -166,24 +172,26 @@ int mucse_mbx_get_macaddr(struct mucse_hw *hw, int pfvfnum, u8 *mac_addr, int port) { - struct mbx_fw_cmd_req req = { - .datalen = cpu_to_le16(sizeof(req.get_mac_addr) + - MUCSE_MBX_REQ_HDR_LEN), - .opcode = cpu_to_le16(GET_MAC_ADDRESS), - .get_mac_addr = { - .port_mask = cpu_to_le32(BIT(port)), - .pfvf_num = cpu_to_le32(pfvfnum), + union mbx_fw_cmd_req_u req = { + .r = { + .datalen = cpu_to_le16(sizeof(req.r.get_mac_addr) + + MUCSE_MBX_REQ_HDR_LEN), + .opcode = cpu_to_le16(GET_MAC_ADDRESS), + .get_mac_addr = { + .port_mask = cpu_to_le32(BIT(port)), + .pfvf_num = cpu_to_le32(pfvfnum), + }, }, }; - struct mbx_fw_cmd_reply reply = {}; + union mbx_fw_cmd_reply_u reply = {}; int err; err = mucse_fw_send_cmd_wait_resp(hw, &req, &reply); if (err) return err; - if (le32_to_cpu(reply.mac_addr.ports) & BIT(port)) - memcpy(mac_addr, reply.mac_addr.addrs[port].mac, ETH_ALEN); + if (le32_to_cpu(reply.r.mac_addr.ports) & BIT(port)) + memcpy(mac_addr, reply.r.mac_addr.addrs[port].mac, ETH_ALEN); else return -ENODATA; diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h index fb24fc12b6139..fe996aeffc4de 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h @@ -80,6 +80,20 @@ struct mbx_fw_cmd_reply { }; } __packed; +/* Union wrappers to expose struct as __le32 dword array for mailbox + * transport, eliminating the need for pointer casts. The __packed + * structs have no padding, so dwords[] overlays the fields exactly. + */ +union mbx_fw_cmd_req_u { + struct mbx_fw_cmd_req r; + __le32 dwords[sizeof(struct mbx_fw_cmd_req) / sizeof(__le32)]; +}; + +union mbx_fw_cmd_reply_u { + struct mbx_fw_cmd_reply r; + __le32 dwords[sizeof(struct mbx_fw_cmd_reply) / sizeof(__le32)]; +}; + int mucse_mbx_sync_fw(struct mucse_hw *hw); int mucse_mbx_powerup(struct mucse_hw *hw, bool is_powerup); int mucse_mbx_reset_hw(struct mucse_hw *hw); diff --git a/drivers/net/ethernet/oa_tc6.c b/drivers/net/ethernet/oa_tc6.c index 91a906a7918a2..cfe1f6e3cd12c 100644 --- a/drivers/net/ethernet/oa_tc6.c +++ b/drivers/net/ethernet/oa_tc6.c @@ -752,6 +752,17 @@ static int oa_tc6_process_rx_chunk_footer(struct oa_tc6 *tc6, u32 footer) static void oa_tc6_submit_rx_skb(struct oa_tc6 *tc6) { + /* MAC-PHY delivers each frame with its Ethernet FCS attached. + * Strip it before handing over to the stack, unless the user + * has asked to keep it via NETIF_F_RXFCS. Keeping the FCS + * in the frame is harmless for IP traffic, but is parsed as + * a (malformed) suffix TLV by PTP, which makes ptp4l reject + * every message with "bad message" error. + */ + if (!(tc6->netdev->features & NETIF_F_RXFCS) && + tc6->rx_skb->len > ETH_FCS_LEN) + skb_trim(tc6->rx_skb, tc6->rx_skb->len - ETH_FCS_LEN); + tc6->rx_skb->protocol = eth_type_trans(tc6->rx_skb, tc6->netdev); tc6->netdev->stats.rx_packets++; tc6->netdev->stats.rx_bytes += tc6->rx_skb->len; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 78a802eb159f0..6069fa4609131 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -116,8 +116,15 @@ static void ionic_get_link_ext_stats(struct net_device *netdev, { struct ionic_lif *lif = netdev_priv(netdev); - if (lif->ionic->pdev->is_physfn) - stats->link_down_events = lif->link_down_count; + if (lif->ionic->pdev->is_virtfn) + return; + + if (!lif->ionic->idev.port_info) { + netdev_err_once(netdev, "port_info not initialized\n"); + return; + } + + stats->link_down_events = lif->link_down_count; } static int ionic_get_link_ksettings(struct net_device *netdev, diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index e338bfc8b7b2f..c11e0d8f98aa5 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -765,6 +765,9 @@ qede_tpa_rx_build_skb(struct qede_dev *edev, struct sk_buff *skb; skb = qede_build_skb(rxq, bd, len, pad); + if (unlikely(!skb)) + return NULL; + bd->page_offset += rxq->rx_buf_seg_size; if (bd->page_offset == PAGE_SIZE) { @@ -812,6 +815,8 @@ qede_rx_build_skb(struct qede_dev *edev, } skb = qede_build_skb(rxq, bd, len, pad); + if (unlikely(!skb)) + return NULL; if (unlikely(qede_realloc_rx_buffer(rxq, bd))) { /* Incr page ref count to reuse on allocation failure so @@ -961,7 +966,7 @@ static inline void qede_tpa_cont(struct qede_dev *edev, { int i; - for (i = 0; cqe->len_list[i] && i < ARRAY_SIZE(cqe->len_list); i++) + for (i = 0; i < ARRAY_SIZE(cqe->len_list) && cqe->len_list[i]; i++) qede_fill_frag_skb(edev, rxq, cqe->tpa_agg_index, le16_to_cpu(cqe->len_list[i])); @@ -986,7 +991,7 @@ static int qede_tpa_end(struct qede_dev *edev, dma_unmap_page(rxq->dev, tpa_info->buffer.mapping, PAGE_SIZE, rxq->data_direction); - for (i = 0; cqe->len_list[i] && i < ARRAY_SIZE(cqe->len_list); i++) + for (i = 0; i < ARRAY_SIZE(cqe->len_list) && cqe->len_list[i]; i++) qede_fill_frag_skb(edev, rxq, cqe->tpa_agg_index, le16_to_cpu(cqe->len_list[i])); if (unlikely(i > 1)) diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c index 9f3479500f85a..d055a2628d8c9 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c @@ -126,7 +126,10 @@ rmnet_map_ingress_handler(struct sk_buff *skb, consume_skb(skb); } else { - __rmnet_map_ingress_handler(skb, port); + if (rmnet_map_validate_packet_len(skb, port)) + __rmnet_map_ingress_handler(skb, port); + else + kfree_skb(skb); } } diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h index b70284095568c..60ca8b780c88a 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h @@ -59,5 +59,6 @@ void rmnet_map_tx_aggregate_init(struct rmnet_port *port); void rmnet_map_tx_aggregate_exit(struct rmnet_port *port); void rmnet_map_update_ul_agg_config(struct rmnet_port *port, u32 size, u32 count, u32 time); +u32 rmnet_map_validate_packet_len(struct sk_buff *skb, struct rmnet_port *port); #endif /* _RMNET_MAP_H_ */ diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c index 8b4640c5d61eb..305ae15ae8f39 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c @@ -333,54 +333,62 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, return map_header; } -/* Deaggregates a single packet - * A whole new buffer is allocated for each portion of an aggregated frame. - * Caller should keep calling deaggregate() on the source skb until 0 is - * returned, indicating that there are no more packets to deaggregate. Caller - * is responsible for freeing the original skb. - */ -struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, - struct rmnet_port *port) +u32 rmnet_map_validate_packet_len(struct sk_buff *skb, struct rmnet_port *port) { struct rmnet_map_v5_csum_header *next_hdr = NULL; struct rmnet_map_header *maph; void *data = skb->data; - struct sk_buff *skbn; - u8 nexthdr_type; u32 packet_len; - if (skb->len == 0) - return NULL; + if (skb->len < sizeof(*maph)) + return 0; maph = (struct rmnet_map_header *)skb->data; + + /* Some hardware can send us empty frames. Catch them */ + if (!maph->pkt_len) + return 0; + packet_len = ntohs(maph->pkt_len) + sizeof(*maph); if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { packet_len += sizeof(struct rmnet_map_dl_csum_trailer); - } else if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) { - if (!(maph->flags & MAP_CMD_FLAG)) { - packet_len += sizeof(*next_hdr); - if (maph->flags & MAP_NEXT_HEADER_FLAG) - next_hdr = data + sizeof(*maph); - else - /* Mapv5 data pkt without csum hdr is invalid */ - return NULL; - } + } else if ((port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) && + !(maph->flags & MAP_CMD_FLAG)) { + /* Mapv5 data pkt without csum hdr is invalid */ + if (!(maph->flags & MAP_NEXT_HEADER_FLAG)) + return 0; + + packet_len += sizeof(*next_hdr); + next_hdr = data + sizeof(*maph); } - if (((int)skb->len - (int)packet_len) < 0) - return NULL; + if (skb->len < packet_len) + return 0; - /* Some hardware can send us empty frames. Catch them */ - if (!maph->pkt_len) - return NULL; + if (next_hdr && + u8_get_bits(next_hdr->header_info, MAPV5_HDRINFO_HDR_TYPE_FMASK) != + RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD) + return 0; - if (next_hdr) { - nexthdr_type = u8_get_bits(next_hdr->header_info, - MAPV5_HDRINFO_HDR_TYPE_FMASK); - if (nexthdr_type != RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD) - return NULL; - } + return packet_len; +} + +/* Deaggregates a single packet + * A whole new buffer is allocated for each portion of an aggregated frame. + * Caller should keep calling deaggregate() on the source skb until 0 is + * returned, indicating that there are no more packets to deaggregate. Caller + * is responsible for freeing the original skb. + */ +struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, + struct rmnet_port *port) +{ + struct sk_buff *skbn; + u32 packet_len; + + packet_len = rmnet_map_validate_packet_len(skb, port); + if (!packet_len) + return NULL; skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, GFP_ATOMIC); if (!skbn) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c index 223754cc5c79b..322bdf167a4a6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c @@ -18,10 +18,12 @@ #include "stmmac_platform.h" /* ctrl register bits */ -#define CTRL_PHY_INTF_RGMII BIT(3) -#define CTRL_PHY_INTF_MII BIT(4) -#define CTRL_WAKE_IRQ_EN BIT(9) -#define CTRL_PHY_IRQ_EN BIT(12) +#define CTRL_PHY_INTF_MODE GENMASK(4, 3) +#define CTRL_PHY_INTF_RMII FIELD_PREP(CTRL_PHY_INTF_MODE, 0) +#define CTRL_PHY_INTF_RGMII FIELD_PREP(CTRL_PHY_INTF_MODE, 1) +#define CTRL_PHY_INTF_MII FIELD_PREP(CTRL_PHY_INTF_MODE, 3) +#define CTRL_LPI_IRQ_EN BIT(9) +#define CTRL_WAKE_IRQ_EN BIT(12) /* dline register bits */ #define RGMII_RX_DLINE_EN BIT(0) @@ -118,7 +120,7 @@ static void spacemit_get_interfaces(struct stmmac_priv *priv, void *bsp_priv, static int spacemit_set_phy_intf_sel(void *bsp_priv, u8 phy_intf_sel) { - unsigned int mask = CTRL_PHY_INTF_MII | CTRL_PHY_INTF_RGMII; + unsigned int mask = CTRL_PHY_INTF_MODE; struct spacmit_dwmac *dwmac = bsp_priv; unsigned int val = 0; @@ -128,6 +130,7 @@ static int spacemit_set_phy_intf_sel(void *bsp_priv, u8 phy_intf_sel) break; case PHY_INTF_SEL_RMII: + val = CTRL_PHY_INTF_RMII; break; case PHY_INTF_SEL_RGMII: diff --git a/drivers/net/ethernet/sun/sungem.c b/drivers/net/ethernet/sun/sungem.c index 8e69d917d827c..26974ee713525 100644 --- a/drivers/net/ethernet/sun/sungem.c +++ b/drivers/net/ethernet/sun/sungem.c @@ -2986,10 +2986,10 @@ static int gem_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->max_mtu = GEM_MAX_MTU; /* Register with kernel */ - if (register_netdev(dev)) { + err = register_netdev(dev); + if (err) { pr_err("Cannot register net device, aborting\n"); - err = -ENOMEM; - goto err_out_free_consistent; + goto err_out_clear_drvdata; } /* Undo the get_cell with appropriate locking (we could use @@ -3003,8 +3003,13 @@ static int gem_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->dev_addr); return 0; +err_out_clear_drvdata: + pci_set_drvdata(pdev, NULL); + netif_napi_del(&gp->napi); + err_out_free_consistent: - gem_remove_one(pdev); + dma_free_coherent(&pdev->dev, sizeof(struct gem_init_block), + gp->init_block, gp->gblock_dvma); err_out_iounmap: gem_put_cell(gp); iounmap(gp->regs); diff --git a/drivers/net/ethernet/sunplus/spl2sw_phy.c b/drivers/net/ethernet/sunplus/spl2sw_phy.c index 6f899e48f51df..a4889c52e00e4 100644 --- a/drivers/net/ethernet/sunplus/spl2sw_phy.c +++ b/drivers/net/ethernet/sunplus/spl2sw_phy.c @@ -79,12 +79,14 @@ int spl2sw_phy_connect(struct spl2sw_common *comm) void spl2sw_phy_remove(struct spl2sw_common *comm) { struct net_device *ndev; + struct spl2sw_mac *mac; int i; for (i = 0; i < MAX_NETDEV_NUM; i++) if (comm->ndev[i]) { ndev = comm->ndev[i]; - if (ndev) - phy_disconnect(ndev->phydev); + mac = netdev_priv(ndev); + phy_disconnect(ndev->phydev); + of_node_put(mac->phy_node); } } diff --git a/drivers/net/ethernet/ti/icssg/icssg_common.c b/drivers/net/ethernet/ti/icssg/icssg_common.c index a28a608f9bf4b..4a7d1a6f470b1 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_common.c +++ b/drivers/net/ethernet/ti/icssg/icssg_common.c @@ -93,8 +93,8 @@ void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num) } EXPORT_SYMBOL_GPL(prueth_ndev_del_tx_napi); -static int emac_xsk_xmit_zc(struct prueth_emac *emac, - unsigned int q_idx) +static void emac_xsk_xmit_zc(struct prueth_emac *emac, + unsigned int q_idx) { struct prueth_tx_chn *tx_chn = &emac->tx_chns[q_idx]; struct xsk_buff_pool *pool = tx_chn->xsk_pool; @@ -105,6 +105,7 @@ static int emac_xsk_xmit_zc(struct prueth_emac *emac, struct xdp_desc xdp_desc; int num_tx = 0, pkt_len; int descs_avail, ret; + u32 dst_tag_id; u32 *epib; int i; @@ -114,7 +115,7 @@ static int emac_xsk_xmit_zc(struct prueth_emac *emac, * necessary */ if (descs_avail <= MAX_SKB_FRAGS) - return 0; + return; descs_avail -= MAX_SKB_FRAGS; @@ -137,9 +138,17 @@ static int emac_xsk_xmit_zc(struct prueth_emac *emac, epib[0] = 0; epib[1] = 0; cppi5_hdesc_set_pktlen(host_desc, pkt_len); - cppi5_desc_set_tags_ids(&host_desc->hdr, 0, - (emac->port_id | (q_idx << 8))); + dst_tag_id = emac->port_id | (q_idx << 8); + + if (emac->prueth->is_hsr_offload_mode && + (ndev->features & NETIF_F_HW_HSR_DUP)) + dst_tag_id = PRUETH_UNDIRECTED_PKT_DST_TAG; + + if (emac->prueth->is_hsr_offload_mode && + (ndev->features & NETIF_F_HW_HSR_TAG_INS)) + epib[1] |= PRUETH_UNDIRECTED_PKT_TAG_INS; + cppi5_desc_set_tags_ids(&host_desc->hdr, 0, dst_tag_id); k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf); cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf, pkt_len); @@ -161,8 +170,8 @@ static int emac_xsk_xmit_zc(struct prueth_emac *emac, num_tx++; } - xsk_tx_release(tx_chn->xsk_pool); - return num_tx; + if (num_tx) + xsk_tx_release(tx_chn->xsk_pool); } void prueth_xmit_free(struct prueth_tx_chn *tx_chn, @@ -270,9 +279,6 @@ int emac_tx_complete_packets(struct prueth_emac *emac, int chn, num_tx++; } - if (!num_tx) - return 0; - netif_txq = netdev_get_tx_queue(ndev, chn); netdev_tx_completed_queue(netif_txq, num_tx, total_bytes); @@ -288,16 +294,18 @@ int emac_tx_complete_packets(struct prueth_emac *emac, int chn, __netif_tx_unlock(netif_txq); } - if (tx_chn->xsk_pool) { - if (xsk_frames_done) + if (budget && tx_chn->xsk_pool) { + if (xsk_frames_done) { xsk_tx_completed(tx_chn->xsk_pool, xsk_frames_done); + txq_trans_cond_update(netif_txq); + } if (xsk_uses_need_wakeup(tx_chn->xsk_pool)) xsk_set_tx_need_wakeup(tx_chn->xsk_pool); - netif_txq = netdev_get_tx_queue(ndev, chn); - txq_trans_cond_update(netif_txq); + __netif_tx_lock(netif_txq, smp_processor_id()); emac_xsk_xmit_zc(emac, chn); + __netif_tx_unlock(netif_txq); } return num_tx; @@ -696,6 +704,7 @@ u32 emac_xmit_xdp_frame(struct prueth_emac *emac, dma_addr_t desc_dma, buf_dma; struct prueth_swdata *swdata; struct page *page; + u32 dst_tag_id; u32 *epib; int ret; @@ -737,9 +746,25 @@ u32 emac_xmit_xdp_frame(struct prueth_emac *emac, /* set dst tag to indicate internal qid at the firmware which is at * bit8..bit15. bit0..bit7 indicates port num for directed - * packets in case of switch mode operation + * packets in case of switch mode operation and port num 0 + * for undirected packets in case of HSR offload mode. + * + * XDP_TX frames arrive on a slave port with the HSR tag already + * stripped by the PRU firmware. Like skb TX via hsr0, they must + * be sent as undirected so the PRU duplicates them to both ports + * and re-inserts the HSR sequence tag. */ - cppi5_desc_set_tags_ids(&first_desc->hdr, 0, (emac->port_id | (q_idx << 8))); + dst_tag_id = emac->port_id | (q_idx << 8); + + if (emac->prueth->is_hsr_offload_mode && + (ndev->features & NETIF_F_HW_HSR_DUP)) + dst_tag_id = PRUETH_UNDIRECTED_PKT_DST_TAG; + + if (emac->prueth->is_hsr_offload_mode && + (ndev->features & NETIF_F_HW_HSR_TAG_INS)) + epib[1] |= PRUETH_UNDIRECTED_PKT_TAG_INS; + + cppi5_desc_set_tags_ids(&first_desc->hdr, 0, dst_tag_id); k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); cppi5_hdesc_attach_buf(first_desc, buf_dma, xdpf->len, buf_dma, xdpf->len); swdata = cppi5_hdesc_get_swdata(first_desc); @@ -927,6 +952,7 @@ static int emac_rx_packet_zc(struct prueth_emac *emac, u32 flow_id, struct cppi5_host_desc_t *desc_rx; struct prueth_swdata *swdata; dma_addr_t desc_dma, buf_dma; + int avail_desc, alloc_budget; struct xdp_buff *xdp; int xdp_status = 0; int count = 0; @@ -993,16 +1019,13 @@ static int emac_rx_packet_zc(struct prueth_emac *emac, u32 flow_id, if (xdp_status & ICSSG_XDP_REDIR) xdp_do_flush(); - /* Allocate xsk buffers from the pool for the "count" number of - * packets processed in order to be able to receive more packets. - */ - ret = prueth_rx_alloc_zc(emac, count); + avail_desc = k3_cppi_desc_pool_avail(rx_chn->desc_pool); + alloc_budget = min_t(int, budget, avail_desc); + + ret = prueth_rx_alloc_zc(emac, alloc_budget); if (xsk_uses_need_wakeup(rx_chn->xsk_pool)) { - /* If the user space doesn't provide enough buffers then it must - * explicitly wake up the kernel when new buffers are available - */ - if (ret < count) + if (ret < alloc_budget) xsk_set_rx_need_wakeup(rx_chn->xsk_pool); else xsk_clear_rx_need_wakeup(rx_chn->xsk_pool); @@ -1628,28 +1651,35 @@ void icssg_ndo_get_stats64(struct net_device *ndev, stats->rx_over_errors = emac_get_stat_by_name(emac, "rx_over_errors"); stats->multicast = emac_get_stat_by_name(emac, "rx_multicast_frames"); - stats->rx_errors = ndev->stats.rx_errors + - emac_get_stat_by_name(emac, "FW_RX_ERROR") + - emac_get_stat_by_name(emac, "FW_RX_EOF_SHORT_FRMERR") + - emac_get_stat_by_name(emac, "FW_RX_B0_DROP_EARLY_EOF") + - emac_get_stat_by_name(emac, "FW_RX_EXP_FRAG_Q_DROP") + - emac_get_stat_by_name(emac, "FW_RX_FIFO_OVERRUN"); - stats->rx_dropped = ndev->stats.rx_dropped + - emac_get_stat_by_name(emac, "FW_DROPPED_PKT") + - emac_get_stat_by_name(emac, "FW_INF_PORT_DISABLED") + - emac_get_stat_by_name(emac, "FW_INF_SAV") + - emac_get_stat_by_name(emac, "FW_INF_SA_DL") + - emac_get_stat_by_name(emac, "FW_INF_PORT_BLOCKED") + - emac_get_stat_by_name(emac, "FW_INF_DROP_TAGGED") + - emac_get_stat_by_name(emac, "FW_INF_DROP_PRIOTAGGED") + - emac_get_stat_by_name(emac, "FW_INF_DROP_NOTAG") + - emac_get_stat_by_name(emac, "FW_INF_DROP_NOTMEMBER"); + stats->rx_errors = ndev->stats.rx_errors; + stats->rx_dropped = ndev->stats.rx_dropped; stats->tx_errors = ndev->stats.tx_errors; - stats->tx_dropped = ndev->stats.tx_dropped + - emac_get_stat_by_name(emac, "FW_RTU_PKT_DROP") + - emac_get_stat_by_name(emac, "FW_TX_DROPPED_PACKET") + - emac_get_stat_by_name(emac, "FW_TX_TS_DROPPED_PACKET") + - emac_get_stat_by_name(emac, "FW_TX_JUMBO_FRM_CUTOFF"); + stats->tx_dropped = ndev->stats.tx_dropped; + + if (!emac->prueth->pa_stats) + return; + + stats->rx_errors += + emac_get_stat_by_name(emac, "FW_RX_ERROR") + + emac_get_stat_by_name(emac, "FW_RX_EOF_SHORT_FRMERR") + + emac_get_stat_by_name(emac, "FW_RX_B0_DROP_EARLY_EOF") + + emac_get_stat_by_name(emac, "FW_RX_EXP_FRAG_Q_DROP") + + emac_get_stat_by_name(emac, "FW_RX_FIFO_OVERRUN"); + stats->rx_dropped += + emac_get_stat_by_name(emac, "FW_DROPPED_PKT") + + emac_get_stat_by_name(emac, "FW_INF_PORT_DISABLED") + + emac_get_stat_by_name(emac, "FW_INF_SAV") + + emac_get_stat_by_name(emac, "FW_INF_SA_DL") + + emac_get_stat_by_name(emac, "FW_INF_PORT_BLOCKED") + + emac_get_stat_by_name(emac, "FW_INF_DROP_TAGGED") + + emac_get_stat_by_name(emac, "FW_INF_DROP_PRIOTAGGED") + + emac_get_stat_by_name(emac, "FW_INF_DROP_NOTAG") + + emac_get_stat_by_name(emac, "FW_INF_DROP_NOTMEMBER"); + stats->tx_dropped += + emac_get_stat_by_name(emac, "FW_RTU_PKT_DROP") + + emac_get_stat_by_name(emac, "FW_TX_DROPPED_PACKET") + + emac_get_stat_by_name(emac, "FW_TX_TS_DROPPED_PACKET") + + emac_get_stat_by_name(emac, "FW_TX_JUMBO_FRM_CUTOFF"); } EXPORT_SYMBOL_GPL(icssg_ndo_get_stats64); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index 746623fa59b40..5c2cd5756f8a9 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -1802,6 +1802,7 @@ static bool wx_set_vmdq_queues(struct wx *wx) rss_i = 4; } } else { + vmdq_m = WX_VMDQ_1Q_MASK; /* double check we are limited to maximum pools */ vmdq_i = min_t(u16, 8, vmdq_i); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 0da5565ee4ffa..1b3dc2a808f90 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -486,6 +486,7 @@ enum WX_MSCA_CMD_value { #define WX_VMDQ_4Q_MASK 0x7C #define WX_VMDQ_2Q_MASK 0x7E +#define WX_VMDQ_1Q_MASK 0x7F /****************** Manageablility Host Interface defines ********************/ #define WX_HI_MAX_BLOCK_BYTE_LENGTH 256 /* Num of bytes in range */ diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 715180c3a1b34..c55bdb805a644 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -603,7 +603,7 @@ static int geneve_post_decap_hint(const struct sock *sk, struct sk_buff *skb, ipv6h = (void *)skb->data + gro_hint->nested_nh_offset; iph = (struct iphdr *)ipv6h; total_len = skb->len - gro_hint->nested_nh_offset; - if (total_len > GRO_LEGACY_MAX_SIZE) + if (total_len >= GRO_LEGACY_MAX_SIZE) return -E2BIG; /* @@ -948,13 +948,27 @@ static int geneve_gro_complete(struct sock *sk, struct sk_buff *skb, struct genevehdr *gh; struct packet_offload *ptype; __be16 type; - int gh_len; + unsigned int gh_len; int err = -ENOSYS; gh = (struct genevehdr *)(skb->data + nhoff); gh_len = geneve_hlen(gh); type = gh->proto_type; - geneve_opt_gro_hint_off(gh, &type, &gh_len); + geneve_sk_gro_hint_off(sk, gh, &type, &gh_len); + + /* Bail out if we are about to dispatch past the inner network header + * gro_receive() validated. An inner VLAN tag only pushes + * inner_network_offset out, so use a lower bound. + */ + if (skb->encapsulation) { + unsigned int inner_nh = nhoff + gh_len; + + if (type == htons(ETH_P_TEB)) + inner_nh += ETH_HLEN; + + if (unlikely(inner_nh > NAPI_GRO_CB(skb)->inner_network_offset)) + return -EINVAL; + } /* since skb->encapsulation is set, eth_gro_complete() sets the inner mac header */ if (likely(type == htons(ETH_P_TEB))) @@ -1006,7 +1020,7 @@ static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port, tunnel_cfg.encap_rcv = geneve_udp_encap_recv; tunnel_cfg.encap_err_lookup = geneve_udp_encap_err_lookup; tunnel_cfg.encap_destroy = NULL; - setup_udp_tunnel_sock(net, sock, &tunnel_cfg); + setup_udp_tunnel_sock(net, sock->sk, &tunnel_cfg); list_add(&gs->list, &gn->sock_list); return gs; } @@ -1018,7 +1032,7 @@ static void __geneve_sock_release(struct geneve_sock *gs) list_del(&gs->list); udp_tunnel_notify_del_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE); - udp_tunnel_sock_release(gs->sock); + udp_tunnel_sock_release(gs->sock->sk); kfree_rcu(gs, rcu); } diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 5150f2e4f66be..a60ef32b35b82 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -885,8 +885,8 @@ static void gtp_encap_disable_sock(struct sock *sk) static void gtp_encap_disable(struct gtp_dev *gtp) { if (gtp->sk_created) { - udp_tunnel_sock_release(gtp->sk0->sk_socket); - udp_tunnel_sock_release(gtp->sk1u->sk_socket); + udp_tunnel_sock_release(gtp->sk0); + udp_tunnel_sock_release(gtp->sk1u); gtp->sk_created = false; gtp->sk0 = NULL; gtp->sk1u = NULL; @@ -1434,7 +1434,7 @@ static struct sock *gtp_create_sock(int type, struct gtp_dev *gtp, tuncfg.encap_rcv = gtp_encap_recv; tuncfg.encap_destroy = NULL; - setup_udp_tunnel_sock(net, sock, &tuncfg); + setup_udp_tunnel_sock(net, sock->sk, &tuncfg); return sock->sk; } @@ -1451,7 +1451,7 @@ static int gtp_create_sockets(struct gtp_dev *gtp, const struct nlattr *nla, sk1u = gtp_create_sock(UDP_ENCAP_GTP1U, gtp, nla, family); if (IS_ERR(sk1u)) { - udp_tunnel_sock_release(sk0->sk_socket); + udp_tunnel_sock_release(sk0); return PTR_ERR(sk1u); } @@ -1689,7 +1689,7 @@ static struct sock *gtp_encap_enable_socket(int fd, int type, tuncfg.encap_rcv = gtp_encap_recv; tuncfg.encap_destroy = gtp_encap_destroy; - setup_udp_tunnel_sock(sock_net(sock->sk), sock, &tuncfg); + setup_udp_tunnel_sock(sock_net(sock->sk), sk, &tuncfg); out_rel_sock: release_sock(sock->sk); diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c index ed4178155a5d6..01af4f9cf7f29 100644 --- a/drivers/net/ieee802154/ca8210.c +++ b/drivers/net/ieee802154/ca8210.c @@ -595,7 +595,7 @@ static int ca8210_test_int_driver_write( fifo_buffer = kmemdup(buf, len, GFP_KERNEL); if (!fifo_buffer) return -ENOMEM; - kfifo_in(&test->up_fifo, &fifo_buffer, 4); + kfifo_in(&test->up_fifo, &fifo_buffer, sizeof(fifo_buffer)); wake_up_interruptible(&priv->test.readq); return 0; @@ -919,9 +919,10 @@ static int ca8210_spi_transfer( if (status < 0) { dev_crit( &spi->dev, - "status %d from spi_sync in write\n", + "status %d from spi_async in write\n", status ); + kfree(cas_ctl); } return status; @@ -2525,6 +2526,7 @@ static ssize_t ca8210_test_int_user_read( struct ca8210_priv *priv = filp->private_data; unsigned char *fifo_buffer; unsigned long bytes_not_copied; + unsigned int copied; if (filp->f_flags & O_NONBLOCK) { /* Non-blocking mode */ @@ -2538,7 +2540,8 @@ static ssize_t ca8210_test_int_user_read( ); } - if (kfifo_out(&priv->test.up_fifo, &fifo_buffer, 4) != 4) { + copied = kfifo_out(&priv->test.up_fifo, &fifo_buffer, sizeof(fifo_buffer)); + if (copied != sizeof(fifo_buffer)) { dev_err( &priv->spi->dev, "test_interface: Wrong number of elements popped from upstream fifo\n" diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c index 2f0ccdd937ccb..331c00ad02c05 100644 --- a/drivers/net/ipa/ipa_smp2p.c +++ b/drivers/net/ipa/ipa_smp2p.c @@ -232,19 +232,27 @@ ipa_smp2p_init(struct ipa *ipa, struct platform_device *pdev, bool modem_init) &valid_bit); if (IS_ERR(valid_state)) return PTR_ERR(valid_state); - if (valid_bit >= 32) /* BITS_PER_U32 */ - return -EINVAL; + if (valid_bit >= 32) { /* BITS_PER_U32 */ + ret = -EINVAL; + goto err_valid_state_put; + } enabled_state = qcom_smem_state_get(dev, "ipa-clock-enabled", &enabled_bit); - if (IS_ERR(enabled_state)) - return PTR_ERR(enabled_state); - if (enabled_bit >= 32) /* BITS_PER_U32 */ - return -EINVAL; + if (IS_ERR(enabled_state)) { + ret = PTR_ERR(enabled_state); + goto err_valid_state_put; + } + if (enabled_bit >= 32) { /* BITS_PER_U32 */ + ret = -EINVAL; + goto err_enabled_state_put; + } smp2p = kzalloc_obj(*smp2p); - if (!smp2p) - return -ENOMEM; + if (!smp2p) { + ret = -ENOMEM; + goto err_enabled_state_put; + } smp2p->ipa = ipa; @@ -289,6 +297,10 @@ ipa_smp2p_init(struct ipa *ipa, struct platform_device *pdev, bool modem_init) ipa->smp2p = NULL; mutex_destroy(&smp2p->mutex); kfree(smp2p); +err_enabled_state_put: + qcom_smem_state_put(enabled_state); +err_valid_state_put: + qcom_smem_state_put(valid_state); return ret; } @@ -305,6 +317,8 @@ void ipa_smp2p_exit(struct ipa *ipa) ipa_smp2p_power_release(ipa); ipa->smp2p = NULL; mutex_destroy(&smp2p->mutex); + qcom_smem_state_put(smp2p->enabled_state); + qcom_smem_state_put(smp2p->valid_state); kfree(smp2p); } diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index fb009120a9241..ee0e2eb7dbc61 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -646,7 +646,7 @@ static struct sk_buff *macsec_encrypt(struct sk_buff *skb, } unprotected_len = skb->len; - eth = eth_hdr(skb); + eth = skb_eth_hdr(skb); sci_present = macsec_send_sci(secy); hh = skb_push(skb, macsec_extra_len(sci_present)); memmove(hh, eth, 2 * ETH_ALEN); @@ -3615,19 +3615,22 @@ static int macsec_dev_open(struct net_device *dev) ops = macsec_get_ops(netdev_priv(dev), &ctx); if (!ops) { err = -EOPNOTSUPP; - goto clear_allmulti; + goto clear_promisc; } ctx.secy = &macsec->secy; err = macsec_offload(ops->mdo_dev_open, &ctx); if (err) - goto clear_allmulti; + goto clear_promisc; } if (netif_carrier_ok(real_dev)) netif_carrier_on(dev); return 0; +clear_promisc: + if (dev->flags & IFF_PROMISC) + dev_set_promiscuity(real_dev, -1); clear_allmulti: if (dev->flags & IFF_ALLMULTI) dev_set_allmulti(real_dev, -1); diff --git a/drivers/net/mdio/Kconfig b/drivers/net/mdio/Kconfig index c71132f33f847..733fa0b1cd3ae 100644 --- a/drivers/net/mdio/Kconfig +++ b/drivers/net/mdio/Kconfig @@ -122,7 +122,8 @@ config MDIO_MVUSB config MDIO_MSCC_MIIM tristate "Microsemi MIIM interface support" - depends on HAS_IOMEM && REGMAP_MMIO + depends on HAS_IOMEM + select REGMAP_MMIO help This driver supports the MIIM (MDIO) interface found in the network switches of the Microsemi SoCs; it is recommended to switch on diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 57dd6821a8aa9..bfab0a47678c9 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -184,8 +184,10 @@ struct netconsole_target { bool extended; bool release; struct netpoll np; - /* protected by target_list_lock */ - char buf[MAX_PRINT_CHUNK]; + /* protected by target_list_lock; +1 gives scnprintf() room for its + * NUL terminator so a full MAX_PRINT_CHUNK payload is not truncated + */ + char buf[MAX_PRINT_CHUNK + 1]; struct work_struct resume_wq; }; @@ -1692,7 +1694,7 @@ static void send_msg_no_fragmentation(struct netconsole_target *nt, if (release_len) { release = init_utsname()->release; - scnprintf(nt->buf, MAX_PRINT_CHUNK, "%s,%.*s", release, + scnprintf(nt->buf, sizeof(nt->buf), "%s,%.*s", release, msg_len, msg); msg_len += release_len; } else { @@ -1701,12 +1703,12 @@ static void send_msg_no_fragmentation(struct netconsole_target *nt, if (userdata) msg_len += scnprintf(&nt->buf[msg_len], - MAX_PRINT_CHUNK - msg_len, "%s", + sizeof(nt->buf) - msg_len, "%s", userdata); if (sysdata) msg_len += scnprintf(&nt->buf[msg_len], - MAX_PRINT_CHUNK - msg_len, "%s", + sizeof(nt->buf) - msg_len, "%s", sysdata); send_udp(nt, nt->buf, msg_len); diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c index 8811aa9eedeba..7f69e8890b5b5 100644 --- a/drivers/net/ovpn/udp.c +++ b/drivers/net/ovpn/udp.c @@ -399,7 +399,7 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock, if (!old_data) { /* socket is currently unused - we can take it */ rcu_read_unlock(); - setup_udp_tunnel_sock(sock_net(ovpn_sock->sk), sock, &cfg); + setup_udp_tunnel_sock(sock_net(ovpn_sock->sk), sock->sk, &cfg); return 0; } diff --git a/drivers/net/pfcp.c b/drivers/net/pfcp.c index 28e6bc4a1f14c..5559bac41a9e2 100644 --- a/drivers/net/pfcp.c +++ b/drivers/net/pfcp.c @@ -104,7 +104,7 @@ static int pfcp_encap_recv(struct sock *sk, struct sk_buff *skb) static void pfcp_del_sock(struct pfcp_dev *pfcp) { - udp_tunnel_sock_release(pfcp->sock); + udp_tunnel_sock_release(pfcp->sock->sk); pfcp->sock = NULL; } @@ -148,6 +148,7 @@ static void pfcp_link_setup(struct net_device *dev) dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; dev->priv_flags |= IFF_NO_QUEUE; + dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; netif_keep_dst(dev); } @@ -172,7 +173,7 @@ static struct socket *pfcp_create_sock(struct pfcp_dev *pfcp) tuncfg.encap_rcv = pfcp_encap_recv; tuncfg.encap_type = 1; - setup_udp_tunnel_sock(net, sock, &tuncfg); + setup_udp_tunnel_sock(net, sock->sk, &tuncfg); return sock; } diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c index 79c867ef64dad..01bbddeb4a306 100644 --- a/drivers/net/phy/realtek/realtek_main.c +++ b/drivers/net/phy/realtek/realtek_main.c @@ -1632,7 +1632,8 @@ static int rtl822x_config_aneg(struct phy_device *phydev) ret = phy_modify_mmd_changed(phydev, MDIO_MMD_VEND2, RTL_MDIO_AN_10GBT_CTRL, MDIO_AN_10GBT_CTRL_ADV2_5G | - MDIO_AN_10GBT_CTRL_ADV5G, adv); + MDIO_AN_10GBT_CTRL_ADV5G | + MDIO_AN_10GBT_CTRL_ADV10G, adv); if (ret < 0) return ret; } diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c index b94b9c433a219..c47536b456e38 100644 --- a/drivers/net/phy/sfp.c +++ b/drivers/net/phy/sfp.c @@ -852,6 +852,7 @@ static int sfp_i2c_mdiobus_create(struct sfp *sfp) static void sfp_i2c_mdiobus_destroy(struct sfp *sfp) { mdiobus_unregister(sfp->i2c_mii); + mdiobus_free(sfp->i2c_mii); sfp->i2c_mii = NULL; } diff --git a/drivers/net/pse-pd/pse_core.c b/drivers/net/pse-pd/pse_core.c index 69dbdbde9d71a..a5e6d7b26b9fd 100644 --- a/drivers/net/pse-pd/pse_core.c +++ b/drivers/net/pse-pd/pse_core.c @@ -1367,7 +1367,7 @@ static void __pse_control_release(struct kref *kref) if (psec->pcdev->pi[psec->id].admin_state_enabled) regulator_disable(psec->ps); - devm_regulator_put(psec->ps); + regulator_put(psec->ps); module_put(psec->pcdev->owner); @@ -1436,8 +1436,8 @@ pse_control_get_internal(struct pse_controller_dev *pcdev, unsigned int index, goto free_psec; pcdev->pi[index].admin_state_enabled = ret; - psec->ps = devm_regulator_get_exclusive(pcdev->dev, - rdev_get_name(pcdev->pi[index].rdev)); + psec->ps = regulator_get_exclusive(pcdev->dev, + rdev_get_name(pcdev->pi[index].rdev)); if (IS_ERR(psec->ps)) { ret = PTR_ERR(psec->ps); goto put_module; diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index 7aae5d915a1eb..ac016890646cf 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -787,8 +787,12 @@ static bool tbnet_check_frame(struct tbnet *net, const struct tbnet_frame *tf, return true; } - /* Start of packet, validate the frame header */ - if (frame_count == 0 || frame_count > TBNET_RING_SIZE / 4) { + /* Start of packet, validate the frame header. tbnet_poll() puts the + * first frame in the skb linear area and every further frame in a page + * fragment, so a packet may not span more than MAX_SKB_FRAGS + 1 frames + * without overflowing skb_shinfo()->frags[]. + */ + if (frame_count == 0 || frame_count > MAX_SKB_FRAGS + 1) { net->stats.rx_length_errors++; return false; } diff --git a/drivers/net/usb/gl620a.c b/drivers/net/usb/gl620a.c index 0bfa37c140591..09afd137b64e1 100644 --- a/drivers/net/usb/gl620a.c +++ b/drivers/net/usb/gl620a.c @@ -104,6 +104,9 @@ static int genelink_rx_fixup(struct usbnet *dev, struct sk_buff *skb) return 0; } + if (!skb_pull(skb, size + 4)) + return 0; + // allocate the skb for the individual packet gl_skb = alloc_skb(size, GFP_ATOMIC); if (gl_skb) { @@ -116,9 +119,6 @@ static int genelink_rx_fixup(struct usbnet *dev, struct sk_buff *skb) // advance to the next packet packet = (struct gl_packet *)&packet->packet_data[size]; count--; - - // shift the data pointer to the next gl_packet - skb_pull(skb, size + 4); } // skip the packet length field 4 bytes diff --git a/drivers/net/usb/kalmia.c b/drivers/net/usb/kalmia.c index ee9c48f7f68f9..0dd0a30c3db41 100644 --- a/drivers/net/usb/kalmia.c +++ b/drivers/net/usb/kalmia.c @@ -276,6 +276,14 @@ kalmia_rx_fixup(struct usbnet *dev, struct sk_buff *skb) "Received header: %6phC. Package length: %i\n", header_start, skb->len - KALMIA_HEADER_LENGTH); + /* both framing headers must be present before we subtract + * them, otherwise usb_packet_length underflows and the + * device-supplied ether_packet_length drives an out of bounds + * access below + */ + if (skb->len < 2 * KALMIA_HEADER_LENGTH) + return 0; + /* subtract start header and end header */ usb_packet_length = skb->len - (2 * KALMIA_HEADER_LENGTH); ether_packet_length = get_unaligned_le16(&header_start[2]); diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index bcf293ea1bd38..cb782d81d84f2 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -1452,6 +1452,15 @@ static inline u32 lan78xx_hash(char addr[ETH_ALEN]) return (ether_crc(ETH_ALEN, addr) >> 23) & 0x1ff; } +static int lan78xx_write_mchash_table(struct lan78xx_net *dev) +{ + struct lan78xx_priv *pdata = (struct lan78xx_priv *)(dev->data[0]); + + return lan78xx_dataport_write(dev, DP_SEL_RSEL_VLAN_DA_, + DP_SEL_VHF_VLAN_LEN, + DP_SEL_VHF_HASH_LEN, pdata->mchash_table); +} + static void lan78xx_deferred_multicast_write(struct work_struct *param) { struct lan78xx_priv *pdata = @@ -1462,9 +1471,7 @@ static void lan78xx_deferred_multicast_write(struct work_struct *param) netif_dbg(dev, drv, dev->net, "deferred multicast write 0x%08x\n", pdata->rfe_ctl); - ret = lan78xx_dataport_write(dev, DP_SEL_RSEL_VLAN_DA_, - DP_SEL_VHF_VLAN_LEN, - DP_SEL_VHF_HASH_LEN, pdata->mchash_table); + ret = lan78xx_write_mchash_table(dev); if (ret < 0) goto multicast_write_done; @@ -1492,6 +1499,17 @@ static void lan78xx_deferred_multicast_write(struct work_struct *param) return; } +static void lan78xx_update_vlan_filter(struct lan78xx_priv *pdata, + struct net_device *netdev, + netdev_features_t features) +{ + if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && + !(netdev->flags & IFF_PROMISC)) + pdata->rfe_ctl |= RFE_CTL_VLAN_FILTER_; + else + pdata->rfe_ctl &= ~RFE_CTL_VLAN_FILTER_; +} + static void lan78xx_set_multicast(struct net_device *netdev) { struct lan78xx_net *dev = netdev_priv(netdev); @@ -1526,6 +1544,8 @@ static void lan78xx_set_multicast(struct net_device *netdev) } } + lan78xx_update_vlan_filter(pdata, dev->net, dev->net->features); + if (netdev_mc_count(dev->net)) { struct netdev_hw_addr *ha; int i; @@ -1557,6 +1577,7 @@ static void lan78xx_set_multicast(struct net_device *netdev) } static void lan78xx_rx_urb_submit_all(struct lan78xx_net *dev); +static int lan78xx_write_vlan_table(struct lan78xx_net *dev); static int lan78xx_mac_reset(struct lan78xx_net *dev) { @@ -2514,6 +2535,17 @@ static void lan78xx_mac_link_up(struct phylink_config *config, if (ret < 0) goto link_up_fail; + /* The RFE clears the VLAN/DA hash filter (VHF) on a link down/up + * cycle, so reprogram both tables from their shadow copies. + */ + ret = lan78xx_write_vlan_table(dev); + if (ret < 0) + goto link_up_fail; + + ret = lan78xx_write_mchash_table(dev); + if (ret < 0) + goto link_up_fail; + netif_start_queue(net); return; @@ -3055,24 +3087,27 @@ static int lan78xx_set_features(struct net_device *netdev, else pdata->rfe_ctl &= ~RFE_CTL_VLAN_STRIP_; - if (features & NETIF_F_HW_VLAN_CTAG_FILTER) - pdata->rfe_ctl |= RFE_CTL_VLAN_FILTER_; - else - pdata->rfe_ctl &= ~RFE_CTL_VLAN_FILTER_; + lan78xx_update_vlan_filter(pdata, netdev, features); spin_unlock_irqrestore(&pdata->rfe_ctl_lock, flags); return lan78xx_write_reg(dev, RFE_CTL, pdata->rfe_ctl); } +static int lan78xx_write_vlan_table(struct lan78xx_net *dev) +{ + struct lan78xx_priv *pdata = (struct lan78xx_priv *)(dev->data[0]); + + return lan78xx_dataport_write(dev, DP_SEL_RSEL_VLAN_DA_, 0, + DP_SEL_VHF_VLAN_LEN, pdata->vlan_table); +} + static void lan78xx_deferred_vlan_write(struct work_struct *param) { struct lan78xx_priv *pdata = container_of(param, struct lan78xx_priv, set_vlan); - struct lan78xx_net *dev = pdata->dev; - lan78xx_dataport_write(dev, DP_SEL_RSEL_VLAN_DA_, 0, - DP_SEL_VHF_VLAN_LEN, pdata->vlan_table); + lan78xx_write_vlan_table(pdata->dev); } static int lan78xx_vlan_rx_add_vid(struct net_device *netdev, diff --git a/drivers/net/usb/net1080.c b/drivers/net/usb/net1080.c index 5d4a1fd2b5244..19f6e1222d931 100644 --- a/drivers/net/usb/net1080.c +++ b/drivers/net/usb/net1080.c @@ -381,7 +381,7 @@ static int net1080_rx_fixup(struct usbnet *dev, struct sk_buff *skb) skb_trim(skb, skb->len - sizeof *trailer); if ((packet_len & 0x01) == 0) { - if (skb->data [packet_len] != PAD_BYTE) { + if (packet_len >= skb->len || skb->data[packet_len] != PAD_BYTE) { dev->net->stats.rx_frame_errors++; netdev_dbg(dev->net, "bad pad\n"); return 0; diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 0cfb19b760dd5..1c51421491753 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1137,6 +1137,8 @@ static int veth_enable_xdp_range(struct net_device *dev, int start, int end, err_reg_mem: xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); err_rxq_reg: + if (!napi_already_on) + netif_napi_del(&priv->rq[i].xdp_napi); for (i--; i >= start; i--) { struct veth_rq *rq = &priv->rq[i]; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index f4adcfee7a800..3e2a5876c6c8c 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -1999,15 +1999,18 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) { struct page *page = buf; + unsigned long max_len; struct sk_buff *skb; + max_len = (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE - + sizeof(struct padded_vnet_hdr) + vi->hdr_len; + /* Make sure that len does not exceed the size allocated in * add_recvbuf_big. */ - if (unlikely(len > (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE)) { + if (unlikely(len > max_len)) { pr_debug("%s: rx error: len %u exceeds allocated size %lu\n", - dev->name, len, - (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE); + dev->name, len, max_len); goto err; } @@ -3008,6 +3011,9 @@ static int virtnet_poll(struct napi_struct *napi, int budget) unsigned int xdp_xmit = 0; bool napi_complete; + if (budget) + virtqueue_disable_cb(rq->vq); + virtnet_poll_cleantx(rq, budget); received = virtnet_receive(rq, budget, &xdp_xmit); @@ -6222,6 +6228,19 @@ static void virtnet_free_irq_moder(struct virtnet_info *vi) rtnl_unlock(); } +static netdev_features_t virtnet_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ + /* Inner csum offload is only available for GSO packets. */ + if (skb->encapsulation && + (!skb_is_gso(skb) || netif_needs_gso(skb, features))) + return features & ~NETIF_F_CSUM_MASK; + + /* Passthru. */ + return features; +} + static const struct net_device_ops virtnet_netdev = { .ndo_open = virtnet_open, .ndo_stop = virtnet_close, @@ -6235,7 +6254,7 @@ static const struct net_device_ops virtnet_netdev = { .ndo_bpf = virtnet_xdp, .ndo_xdp_xmit = virtnet_xdp_xmit, .ndo_xsk_wakeup = virtnet_xsk_wakeup, - .ndo_features_check = passthru_features_check, + .ndo_features_check = virtnet_features_check, .ndo_get_phys_port_name = virtnet_get_phys_port_name, .ndo_set_features = virtnet_set_features, .ndo_tx_timeout = virtnet_tx_timeout, diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index b5b1253ac08ba..dae82a4d6027f 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -657,14 +657,18 @@ static struct vxlanhdr *vxlan_gro_prepare_receive(struct sock *sk, struct sk_buff *skb, struct gro_remcsum *grc) { - struct sk_buff *p; struct vxlanhdr *vh, *vh2; unsigned int hlen, off_vx; - struct vxlan_sock *vs = rcu_dereference_sk_user_data(sk); + struct vxlan_sock *vs; + struct sk_buff *p; __be32 flags; skb_gro_remcsum_init(grc); + vs = rcu_dereference_sk_user_data(sk); + if (!vs) + return NULL; + off_vx = skb_gro_offset(skb); hlen = off_vx + sizeof(*vh); vh = skb_gro_header(skb, hlen, off_vx); @@ -1519,13 +1523,13 @@ static void vxlan_sock_release(struct vxlan_dev *vxlan) vxlan_vs_del_dev(vxlan); if (__vxlan_sock_release_prep(sock4)) { - udp_tunnel_sock_release(sock4->sock); + udp_tunnel_sock_release(sock4->sock->sk); kfree(sock4); } #if IS_ENABLED(CONFIG_IPV6) if (__vxlan_sock_release_prep(sock6)) { - udp_tunnel_sock_release(sock6->sock); + udp_tunnel_sock_release(sock6->sock->sk); kfree(sock6); } #endif @@ -3620,7 +3624,7 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, bool ipv6, tunnel_cfg.gro_complete = vxlan_gro_complete; } - setup_udp_tunnel_sock(net, sock, &tunnel_cfg); + setup_udp_tunnel_sock(net, sock->sk, &tunnel_cfg); return vs; } diff --git a/drivers/net/wan/hdlc_ppp.c b/drivers/net/wan/hdlc_ppp.c index 159295c4bd6d8..302ed27944e7c 100644 --- a/drivers/net/wan/hdlc_ppp.c +++ b/drivers/net/wan/hdlc_ppp.c @@ -619,7 +619,6 @@ static void ppp_start(struct net_device *dev) struct proto *proto = &ppp->protos[i]; proto->dev = dev; - timer_setup(&proto->timer, ppp_timer, 0); proto->state = CLOSED; } ppp->protos[IDX_LCP].pid = PID_LCP; @@ -639,6 +638,15 @@ static void ppp_close(struct net_device *dev) ppp_tx_flush(); } +static void ppp_timer_release(struct net_device *dev) +{ + struct ppp *ppp = get_ppp(dev); + int i; + + for (i = 0; i < IDX_COUNT; i++) + timer_shutdown_sync(&ppp->protos[i].timer); +} + static struct hdlc_proto proto = { .start = ppp_start, .stop = ppp_stop, @@ -647,6 +655,7 @@ static struct hdlc_proto proto = { .ioctl = ppp_ioctl, .netif_rx = ppp_rx, .module = THIS_MODULE, + .detach = ppp_timer_release, }; static const struct header_ops ppp_header_ops = { @@ -657,7 +666,7 @@ static int ppp_ioctl(struct net_device *dev, struct if_settings *ifs) { hdlc_device *hdlc = dev_to_hdlc(dev); struct ppp *ppp; - int result; + int i, result; switch (ifs->type) { case IF_GET_PROTO: @@ -685,6 +694,8 @@ static int ppp_ioctl(struct net_device *dev, struct if_settings *ifs) return result; ppp = get_ppp(dev); + for (i = 0; i < IDX_COUNT; i++) + timer_setup(&ppp->protos[i].timer, ppp_timer, 0); spin_lock_init(&ppp->lock); ppp->req_timeout = 2; ppp->cr_retries = 10; diff --git a/drivers/net/wan/ixp4xx_hss.c b/drivers/net/wan/ixp4xx_hss.c index 720c5dc889ea3..7f4645ff90aa9 100644 --- a/drivers/net/wan/ixp4xx_hss.c +++ b/drivers/net/wan/ixp4xx_hss.c @@ -1487,11 +1487,11 @@ static int ixp4xx_hss_probe(struct platform_device *pdev) "unable to get CLK internal GPIO\n"); ndev = alloc_hdlcdev(port); - port->netdev = alloc_hdlcdev(port); - if (!port->netdev) { + if (!ndev) { err = -ENOMEM; goto err_plat; } + port->netdev = ndev; SET_NETDEV_DEV(ndev, &pdev->dev); hdlc = dev_to_hdlc(ndev); diff --git a/drivers/net/wireguard/socket.c b/drivers/net/wireguard/socket.c index c362c78d908eb..0028ef17dc716 100644 --- a/drivers/net/wireguard/socket.c +++ b/drivers/net/wireguard/socket.c @@ -335,7 +335,7 @@ static void sock_free(struct sock *sock) if (unlikely(!sock)) return; sk_clear_memalloc(sock); - udp_tunnel_sock_release(sock->sk_socket); + udp_tunnel_sock_release(sock); } static void set_sock_opts(struct socket *sock) @@ -389,14 +389,14 @@ int wg_socket_init(struct wg_device *wg, u16 port) goto out; } set_sock_opts(new4); - setup_udp_tunnel_sock(net, new4, &cfg); + setup_udp_tunnel_sock(net, new4->sk, &cfg); #if IS_ENABLED(CONFIG_IPV6) if (ipv6_mod_enabled()) { port6.local_udp_port = inet_sk(new4->sk)->inet_sport; ret = udp_sock_create(net, &port6, &new6); if (ret < 0) { - udp_tunnel_sock_release(new4); + udp_tunnel_sock_release(new4->sk); if (ret == -EADDRINUSE && !port && retries++ < 100) goto retry; pr_err("%s: Could not create IPv6 socket\n", @@ -404,7 +404,7 @@ int wg_socket_init(struct wg_device *wg, u16 port) goto out; } set_sock_opts(new6); - setup_udp_tunnel_sock(net, new6, &cfg); + setup_udp_tunnel_sock(net, new6->sk, &cfg); } #endif diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c index bbb86f1651419..5a50b623bd07b 100644 --- a/drivers/net/wireless/ath/ath11k/dp.c +++ b/drivers/net/wireless/ath/ath11k/dp.c @@ -1040,6 +1040,7 @@ void ath11k_dp_free(struct ath11k_base *ab) idr_destroy(&dp->tx_ring[i].txbuf_idr); spin_unlock_bh(&dp->tx_ring[i].tx_idr_lock); kfree(dp->tx_ring[i].tx_status); + dp->tx_ring[i].tx_status = NULL; } /* Deinit any SOC level resource */ diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c index f994233df2bb9..a6c9ff112c68f 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.c +++ b/drivers/net/wireless/ath/ath11k/mhi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -282,8 +282,10 @@ static void ath11k_mhi_op_status_cb(struct mhi_controller *mhi_cntrl, break; } + spin_lock_bh(&ab->base_lock); if (!(test_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags))) queue_work(ab->workqueue_aux, &ab->reset_work); + spin_unlock_bh(&ab->base_lock); break; default: diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c index 7114eca8810db..35bb9e7a63a20 100644 --- a/drivers/net/wireless/ath/ath11k/pci.c +++ b/drivers/net/wireless/ath/ath11k/pci.c @@ -1210,6 +1210,14 @@ static void ath11k_pci_shutdown(struct pci_dev *pdev) struct ath11k_pci *ab_pci = ath11k_pci_priv(ab); ath11k_pci_set_irq_affinity_hint(ab_pci, NULL); + + spin_lock_bh(&ab->base_lock); + set_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags); + spin_unlock_bh(&ab->base_lock); + + cancel_work_sync(&ab->reset_work); + cancel_work_sync(&ab->dump_work); + ath11k_pci_power_down(ab, false); } diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c index 2dcf0a52e4c15..30733a244454e 100644 --- a/drivers/net/wireless/ath/ath12k/ahb.c +++ b/drivers/net/wireless/ath/ath12k/ahb.c @@ -583,31 +583,36 @@ static int ath12k_ahb_config_ext_irq(struct ath12k_base *ab) netif_napi_add(irq_grp->napi_ndev, &irq_grp->napi, ath12k_ahb_ext_grp_napi_poll); - for (j = 0; j < ATH12K_EXT_IRQ_NUM_MAX; j++) { - /* For TX ring, ensure that the ring mask and the - * tcl_to_wbm_rbm_map point to the same ring number. - */ + for (j = 0; j < DP_TCL_NUM_RING_MAX; j++) { if (ring_mask->tx[i] & - BIT(ab->hal.tcl_to_wbm_rbm_map[j].wbm_ring_num)) { + BIT(ab->hal.tcl_to_wbm_rbm_map[j].wbm_ring_num) && + num_irq < ATH12K_EXT_IRQ_NUM_MAX) { irq_grp->irqs[num_irq++] = wbm2host_tx_completions_ring1 - j; } + } - if (ring_mask->rx[i] & BIT(j)) { + for (j = 0; j < ATH12K_EXT_IRQ_NUM_MAX; j++) { + if (ring_mask->rx[i] & BIT(j) && + num_irq < ATH12K_EXT_IRQ_NUM_MAX) { irq_grp->irqs[num_irq++] = reo2host_destination_ring1 - j; } - if (ring_mask->rx_err[i] & BIT(j)) + if (ring_mask->rx_err[i] & BIT(j) && + num_irq < ATH12K_EXT_IRQ_NUM_MAX) irq_grp->irqs[num_irq++] = reo2host_exception; - if (ring_mask->rx_wbm_rel[i] & BIT(j)) + if (ring_mask->rx_wbm_rel[i] & BIT(j) && + num_irq < ATH12K_EXT_IRQ_NUM_MAX) irq_grp->irqs[num_irq++] = wbm2host_rx_release; - if (ring_mask->reo_status[i] & BIT(j)) + if (ring_mask->reo_status[i] & BIT(j) && + num_irq < ATH12K_EXT_IRQ_NUM_MAX) irq_grp->irqs[num_irq++] = reo2host_status; - if (ring_mask->rx_mon_dest[i] & BIT(j)) + if (ring_mask->rx_mon_dest[i] & BIT(j) && + num_irq < ATH12K_EXT_IRQ_NUM_MAX) irq_grp->irqs[num_irq++] = rxdma2host_monitor_destination_mac1; } diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c index 90802ed1aa59f..af5f11fc1d84a 100644 --- a/drivers/net/wireless/ath/ath12k/dp.c +++ b/drivers/net/wireless/ath/ath12k/dp.c @@ -943,11 +943,11 @@ void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif) dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id); - dp_link_vif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | - u32_encode_bits(arvif->vdev_id, - HTT_TCL_META_DATA_VDEV_ID) | - u32_encode_bits(ar->pdev->pdev_id, - HTT_TCL_META_DATA_PDEV_ID); + dp_link_vif->tcl_metadata = u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | + u32_encode_bits(arvif->vdev_id, + HTT_TCL_META_DATA_VDEV_ID) | + u32_encode_bits(ar->pdev->pdev_id, + HTT_TCL_META_DATA_PDEV_ID); /* set HTT extension valid bit to 0 by default */ dp_link_vif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT; diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c index 737287a9aa462..44c5cff75f169 100644 --- a/drivers/net/wireless/ath/ath12k/dp_mon.c +++ b/drivers/net/wireless/ath/ath12k/dp_mon.c @@ -115,13 +115,14 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k_pdev_dp *dp_pdev, bool is_cck; pkt_type = ppdu_info->preamble_type; - rate_mcs = ppdu_info->rate; + rate_mcs = ppdu_info->mcs; nss = ppdu_info->nss; sgi = ppdu_info->gi; switch (pkt_type) { case RX_MSDU_START_PKT_TYPE_11A: case RX_MSDU_START_PKT_TYPE_11B: + rate_mcs = ppdu_info->rate; is_cck = (pkt_type == RX_MSDU_START_PKT_TYPE_11B); if (rx_status->band < NUM_NL80211_BANDS) { struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); @@ -471,13 +472,10 @@ void ath12k_dp_mon_update_radiotap(struct ath12k_pdev_dp *dp_pdev, rxs->encoding = RX_ENC_HE; ptr = skb_push(mon_skb, sizeof(struct ieee80211_radiotap_he)); ath12k_dp_mon_rx_update_radiotap_he(ppduinfo, ptr); - rxs->rate_idx = ppduinfo->rate; } else if (ppduinfo->vht_flags) { rxs->encoding = RX_ENC_VHT; - rxs->rate_idx = ppduinfo->rate; } else if (ppduinfo->ht_flags) { rxs->encoding = RX_ENC_HT; - rxs->rate_idx = ppduinfo->rate; } else { struct ath12k *ar; diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index 2cff9485c95af..7190aafd3ae69 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -8054,16 +8054,16 @@ int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw, continue; arvif = wiphy_dereference(hw->wiphy, ahvif->link[link_id]); - arsta = ath12k_mac_alloc_assign_link_sta(ah, ahsta, ahvif, link_id); + if (!arvif || !arvif->is_created) + continue; - if (!arvif || !arsta) { + arsta = ath12k_mac_alloc_assign_link_sta(ah, ahsta, ahvif, link_id); + if (!arsta) { ath12k_hw_warn(ah, "Failed to alloc/assign link sta"); continue; } ar = arvif->ar; - if (!ar) - continue; ret = ath12k_mac_station_add(ar, arvif, arsta); if (ret) { @@ -8399,6 +8399,10 @@ ath12k_create_vht_cap(struct ath12k *ar, u32 rate_cap_tx_chainmask, vht_cap.vht_supported = 1; vht_cap.cap = ar->pdev->cap.vht_cap; + if (ar->pdev->cap.nss_ratio_enabled) + vht_cap.vht_mcs.tx_highest |= + cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE); + ath12k_set_vht_txbf_cap(ar, &vht_cap.cap); /* 80P80 is not supported */ @@ -10293,7 +10297,7 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) if (ret) { ath12k_warn(ab, "failed to create WMI vdev %d: %d\n", arvif->vdev_id, ret); - return ret; + goto err; } ar->num_created_vdevs++; @@ -10440,13 +10444,13 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) if (ret) { ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n", arvif->vdev_id, arvif->bssid); - goto err; + goto err_dp_peer_del; } ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id, arvif->bssid); if (ret) - goto err_vdev_del; + goto err_dp_peer_del; ar->num_peers--; } @@ -10463,8 +10467,6 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) ath12k_wmi_vdev_delete(ar, arvif->vdev_id); ar->num_created_vdevs--; - arvif->is_created = false; - arvif->ar = NULL; ar->allocated_vdev_map &= ~(1LL << arvif->vdev_id); ab->free_vdev_map |= 1LL << arvif->vdev_id; ab->free_vdev_stats_id_map &= ~(1LL << arvif->vdev_stats_id); @@ -10473,6 +10475,7 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) spin_unlock_bh(&ar->data_lock); err: + arvif->is_created = false; arvif->ar = NULL; return ret; } diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c index 375277ca2b892..d9a22d6afbb02 100644 --- a/drivers/net/wireless/ath/ath12k/pci.c +++ b/drivers/net/wireless/ath/ath12k/pci.c @@ -1639,7 +1639,7 @@ static int ath12k_pci_probe(struct pci_dev *pdev, ret = ab_pci->device_family_ops->arch_init(ab); if (ret) { ath12k_err(ab, "PCI arch_init failed %d\n", ret); - goto err_pci_msi_free; + goto err_free_irq; } ret = ath12k_core_init(ab); diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c index 945680b3ebdfc..a5e290edaa898 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c @@ -1028,8 +1028,10 @@ static int ath12k_wifi7_dp_rx_h_verify_tkip_mic(struct ath12k_pdev_dp *dp_pdev, skb_pull(msdu, hal_rx_desc_sz); if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, msdu, - rx_info))) + rx_info))) { + dev_kfree_skb_any(msdu); return -EINVAL; + } ath12k_dp_rx_h_ppdu(dp_pdev, rx_info); ath12k_dp_rx_h_undecap(dp_pdev, msdu, HAL_ENCRYPT_TYPE_TKIP_MIC, true, diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c index 0ac9212e42f75..957646b2df4de 100644 --- a/drivers/net/wireless/ath/ath9k/xmit.c +++ b/drivers/net/wireless/ath/ath9k/xmit.c @@ -2746,6 +2746,11 @@ void ath_tx_edma_tasklet(struct ath_softc *sc) continue; } + if (ts.qid >= ATH9K_NUM_TX_QUEUES) { + ath_dbg(common, XMIT, "invalid qid %d\n", ts.qid); + continue; + } + txq = &sc->tx.txq[ts.qid]; ath_txq_lock(sc, txq); diff --git a/drivers/net/wireless/ath/wcn36xx/smd.c b/drivers/net/wireless/ath/wcn36xx/smd.c index 813553edcb789..c0b477345832b 100644 --- a/drivers/net/wireless/ath/wcn36xx/smd.c +++ b/drivers/net/wireless/ath/wcn36xx/smd.c @@ -2599,6 +2599,9 @@ static int wcn36xx_smd_trigger_ba_rsp(void *buf, int len, struct add_ba_info *ba if (rsp->candidate_cnt < 1) return rsp->status ? rsp->status : -EINVAL; + if (len < sizeof(*rsp) + sizeof(*candidate)) + return -EINVAL; + candidate = (struct wcn36xx_hal_trigger_ba_rsp_candidate *)(buf + sizeof(*rsp)); for (i = 0; i < STACFG_MAX_TC; i++) { @@ -2805,6 +2808,12 @@ static int wcn36xx_smd_print_reg_info_ind(struct wcn36xx *wcn, return -EIO; } + if (rsp->count > (len - sizeof(*rsp)) / sizeof(rsp->regs[0])) { + wcn36xx_warn("Truncated print reg info indication: count %u, len %zu\n", + rsp->count, len); + return -EIO; + } + wcn36xx_dbg(WCN36XX_DBG_HAL, "reginfo indication, scenario: 0x%x reason: 0x%x\n", rsp->scenario, rsp->reason); @@ -3293,6 +3302,10 @@ int wcn36xx_smd_rsp_process(struct rpmsg_device *rpdev, case WCN36XX_HAL_EXIT_IMPS_RSP: case WCN36XX_HAL_UPDATE_CHANNEL_LIST_RSP: case WCN36XX_HAL_ADD_BCN_FILTER_RSP: + if (len > WCN36XX_HAL_BUF_SIZE) { + wcn36xx_warn("HAL response too large: %d\n", len); + break; + } memcpy(wcn->hal_buf, buf, len); wcn->hal_rsp_len = len; complete(&wcn->hal_rsp_compl); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c index ce09d44fa73cf..873754be5174b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c @@ -293,6 +293,12 @@ brcmf_notify_auth_frame_rx(struct brcmf_if *ifp, return -EINVAL; } + if (mgmt_frame_len < offsetof(struct ieee80211_mgmt, u)) { + bphy_err(drvr, "Event %s (%d) frame too small. Ignore\n", + brcmf_fweh_event_name(e->event_code), e->event_code); + return -EINVAL; + } + wdev = &ifp->vif->wdev; WARN_ON(!wdev); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/agg.c b/drivers/net/wireless/intel/iwlwifi/mld/agg.c index 3bf36f8f68742..e3627ad0321c8 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/agg.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/agg.c @@ -64,6 +64,9 @@ static void iwl_mld_release_frames_from_notif(struct iwl_mld *mld, } /* pick any STA ID to find the pointer */ + if (WARN_ON_ONCE(!ba_data->sta_mask)) + goto out_unlock; + sta_id = ffs(ba_data->sta_mask) - 1; link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]); if (WARN_ON_ONCE(IS_ERR_OR_NULL(link_sta) || !link_sta->sta)) @@ -166,6 +169,9 @@ void iwl_mld_del_ba(struct iwl_mld *mld, int queue, goto out_unlock; /* pick any STA ID to find the pointer */ + if (WARN_ON_ONCE(!ba_data->sta_mask)) + goto out_unlock; + sta_id = ffs(ba_data->sta_mask) - 1; link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]); if (WARN_ON_ONCE(IS_ERR_OR_NULL(link_sta) || !link_sta->sta)) @@ -347,6 +353,9 @@ static void iwl_mld_rx_agg_session_expired(struct timer_list *t) } /* timer expired, pick any STA ID to find the pointer */ + if (WARN_ON_ONCE(!ba_data->sta_mask)) + goto unlock; + sta_id = ffs(ba_data->sta_mask) - 1; link_sta = rcu_dereference(ba_data->mld->fw_id_to_link_sta[sta_id]); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c index c65f4b56a3270..20ae338e5696d 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c @@ -321,10 +321,10 @@ void iwl_mld_ptp_remove(struct iwl_mld *mld) mld->ptp_data.ptp_clock_info.name, ptp_clock_index(mld->ptp_data.ptp_clock)); + cancel_delayed_work_sync(&mld->ptp_data.dwork); ptp_clock_unregister(mld->ptp_data.ptp_clock); mld->ptp_data.ptp_clock = NULL; mld->ptp_data.last_gp2 = 0; mld->ptp_data.wrap_counter = 0; - cancel_delayed_work_sync(&mld->ptp_data.dwork); } } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c index f7b620136c857..e05e531b09f0b 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c @@ -325,11 +325,11 @@ void iwl_mvm_ptp_remove(struct iwl_mvm *mvm) mvm->ptp_data.ptp_clock_info.name, ptp_clock_index(mvm->ptp_data.ptp_clock)); + cancel_delayed_work_sync(&mvm->ptp_data.dwork); ptp_clock_unregister(mvm->ptp_data.ptp_clock); mvm->ptp_data.ptp_clock = NULL; memset(&mvm->ptp_data.ptp_clock_info, 0, sizeof(mvm->ptp_data.ptp_clock_info)); mvm->ptp_data.last_gp2 = 0; - cancel_delayed_work_sync(&mvm->ptp_data.dwork); } } diff --git a/drivers/net/wireless/marvell/libertas_tf/main.c b/drivers/net/wireless/marvell/libertas_tf/main.c index fb20fe31cd363..42be6fa22f9c6 100644 --- a/drivers/net/wireless/marvell/libertas_tf/main.c +++ b/drivers/net/wireless/marvell/libertas_tf/main.c @@ -174,7 +174,7 @@ static void lbtf_free_adapter(struct lbtf_private *priv) { lbtf_deb_enter(LBTF_DEB_MAIN); lbtf_free_cmd_buffer(priv); - timer_delete(&priv->command_timer); + timer_delete_sync(&priv->command_timer); lbtf_deb_leave(LBTF_DEB_MAIN); } diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c index c9a651bdf8822..ca04aaae1e2f3 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -4333,7 +4333,7 @@ mwifiex_cfg80211_authenticate(struct wiphy *wiphy, return -EOPNOTSUPP; } - if (!priv->auth_flag) { + if (!(priv->auth_flag & HOST_MLME_AUTH_PENDING)) { ret = mwifiex_remain_on_chan_cfg(priv, HostCmd_ACT_GEN_SET, req->bss->channel, AUTH_TX_DEFAULT_WAIT_TIME); diff --git a/drivers/net/wireless/marvell/mwifiex/join.c b/drivers/net/wireless/marvell/mwifiex/join.c index 5a1a0287c1d58..b48f7febaf03f 100644 --- a/drivers/net/wireless/marvell/mwifiex/join.c +++ b/drivers/net/wireless/marvell/mwifiex/join.c @@ -736,7 +736,6 @@ int mwifiex_ret_802_11_associate(struct mwifiex_private *priv, /* Send a Media Connected event, according to the Spec */ priv->media_connected = true; - priv->adapter->ps_state = PS_STATE_AWAKE; priv->adapter->pps_uapsd_mode = false; priv->adapter->tx_lock_flag = false; diff --git a/drivers/net/wireless/mediatek/mt76/channel.c b/drivers/net/wireless/mediatek/mt76/channel.c index 05eee64706ea8..6edcb3b8f2798 100644 --- a/drivers/net/wireless/mediatek/mt76/channel.c +++ b/drivers/net/wireless/mediatek/mt76/channel.c @@ -307,7 +307,7 @@ void mt76_put_vif_phy_link(struct mt76_phy *phy, struct ieee80211_vif *vif, rcu_assign_pointer(mvif->offchannel_link, NULL); dev->drv->vif_link_remove(phy, vif, &vif->bss_conf, mlink); - kfree(mlink); + kfree_rcu(mlink, rcu_head); } void mt76_roc_complete(struct mt76_phy *phy) diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 4ae5e4715a9c5..13c4e8abe2819 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -449,6 +449,8 @@ mt76_phy_init(struct mt76_phy *phy, struct ieee80211_hw *hw) wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_AIRTIME_FAIRNESS); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_AQL); + if (!wiphy->max_remain_on_channel_duration) + wiphy->max_remain_on_channel_duration = 5000; if (!wiphy->available_antennas_tx) wiphy->available_antennas_tx = phy->antenna_mask; if (!wiphy->available_antennas_rx) @@ -1321,7 +1323,7 @@ mt76_check_ccmp_pn(struct sk_buff *skb) * All further fragments will be validated by mac80211 only. */ if (ieee80211_is_frag(hdr) && - !ieee80211_is_first_frag(hdr->frame_control)) + !ieee80211_is_first_frag(hdr->seq_ctrl)) return; } @@ -1576,6 +1578,7 @@ mt76_sta_add(struct mt76_phy *phy, struct ieee80211_vif *vif, { struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; struct mt76_dev *dev = phy->dev; + struct mt76_wcid *published; int ret; int i; @@ -1595,11 +1598,19 @@ mt76_sta_add(struct mt76_phy *phy, struct ieee80211_vif *vif, mtxq->wcid = wcid->idx; } - ewma_signal_init(&wcid->rssi); - rcu_assign_pointer(dev->wcid[wcid->idx], wcid); + published = rcu_dereference_protected(dev->wcid[wcid->idx], + lockdep_is_held(&dev->mutex)); + if (published != wcid) { + WARN_ON_ONCE(published); + ewma_signal_init(&wcid->rssi); + rcu_assign_pointer(dev->wcid[wcid->idx], wcid); + mt76_wcid_init(wcid, phy->band_idx); + } else { + wcid->phy_idx = phy->band_idx; + } + phy->num_sta++; - mt76_wcid_init(wcid, phy->band_idx); out: mutex_unlock(&dev->mutex); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/init.c b/drivers/net/wireless/mediatek/mt76/mt7615/init.c index 42e11ba1206ee..e437e088b2e91 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/init.c @@ -195,7 +195,6 @@ mt7615_check_offload_capability(struct mt7615_dev *dev) ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS); wiphy->flags &= ~WIPHY_FLAG_4ADDR_STATION; - wiphy->max_remain_on_channel_duration = 5000; wiphy->features |= NL80211_FEATURE_SCHED_SCAN_RANDOM_MAC_ADDR | NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR | WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL | diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c b/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c index 01cb3b2830f35..459c4044f0d10 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c @@ -16,6 +16,7 @@ static const struct usb_device_id mt76x2u_device_table[] = { { USB_DEVICE(0x0e8d, 0x7612) }, /* Aukey USBAC1200 - Alfa AWUS036ACM */ { USB_DEVICE(0x057c, 0x8503) }, /* Avm FRITZ!WLAN AC860 */ { USB_DEVICE(0x7392, 0xb711) }, /* Edimax EW 7722 UAC */ + { USB_DEVICE(0x056e, 0x400a) }, /* ELECOM WDC-867SU3S */ { USB_DEVICE(0x0e8d, 0x7632) }, /* HC-M7662BU1 */ { USB_DEVICE(0x0471, 0x2126) }, /* LiteOn WN4516R module, nonstandard USB connector */ { USB_DEVICE(0x0471, 0x7600) }, /* LiteOn WN4519R module, nonstandard USB connector */ diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index cec2c4208255f..334c19ab2b22c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -912,16 +912,16 @@ mt7915_mac_tx_free(struct mt7915_dev *dev, void *data, int len) } if (!mtk_wed_device_active(&mdev->mmio.wed) && wcid) { - u32 tx_retries = 0, tx_failed = 0; + u32 tx_retries = 0, tx_failed = 0, count; if (v3 && (info & MT_TX_FREE_MPDU_HEADER_V3)) { - tx_retries = - FIELD_GET(MT_TX_FREE_COUNT_V3, info) - 1; + count = FIELD_GET(MT_TX_FREE_COUNT_V3, info); + tx_retries = count ? count - 1 : 0; tx_failed = tx_retries + !!FIELD_GET(MT_TX_FREE_STAT_V3, info); } else if (!v3 && (info & MT_TX_FREE_MPDU_HEADER)) { - tx_retries = - FIELD_GET(MT_TX_FREE_COUNT, info) - 1; + count = FIELD_GET(MT_TX_FREE_COUNT, info); + tx_retries = count ? count - 1 : 0; tx_failed = tx_retries + !!FIELD_GET(MT_TX_FREE_STAT, info); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index 03b4960db73f0..668bfa1953807 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -530,8 +530,9 @@ static void mt7921_mac_tx_free(struct mt792x_dev *dev, void *data, int len) stat = FIELD_GET(MT_TX_FREE_STATUS, info); if (wcid) { - wcid->stats.tx_retries += - FIELD_GET(MT_TX_FREE_COUNT, info) - 1; + u32 count = FIELD_GET(MT_TX_FREE_COUNT, info); + + wcid->stats.tx_retries += count ? count - 1 : 0; wcid->stats.tx_failed += !!stat; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 3d74fabe74085..a326f4c95c7c8 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -1508,6 +1508,9 @@ static void mt7921_channel_switch_rx_beacon(struct ieee80211_hw *hw, struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; u16 beacon_interval = vif->bss_conf.beacon_int; + if (!dev->new_ctx) + return; + if (cfg80211_chandef_identical(&chsw->chandef, &dev->new_ctx->def) && chsw->count) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index 7a790ddf43bb6..49a37185f0564 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -343,11 +343,14 @@ static int mt7921_pci_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, mdev); + dev = container_of(mdev, struct mt792x_dev, mt76); + regs = pcim_iomap_region(pdev, 0, pci_name(pdev)); - if (IS_ERR(regs)) - return PTR_ERR(regs); + if (IS_ERR(regs)) { + ret = PTR_ERR(regs); + goto err_free_dev; + } - dev = container_of(mdev, struct mt792x_dev, mt76); dev->fw_features = features; dev->hif_ops = &mt7921_pcie_ops; dev->irq_map = &irq_map; @@ -359,8 +362,10 @@ static int mt7921_pci_probe(struct pci_dev *pdev, /* MT7902 needs a mutable copy because wm2_complete_mask differs */ map = devm_kmemdup(&pdev->dev, &irq_map, sizeof(irq_map), GFP_KERNEL); - if (!map) - return -ENOMEM; + if (!map) { + ret = -ENOMEM; + goto err_free_dev; + } map->rx.wm2_complete_mask = 0; dev->irq_map = map; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index c47bd812b66b9..50034d7c04f0c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -841,7 +841,6 @@ static void mt7925_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb, { struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); struct ieee80211_link_sta *link_sta; - struct mt792x_link_sta *mlink; struct mt792x_sta *msta; bool is_8023; u16 fc, tid; @@ -880,14 +879,14 @@ static void mt7925_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb, msta = (struct mt792x_sta *)sta->drv_priv; - if (sta->mlo && msta->deflink_id != IEEE80211_LINK_UNSPECIFIED) - mlink = rcu_dereference(msta->link[msta->deflink_id]); - else - mlink = &msta->deflink; - - if (!test_and_set_bit(tid, &mlink->wcid.ampdu_state)) { + /* Packets belonging to the same TID can be transmitted over multiple + * links. Keep the TX BA session state in the primary link so all links + * share the same AMPDU bookkeeping. + */ + if (!test_and_set_bit(tid, &msta->deflink.wcid.ampdu_state)) { if (ieee80211_start_tx_ba_session(sta, tid, 0)) - clear_bit(tid, &mlink->wcid.ampdu_state); + clear_bit(tid, &msta->deflink.wcid.ampdu_state); + } } @@ -1141,8 +1140,9 @@ mt7925_mac_tx_free(struct mt792x_dev *dev, void *data, int len) if (info & MT_TXFREE_INFO_HEADER) { if (wcid) { - wcid->stats.tx_retries += - FIELD_GET(MT_TXFREE_INFO_COUNT, info) - 1; + u32 count = FIELD_GET(MT_TXFREE_INFO_COUNT, info); + + wcid->stats.tx_retries += count ? count - 1 : 0; wcid->stats.tx_failed += !!FIELD_GET(MT_TXFREE_INFO_STAT, info); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 73d3722739d0a..8765121b916a2 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -1265,6 +1265,9 @@ mt7925_mac_sta_remove_links(struct mt792x_dev *dev, struct ieee80211_vif *vif, if (vif->type == NL80211_IFTYPE_AP) break; + if (vif->type == NL80211_IFTYPE_STATION && sta->tdls) + continue; + link_sta = mt792x_sta_to_link_sta(vif, sta, link_id); if (!link_sta) continue; @@ -2179,9 +2182,9 @@ mt7925_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, rcu_assign_pointer(mvif->link_conf[link_id], NULL); rcu_assign_pointer(mvif->sta.link[link_id], NULL); - if (mconf != &mvif->bss_conf) + if (mconfs[link_id] != &mvif->bss_conf) devm_kfree(dev->mt76.dev, mconfs[link_id]); - if (mlink != &mvif->sta.deflink) + if (mlinks[link_id] != &mvif->sta.deflink) devm_kfree(dev->mt76.dev, mlinks[link_id]); } @@ -2400,6 +2403,9 @@ static void mt7925_channel_switch_rx_beacon(struct ieee80211_hw *hw, if (ieee80211_vif_is_mld(vif)) return; + if (!dev->new_ctx) + return; + beacon_interval = vif->bss_conf.beacon_int; if (cfg80211_chandef_identical(&chsw->chandef, diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c index c4161754c01df..48837723024d7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c @@ -415,10 +415,12 @@ static int mt7925_pci_probe(struct pci_dev *pdev, ret = mt7925_register_device(dev); if (ret) - goto err_free_irq; + goto err_free_dma; return 0; +err_free_dma: + mt792x_dma_cleanup(dev); err_free_irq: devm_free_irq(&pdev->dev, pdev->irq, dev); err_free_dev: diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/testmode.c b/drivers/net/wireless/mediatek/mt76/mt7925/testmode.c index 3d40aacfc011f..22a8f1ddc321d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/testmode.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/testmode.c @@ -105,6 +105,11 @@ mt7925_tm_query(struct mt792x_dev *dev, struct mt7925_tm_cmd *req, if (ret) goto out; + if (skb->len < MT7925_EVT_RSP_LEN + 8) { + ret = -EINVAL; + goto out; + } + memcpy((char *)evt_resp, (char *)skb->data + 8, MT7925_EVT_RSP_LEN); out: diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_core.c b/drivers/net/wireless/mediatek/mt76/mt792x_core.c index 152cfcca2f908..dc63400c1b996 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_core.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_core.c @@ -105,7 +105,8 @@ void mt792x_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control, wcid = &mvif->sta.deflink.wcid; } - if (vif && control->sta && ieee80211_vif_is_mld(vif)) { + if (vif && control->sta && ieee80211_vif_is_mld(vif) && + !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP)) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; struct ieee80211_link_sta *link_sta; struct ieee80211_bss_conf *conf; @@ -657,7 +658,6 @@ int mt792x_init_wiphy(struct ieee80211_hw *hw) BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO) | BIT(NL80211_IFTYPE_P2P_DEVICE); - wiphy->max_remain_on_channel_duration = 5000; wiphy->max_scan_ie_len = MT76_CONNAC_SCAN_IE_LEN; wiphy->max_scan_ssids = 4; wiphy->max_sched_scan_plan_interval = diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c index 8f5d297dafce2..3d9353811a02e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c @@ -683,7 +683,7 @@ int mt7996_dma_init(struct mt7996_dev *dev) ret = mt7996_init_tx_queues(&dev->phy, MT_TXQ_ID(0), MT7996_NPU_TX_RING_SIZE, MT_TXQ_RING_BASE(0) + hif1_ofs, - NULL); + wed); else ret = mt7996_init_tx_queues(&dev->phy, MT_TXQ_ID(dev->mphy.band_idx), diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index e2a83da3a09c0..2a0511b2e4d29 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -757,6 +757,7 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, bool multicast = is_multicast_ether_addr(hdr->addr1); u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; __le16 fc = hdr->frame_control, sc = hdr->seq_ctrl; + struct ieee80211_vif *vif = info->control.vif; u16 seqno = le16_to_cpu(sc); bool hw_bigtk = false; u8 fc_type, fc_stype; @@ -819,7 +820,7 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, txwi[3] |= cpu_to_le32(MT_TXD3_REM_TX_COUNT); } - if (multicast && ieee80211_vif_is_mld(info->control.vif)) { + if (multicast && vif && ieee80211_vif_is_mld(vif)) { val = MT_TXD3_SN_VALID | FIELD_PREP(MT_TXD3_SEQ, IEEE80211_SEQ_TO_SN(seqno)); txwi[3] |= cpu_to_le32(val); @@ -839,12 +840,12 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, txwi[3] &= ~cpu_to_le32(MT_TXD3_HW_AMSDU); } - if (ieee80211_vif_is_mld(info->control.vif) && + if (vif && ieee80211_vif_is_mld(vif) && (multicast || unlikely(skb->protocol == cpu_to_be16(ETH_P_PAE)))) txwi[5] |= cpu_to_le32(MT_TXD5_FL); if (ieee80211_is_nullfunc(fc) && ieee80211_has_a4(fc) && - ieee80211_vif_is_mld(info->control.vif)) { + vif && ieee80211_vif_is_mld(vif)) { txwi[5] |= cpu_to_le32(MT_TXD5_FL); txwi[6] |= cpu_to_le32(MT_TXD6_DIS_MAT); } @@ -856,7 +857,8 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, - enum mt76_txq_id qid, u32 changed) + enum mt76_txq_id qid, u32 changed, + unsigned int link_id) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); @@ -866,7 +868,6 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, bool is_8023 = info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP; struct mt76_vif_link *mlink = NULL; struct mt7996_vif *mvif; - unsigned int link_id; u16 tx_count = 15; u32 val; bool inband_disc = !!(changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP | @@ -874,17 +875,11 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, bool beacon = !!(changed & (BSS_CHANGED_BEACON | BSS_CHANGED_BEACON_ENABLED)) && (!inband_disc); - if (wcid != &dev->mt76.global_wcid) - link_id = wcid->link_id; - else - link_id = u32_get_bits(info->control.flags, - IEEE80211_TX_CTRL_MLO_LINK); - mvif = vif ? (struct mt7996_vif *)vif->drv_priv : NULL; if (mvif) { if (wcid->offchannel) mlink = rcu_dereference(mvif->mt76.offchannel_link); - if (!mlink) + if (!mlink && link_id != IEEE80211_LINK_UNSPECIFIED) mlink = rcu_dereference(mvif->mt76.link[link_id]); } @@ -1067,11 +1062,11 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, link_conf = rcu_dereference(vif->link_conf[wcid->link_id]); if (!link_conf) - return -EINVAL; + goto error_release_token; link_sta = rcu_dereference(sta->link[wcid->link_id]); if (!link_sta) - return -EINVAL; + goto error_release_token; dma_sync_single_for_cpu(mdev->dma_dev, tx_info->buf[1].addr, tx_info->buf[1].len, DMA_TO_DEVICE); @@ -1096,7 +1091,7 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, /* Transmit non qos data by 802.11 header and need to fill txd by host*/ if (!is_8023 || pid >= MT_PACKET_ID_FIRST) mt7996_mac_write_txwi(dev, txwi_ptr, tx_info->skb, wcid, key, - pid, qid, 0); + pid, qid, 0, link_id); /* MT7996 and MT7992 require driver to provide the MAC TXP for AddBA * req @@ -1176,6 +1171,10 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, tx_info->nbuf = MT_CT_DMA_BUF_NUM; return 0; + +error_release_token: + mt76_token_release(mdev, id, NULL); + return -EINVAL; } u32 mt7996_wed_init_buf(void *ptr, dma_addr_t phys, int token_id) @@ -1360,13 +1359,13 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) cur_info++; continue; } else if (info & MT_TXFREE_INFO_HEADER) { - u32 tx_retries = 0, tx_failed = 0; + u32 tx_retries = 0, tx_failed = 0, count; if (!wcid) continue; - tx_retries = - FIELD_GET(MT_TXFREE_INFO_COUNT, info) - 1; + count = FIELD_GET(MT_TXFREE_INFO_COUNT, info); + tx_retries = count ? count - 1 : 0; tx_failed = tx_retries + !!FIELD_GET(MT_TXFREE_INFO_STAT, info); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index a8a6552d49f69..26b8c91db0a81 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -1959,7 +1959,11 @@ static void mt7996_sta_rate_ctrl_update(void *data, struct ieee80211_sta *sta) { struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; struct mt7996_sta_link *msta_link; - u32 *changed = data; + struct mt7996_vif *mvif = data; + u32 changed = IEEE80211_RC_SUPP_RATES_CHANGED; + + if (msta->vif != mvif) + return; msta_link = rcu_dereference(msta->link[msta->deflink_id]); if (msta_link) @@ -1972,7 +1976,6 @@ mt7996_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif, { struct mt7996_dev *dev = mt7996_hw_dev(hw); struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; - u32 changed = IEEE80211_RC_SUPP_RATES_CHANGED; mvif->deflink.bitrate_mask = *mask; @@ -1985,7 +1988,7 @@ mt7996_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif, * then multiple MCS setting (MCS 4,5,6) is not supported. */ ieee80211_iterate_stations_atomic(hw, mt7996_sta_rate_ctrl_update, - &changed); + mvif); ieee80211_queue_work(hw, &dev->rc_work); return 0; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index 16420375112d1..2748bfeb47974 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -3098,7 +3098,7 @@ mt7996_mcu_beacon_cont(struct mt7996_dev *dev, buf = (u8 *)bcn + sizeof(*bcn); mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, NULL, 0, 0, - BSS_CHANGED_BEACON); + BSS_CHANGED_BEACON, link_conf->link_id); memcpy(buf + MT_TXD_SIZE, skb->data, skb->len); } @@ -3244,7 +3244,8 @@ int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev, buf = (u8 *)tlv + sizeof(*discov); - mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, NULL, 0, 0, changed); + mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, NULL, 0, 0, + changed, link_conf->link_id); memcpy(buf + MT_TXD_SIZE, skb->data, skb->len); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index bdcf724579549..5f39a583bd372 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -855,7 +855,8 @@ void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band); void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, - enum mt76_txq_id qid, u32 changed); + enum mt76_txq_id qid, u32 changed, + unsigned int link_id); void mt7996_mac_update_beacons(struct mt7996_phy *phy); void mt7996_mac_set_coverage_class(struct mt7996_phy *phy); void mt7996_mac_work(struct work_struct *work); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c index 12523ddba6304..b7d9193e042f4 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c @@ -141,7 +141,7 @@ static int mt7996_pci_probe(struct pci_dev *pdev, dev->hif2 = hif2; mt76_npu_init(mdev, pci_resource_start(pdev, 0), - pdev->bus && pci_domain_nr(pdev->bus) ? 3 : 2); + pci_domain_nr(pdev->bus) ? 3 : 2); ret = mt7996_mmio_wed_init(dev, pdev, false, &irq); if (ret < 0) diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index 22f9690634c94..f96d9c4718535 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -933,7 +933,7 @@ mt76_token_release(struct mt76_dev *dev, int token, bool *wake) #endif } - if (dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR && + if (wake && dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR && dev->phy.q_tx[0]->blocked) *wake = true; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c index 82fb230a73bb7..4d94b7062f440 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c @@ -1388,7 +1388,7 @@ int rt2x00lib_probe_dev(struct rt2x00_dev *rt2x00dev) GFP_KERNEL); if (!rt2x00dev->drv_data) { retval = -ENOMEM; - goto exit; + return retval; } } @@ -1422,7 +1422,7 @@ int rt2x00lib_probe_dev(struct rt2x00_dev *rt2x00dev) alloc_ordered_workqueue("%s", 0, wiphy_name(rt2x00dev->hw->wiphy)); if (!rt2x00dev->workqueue) { retval = -ENOMEM; - goto exit; + goto exit_free_drv_data; } INIT_WORK(&rt2x00dev->intf_work, rt2x00lib_intf_scheduled); @@ -1494,6 +1494,14 @@ int rt2x00lib_probe_dev(struct rt2x00_dev *rt2x00dev) exit: rt2x00lib_remove_dev(rt2x00dev); + return retval; + +exit_free_drv_data: + clear_bit(DEVICE_STATE_PRESENT, &rt2x00dev->flags); + + kfree(rt2x00dev->drv_data); + rt2x00dev->drv_data = NULL; + return retval; } EXPORT_SYMBOL_GPL(rt2x00lib_probe_dev); diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8188e.c b/drivers/net/wireless/realtek/rtl8xxxu/8188e.c index 766a7a7c7d281..fea5aec9ced99 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8188e.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8188e.c @@ -1866,6 +1866,7 @@ struct rtl8xxxu_fileops rtl8188eu_fops = { .has_tx_report = 1, .init_reg_pkt_life_time = 1, .gen2_thermal_meter = 1, + .hw_feature_report = 0, .max_sec_cam_num = 32, .adda_1t_init = 0x0b1b25a0, .adda_1t_path_on = 0x0bdb25a0, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8188f.c b/drivers/net/wireless/realtek/rtl8xxxu/8188f.c index 3abf14d7044f3..5556aad1a7ad3 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8188f.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8188f.c @@ -1745,6 +1745,7 @@ struct rtl8xxxu_fileops rtl8188fu_fops = { .init_reg_rxfltmap = 1, .init_reg_pkt_life_time = 1, .init_reg_hmtfr = 1, + .hw_feature_report = 1, .ampdu_max_time = 0x70, .ustime_tsf_edca = 0x28, .max_aggr_num = 0x0c14, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8192c.c b/drivers/net/wireless/realtek/rtl8xxxu/8192c.c index 444872131c663..508322a8f3108 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8192c.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8192c.c @@ -723,6 +723,7 @@ struct rtl8xxxu_fileops rtl8192cu_fops = { .tx_desc_size = sizeof(struct rtl8xxxu_txdesc32), .rx_desc_size = sizeof(struct rtl8xxxu_rxdesc16), .supports_ap = 1, + .hw_feature_report = 0, .max_macid_num = 32, .max_sec_cam_num = 32, .adda_1t_init = 0x0b1b25a0, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8192e.c b/drivers/net/wireless/realtek/rtl8xxxu/8192e.c index 8e123bbfc6659..690ace3031503 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8192e.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8192e.c @@ -1751,6 +1751,7 @@ struct rtl8xxxu_fileops rtl8192eu_fops = { .has_s0s1 = 0, .gen2_thermal_meter = 1, .needs_full_init = 1, + .hw_feature_report = 0, .supports_ap = 1, .max_macid_num = 128, .max_sec_cam_num = 64, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8192f.c b/drivers/net/wireless/realtek/rtl8xxxu/8192f.c index cd2156b7a57ab..5f076da4b5506 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8192f.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8192f.c @@ -2074,6 +2074,7 @@ struct rtl8xxxu_fileops rtl8192fu_fops = { .init_reg_rxfltmap = 1, .init_reg_pkt_life_time = 1, .init_reg_hmtfr = 1, + .hw_feature_report = 1, .ampdu_max_time = 0x5e, .ustime_tsf_edca = 0x50, .max_aggr_num = 0x1f1f, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8710b.c b/drivers/net/wireless/realtek/rtl8xxxu/8710b.c index 11c63c320eaeb..a87e692bfc812 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8710b.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8710b.c @@ -1852,6 +1852,7 @@ struct rtl8xxxu_fileops rtl8710bu_fops = { .init_reg_rxfltmap = 1, .init_reg_pkt_life_time = 1, .init_reg_hmtfr = 1, + .hw_feature_report = 0, /* TODO, it's different */ .ampdu_max_time = 0x5e, /* * The RTL8710BU vendor driver uses 0x50 here and it works fine, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8723a.c b/drivers/net/wireless/realtek/rtl8xxxu/8723a.c index 4f4493d0bfc2d..18e038b82f490 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8723a.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8723a.c @@ -632,6 +632,7 @@ struct rtl8xxxu_fileops rtl8723au_fops = { .rx_agg_buf_size = 16000, .tx_desc_size = sizeof(struct rtl8xxxu_txdesc32), .rx_desc_size = sizeof(struct rtl8xxxu_rxdesc16), + .hw_feature_report = 0, .max_sec_cam_num = 32, .adda_1t_init = 0x0b1b25a0, .adda_1t_path_on = 0x0bdb25a0, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/8723b.c b/drivers/net/wireless/realtek/rtl8xxxu/8723b.c index cc2e60b06f647..e314ef991b388 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/8723b.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/8723b.c @@ -1746,6 +1746,7 @@ struct rtl8xxxu_fileops rtl8723bu_fops = { .gen2_thermal_meter = 1, .needs_full_init = 1, .init_reg_hmtfr = 1, + .hw_feature_report = 0, .ampdu_max_time = 0x5e, .ustime_tsf_edca = 0x50, .max_aggr_num = 0x0c14, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c index f20fade0c0990..40af324f5f0a3 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c @@ -14,6 +14,7 @@ */ #include +#include #include "regs.h" #include "rtl8xxxu.h" @@ -3915,6 +3916,46 @@ static inline u8 rtl8xxxu_get_macid(struct rtl8xxxu_priv *priv, return sta_info->macid; } +static void rtl8xxxu_request_hw_feature(struct rtl8xxxu_priv *priv) +{ + if (!priv->fops->hw_feature_report) + return; + + rtl8xxxu_write8(priv, REG_C2HEVT_MSG_NORMAL, C2H_HW_FEATURE_DUMP); +} + +static int rtl8xxxu_dump_hw_feature(struct rtl8xxxu_priv *priv) +{ + static const u8 bw_map[8] = { 0, 0, 160, 5, 10, 20, 40, 80 }; + struct rtl8xxxu_hw_feature *hw_feature = &priv->hw_feature; + u8 feature[13]; + int i, ret; + u8 id, bw; + + if (!priv->fops->hw_feature_report) { + hw_feature->max_bw = 40; + return 0; + } + + ret = read_poll_timeout(rtl8xxxu_read8, id, + id == C2H_HW_FEATURE_REPORT, + 10000, 800000, false, + priv, REG_C2HEVT_MSG_NORMAL); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(feature); i++) + feature[i] = rtl8xxxu_read8(priv, REG_C2HEVT_MSG_NORMAL + 2 + i); + + rtl8xxxu_write8(priv, REG_C2HEVT_MSG_NORMAL, 0); + + bw = u8_get_bits(feature[6], GENMASK(2, 0)); + + hw_feature->max_bw = bw_map[bw]; + + return 0; +} + static int rtl8xxxu_init_device(struct ieee80211_hw *hw) { struct rtl8xxxu_priv *priv = hw->priv; @@ -3961,6 +4002,8 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw) */ rtl8xxxu_write16(priv, REG_TRXFF_BNDY + 2, fops->trxff_boundary); + rtl8xxxu_request_hw_feature(priv); + for (int retry = 5; retry >= 0 ; retry--) { ret = rtl8xxxu_download_firmware(priv); dev_dbg(dev, "%s: download_firmware %i\n", __func__, ret); @@ -3976,6 +4019,12 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw) if (ret) goto exit; + ret = rtl8xxxu_dump_hw_feature(priv); + if (ret) { + dev_err(dev, "failed to dump hw feature\n"); + goto exit; + } + if (fops->phy_init_antenna_selection) fops->phy_init_antenna_selection(priv); @@ -7835,15 +7884,20 @@ static int rtl8xxxu_probe(struct usb_interface *interface, sband->ht_cap.ht_supported = true; sband->ht_cap.ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; sband->ht_cap.ampdu_density = IEEE80211_HT_MPDU_DENSITY_16; - sband->ht_cap.cap = IEEE80211_HT_CAP_SGI_20 | IEEE80211_HT_CAP_SGI_40 | - IEEE80211_HT_CAP_SUP_WIDTH_20_40; + sband->ht_cap.cap = IEEE80211_HT_CAP_SGI_20; + + if (priv->hw_feature.max_bw >= 40) { + sband->ht_cap.cap |= IEEE80211_HT_CAP_SGI_40; + sband->ht_cap.cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40; + } else { + dev_info(&udev->dev, "hardware doesn't support HT40\n"); + } + memset(&sband->ht_cap.mcs, 0, sizeof(sband->ht_cap.mcs)); sband->ht_cap.mcs.rx_mask[0] = 0xff; sband->ht_cap.mcs.rx_mask[4] = 0x01; - if (priv->rf_paths > 1) { + if (priv->rf_paths > 1) sband->ht_cap.mcs.rx_mask[1] = 0xff; - sband->ht_cap.cap |= IEEE80211_HT_CAP_SGI_40; - } sband->ht_cap.mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; hw->wiphy->bands[NL80211_BAND_2GHZ] = sband; diff --git a/drivers/net/wireless/realtek/rtl8xxxu/regs.h b/drivers/net/wireless/realtek/rtl8xxxu/regs.h index 0741db8d08bff..eafaef1438b97 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/regs.h +++ b/drivers/net/wireless/realtek/rtl8xxxu/regs.h @@ -447,6 +447,8 @@ /* 8188EU */ #define REG_32K_CTRL 0x0194 #define REG_C2HEVT_MSG_NORMAL 0x01a0 +#define C2H_HW_FEATURE_REPORT 0x19 +#define C2H_HW_FEATURE_DUMP 0xfd /* 8192EU/8723BU/8812 */ #define REG_C2HEVT_CMD_ID_8723B 0x01ae #define REG_C2HEVT_CLEAR 0x01af diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h index 9fb2583ffffc3..eeb18eb0e4c0f 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h @@ -1789,11 +1789,17 @@ struct rtl8xxxu_cfo_tracking { #define RTL8XXXU_BC_MC_MACID1 1 #define RTL8XXXU_MAX_SEC_CAM_NUM 64 +struct rtl8xxxu_hw_feature { + u8 max_bw; +}; + struct rtl8xxxu_priv { struct ieee80211_hw *hw; struct usb_device *udev; struct rtl8xxxu_fileops *fops; + struct rtl8xxxu_hw_feature hw_feature; + spinlock_t tx_urb_lock; struct list_head tx_urb_free_list; int tx_urb_free_count; @@ -2009,6 +2015,7 @@ struct rtl8xxxu_fileops { u8 init_reg_pkt_life_time:1; u8 init_reg_hmtfr:1; u8 supports_concurrent:1; + u8 hw_feature_report:1; u8 ampdu_max_time; u8 ustime_tsf_edca; u16 max_aggr_num; diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h index 1155365348f32..d5de09d75f451 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h @@ -291,7 +291,7 @@ static inline int get_rx_desc_paggr(__le32 *__pdesc) static inline int get_rx_status_desc_rpt_sel(__le32 *__pdesc) { - return le32_get_bits(*(__pdesc + 1), BIT(28)); + return le32_get_bits(*(__pdesc + 2), BIT(28)); } static inline int get_rx_desc_rxmcs(__le32 *__pdesc) diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index bba370ad510cf..c2bf44e880cf2 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -1077,6 +1077,11 @@ static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, * discard the frame if none available */ new_len = pkt_stat.pkt_len + pkt_offset; + if (unlikely(new_len > RTK_PCI_RX_BUF_SIZE)) { + rtw_dbg(rtwdev, RTW_DBG_RX, + "oversized RX packet: %u\n", new_len); + goto next_rp; + } new = dev_alloc_skb(new_len); if (WARN_ONCE(!new, "rx routine starvation\n")) goto next_rp; @@ -1711,9 +1716,9 @@ static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev) static pci_ers_result_t rtw_pci_io_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { - struct net_device *netdev = pci_get_drvdata(pdev); + struct ieee80211_hw *hw = pci_get_drvdata(pdev); - netif_device_detach(netdev); + ieee80211_stop_queues(hw); return PCI_ERS_RESULT_NEED_RESET; } @@ -1730,12 +1735,12 @@ static pci_ers_result_t rtw_pci_io_slot_reset(struct pci_dev *pdev) static void rtw_pci_io_resume(struct pci_dev *pdev) { - struct net_device *netdev = pci_get_drvdata(pdev); + struct ieee80211_hw *hw = pci_get_drvdata(pdev); /* ack any pending wake events, disable PME */ pci_enable_wake(pdev, PCI_D0, 0); - netif_device_attach(netdev); + ieee80211_wake_queues(hw); } const struct pci_error_handlers rtw_pci_err_handler = { diff --git a/drivers/net/wireless/realtek/rtw88/tx.c b/drivers/net/wireless/realtek/rtw88/tx.c index 3106edb84fb47..a6e43314a4e95 100644 --- a/drivers/net/wireless/realtek/rtw88/tx.c +++ b/drivers/net/wireless/realtek/rtw88/tx.c @@ -196,6 +196,7 @@ void rtw_tx_report_purge_timer(struct timer_list *t) void rtw_tx_report_enqueue(struct rtw_dev *rtwdev, struct sk_buff *skb, u8 sn) { struct rtw_tx_report *tx_report = &rtwdev->tx_report; + unsigned long timeout = RTW_TX_PROBE_TIMEOUT; unsigned long flags; u8 *drv_data; @@ -207,7 +208,11 @@ void rtw_tx_report_enqueue(struct rtw_dev *rtwdev, struct sk_buff *skb, u8 sn) __skb_queue_tail(&tx_report->queue, skb); spin_unlock_irqrestore(&tx_report->q_lock, flags); - mod_timer(&tx_report->purge_timer, jiffies + RTW_TX_PROBE_TIMEOUT); + if (rtwdev->chip->id == RTW_CHIP_TYPE_8723D && + rtwdev->hci.type == RTW_HCI_TYPE_USB) + timeout = msecs_to_jiffies(2500); + + mod_timer(&tx_report->purge_timer, jiffies + timeout); } EXPORT_SYMBOL(rtw_tx_report_enqueue); diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c index 718940ebba316..1a0bdbf52cb0d 100644 --- a/drivers/net/wireless/realtek/rtw88/usb.c +++ b/drivers/net/wireless/realtek/rtw88/usb.c @@ -399,6 +399,7 @@ static bool rtw_usb_tx_agg_skb(struct rtw_usb *rtwusb, struct sk_buff_head *list int agg_num = 0; unsigned int align_next = 0; u8 qsel; + int ret; if (skb_queue_empty(list)) return false; @@ -456,7 +457,13 @@ static bool rtw_usb_tx_agg_skb(struct rtw_usb *rtwusb, struct sk_buff_head *list tx_desc = (struct rtw_tx_desc *)skb_head->data; qsel = le32_get_bits(tx_desc->w1, RTW_TX_DESC_W1_QSEL); - rtw_usb_write_port(rtwdev, qsel, skb_head, rtw_usb_write_port_tx_complete, txcb); + ret = rtw_usb_write_port(rtwdev, qsel, skb_head, + rtw_usb_write_port_tx_complete, txcb); + if (ret) { + ieee80211_purge_tx_queue(rtwdev->hw, &txcb->tx_ack_queue); + kfree(txcb); + return false; + } return true; } @@ -518,8 +525,10 @@ static int rtw_usb_write_data(struct rtw_dev *rtwdev, ret = rtw_usb_write_port(rtwdev, qsel, skb, rtw_usb_write_port_complete, skb); - if (unlikely(ret)) + if (unlikely(ret)) { rtw_err(rtwdev, "failed to do USB write, ret=%d\n", ret); + dev_kfree_skb_any(skb); + } return ret; } diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index 70feab97dccb5..486c1774fb975 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -3363,7 +3363,8 @@ static bool rtw89_core_skb_pn_valid(struct rtw89_dev *rtwdev, last_pn = tid_stats->last_pn; if (pn > last_pn) { - if (ieee80211_sn_less(mpdu_sn, tid_stats->last_sn)) { + if (last_pn != -1LL && + ieee80211_sn_less(mpdu_sn, tid_stats->last_sn)) { dev_kfree_skb_any(skb); return false; diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index fd29dbbb120d2..05adf20a65be1 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -6492,6 +6492,9 @@ static inline void rtw89_assoc_link_clr(struct rtw89_sta_link *rtwsta_link) static inline struct rtw89_sta_link * rtw89_assoc_link_rcu_dereference(struct rtw89_dev *rtwdev, u8 macid) { + if (unlikely(macid >= RTW89_MAX_MAC_ID_NUM)) + return NULL; + return rcu_dereference(rtwdev->assoc_link_on_macid[macid]); } diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 17704f054727a..089c9071b58f5 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -8318,7 +8318,7 @@ static int rtw89_update_6ghz_rnr_chan_ax(struct rtw89_dev *rtwdev, struct sk_buff *skb; bool found; int ret = 0; - u8 i; + u32 i; if (!req->n_6ghz_params) return 0; diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index 501c3af1da01a..0672d13d13a64 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -964,7 +964,8 @@ static int rtw89_ops_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, rtw89_err(rtwdev, "failed to add key to sec cam\n"); return ret; } - rtw89_core_tid_rx_stats_reset(rtwdev); + if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) + rtw89_core_tid_rx_stats_reset(rtwdev); break; case DISABLE_KEY: flush_work(&rtwdev->txq_work); diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 43c61b3dc969f..64554eb35a72c 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -4624,9 +4624,9 @@ EXPORT_SYMBOL(rtw89_pm_ops); static pci_ers_result_t rtw89_pci_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { - struct net_device *netdev = pci_get_drvdata(pdev); + struct ieee80211_hw *hw = pci_get_drvdata(pdev); - netif_device_detach(netdev); + ieee80211_stop_queues(hw); return PCI_ERS_RESULT_NEED_RESET; } @@ -4643,12 +4643,12 @@ static pci_ers_result_t rtw89_pci_io_slot_reset(struct pci_dev *pdev) static void rtw89_pci_io_resume(struct pci_dev *pdev) { - struct net_device *netdev = pci_get_drvdata(pdev); + struct ieee80211_hw *hw = pci_get_drvdata(pdev); /* ack any pending wake events, disable PME */ pci_enable_wake(pdev, PCI_D0, 0); - netif_device_attach(netdev); + ieee80211_wake_queues(hw); } const struct pci_error_handlers rtw89_pci_err_handler = { diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c index 1c340a4a09308..be583ae331c03 100644 --- a/drivers/net/wireless/ti/wlcore/main.c +++ b/drivers/net/wireless/ti/wlcore/main.c @@ -32,6 +32,15 @@ #define WL1271_BOOT_RETRIES 3 #define WL1271_WAKEUP_TIMEOUT 500 +static const u32 cipher_suites[] = { + WLAN_CIPHER_SUITE_WEP40, + WLAN_CIPHER_SUITE_WEP104, + WLAN_CIPHER_SUITE_TKIP, + WLAN_CIPHER_SUITE_CCMP, + WL1271_CIPHER_SUITE_GEM, + WLAN_CIPHER_SUITE_AES_CMAC, +}; + static char *fwlog_param; static int fwlog_mem_blocks = -1; static int bug_on_recovery = -1; @@ -2367,6 +2376,7 @@ static int wl12xx_init_vif_data(struct wl1271 *wl, struct ieee80211_vif *vif) static int wl12xx_init_fw(struct wl1271 *wl) { + struct wlcore_platdev_data *pdev_data = dev_get_platdata(&wl->pdev->dev); int retries = WL1271_BOOT_RETRIES; bool booted = false; struct wiphy *wiphy = wl->hw->wiphy; @@ -2421,8 +2431,9 @@ static int wl12xx_init_fw(struct wl1271 *wl) /* WLAN_CIPHER_SUITE_AES_CMAC must be last in cipher_suites; support only with firmware 8.9.1 and newer */ - if (wl->chip.fw_ver[FW_VER_MAJOR] < 1) - wl->hw->wiphy->n_cipher_suites--; + if (wl->chip.fw_ver[FW_VER_MAJOR] < 1 || + (!strncmp(pdev_data->family->name, "wl12", 4))) + wl->hw->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites) - 1; /* * Now we know if 11a is supported (info from the NVS), so disable @@ -6198,14 +6209,6 @@ static void wl1271_unregister_hw(struct wl1271 *wl) static int wl1271_init_ieee80211(struct wl1271 *wl) { int i; - static const u32 cipher_suites[] = { - WLAN_CIPHER_SUITE_WEP40, - WLAN_CIPHER_SUITE_WEP104, - WLAN_CIPHER_SUITE_TKIP, - WLAN_CIPHER_SUITE_CCMP, - WL1271_CIPHER_SUITE_GEM, - WLAN_CIPHER_SUITE_AES_CMAC, - }; /* The tx descriptor buffer */ wl->hw->extra_tx_headroom = sizeof(struct wl1271_tx_hw_descr); diff --git a/drivers/net/wwan/iosm/iosm_ipc_mux_codec.c b/drivers/net/wwan/iosm/iosm_ipc_mux_codec.c index bff46f7ca59f3..0bbd41263cc2f 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_mux_codec.c +++ b/drivers/net/wwan/iosm/iosm_ipc_mux_codec.c @@ -553,19 +553,21 @@ static int mux_dl_process_dg(struct iosm_mux *ipc_mux, struct mux_adbh *adbh, u32 packet_offset, i, rc, dg_len; for (i = 0; i < nr_of_dg; i++, dg++) { - if (le32_to_cpu(dg->datagram_index) - < sizeof(struct mux_adbh)) + u32 dg_index = le32_to_cpu(dg->datagram_index); + + dg_len = le16_to_cpu(dg->datagram_length); + + if (dg_index < sizeof(struct mux_adbh)) goto dg_error; - /* Is the packet inside of the ADB */ - if (le32_to_cpu(dg->datagram_index) >= - le32_to_cpu(adbh->block_length)) { + /* Is the packet inside of the ADB and the received skb ? */ + if (dg_index >= le32_to_cpu(adbh->block_length) || + dg_index >= skb->len || + dg_len > skb->len - dg_index || + dl_head_pad_len >= dg_len) { goto dg_error; } else { - packet_offset = - le32_to_cpu(dg->datagram_index) + - dl_head_pad_len; - dg_len = le16_to_cpu(dg->datagram_length); + packet_offset = dg_index + dl_head_pad_len; /* Pass the packet to the netif layer. */ rc = ipc_mux_net_receive(ipc_mux, if_id, ipc_mux->wwan, packet_offset, @@ -589,12 +591,16 @@ static void mux_dl_adb_decode(struct iosm_mux *ipc_mux, struct mux_adbh *adbh; struct mux_adth *adth; int nr_of_dg, if_id; - u32 adth_index; + u32 adth_index, prev_index = 0; u8 *block; block = skb->data; adbh = (struct mux_adbh *)block; + /* The block header itself must fit in the received skb. */ + if (skb->len < sizeof(struct mux_adbh)) + goto adb_decode_err; + /* Process the aggregated datagram tables. */ adth_index = le32_to_cpu(adbh->first_table_index); @@ -606,6 +612,16 @@ static void mux_dl_adb_decode(struct iosm_mux *ipc_mux, /* Loop through mixed session tables. */ while (adth_index) { + /* The table header must lie within the received skb, and the + * chain must move forward so a modem cannot make the loop + * cycle between two tables. + */ + if (adth_index <= prev_index || + adth_index < sizeof(struct mux_adbh) || + adth_index > skb->len - sizeof(struct mux_adth)) + goto adb_decode_err; + prev_index = adth_index; + /* Get the reference to the table header. */ adth = (struct mux_adth *)(block + adth_index); @@ -629,6 +645,10 @@ static void mux_dl_adb_decode(struct iosm_mux *ipc_mux, if (le16_to_cpu(adth->table_length) < sizeof(struct mux_adth)) goto adb_decode_err; + /* The whole datagram table must fit in the received skb. */ + if (le16_to_cpu(adth->table_length) > skb->len - adth_index) + goto adb_decode_err; + /* Calculate the number of datagrams. */ nr_of_dg = (le16_to_cpu(adth->table_length) - sizeof(struct mux_adth)) / diff --git a/drivers/net/wwan/t7xx/t7xx_hif_cldma.c b/drivers/net/wwan/t7xx/t7xx_hif_cldma.c index e10cb4f9104e3..2917cee9b8028 100644 --- a/drivers/net/wwan/t7xx/t7xx_hif_cldma.c +++ b/drivers/net/wwan/t7xx/t7xx_hif_cldma.c @@ -1063,6 +1063,9 @@ static int t7xx_cldma_late_init(struct cldma_ctrl *md_ctrl) while (i--) t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE); + dma_pool_destroy(md_ctrl->gpd_dmapool); + md_ctrl->gpd_dmapool = NULL; + return ret; } diff --git a/drivers/net/wwan/t7xx/t7xx_port_wwan.c b/drivers/net/wwan/t7xx/t7xx_port_wwan.c index 7fc569565ff99..d2529df7592a7 100644 --- a/drivers/net/wwan/t7xx/t7xx_port_wwan.c +++ b/drivers/net/wwan/t7xx/t7xx_port_wwan.c @@ -106,6 +106,8 @@ static int t7xx_port_ctrl_tx(struct t7xx_port *port, struct sk_buff *skb) while (cur) { cloned = skb_clone(cur, GFP_KERNEL); + if (!cloned) + return cnt ? cnt : -ENOMEM; cloned->len = skb_headlen(cur); ret = t7xx_port_send_skb(port, cloned, 0, 0); if (ret) { diff --git a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c index d3ecf25a51625..200409b4e4ddf 100644 --- a/drivers/ntb/hw/epf/ntb_hw_epf.c +++ b/drivers/ntb/hw/epf/ntb_hw_epf.c @@ -6,6 +6,7 @@ * Author: Kishon Vijay Abraham I */ +#include #include #include #include @@ -69,8 +70,21 @@ enum epf_ntb_bar { NTB_BAR_NUM, }; +enum epf_irq_slot { + EPF_IRQ_LINK = 0, + EPF_IRQ_RESERVED_DB, /* Historically skipped slot */ + EPF_IRQ_DB_START, +}; + #define NTB_EPF_MAX_MW_COUNT (NTB_BAR_NUM - BAR_MW1) +struct ntb_epf_dev; + +struct ntb_epf_irq_ctx { + struct ntb_epf_dev *ndev; + unsigned int irq_no; +}; + struct ntb_epf_dev { struct ntb_dev ntb; struct device *dev; @@ -90,8 +104,9 @@ struct ntb_epf_dev { unsigned int self_spad; unsigned int peer_spad; - int db_val; + atomic64_t db_val; u64 db_valid_mask; + struct ntb_epf_irq_ctx irq_ctx[NTB_EPF_MAX_DB_COUNT + 1]; }; #define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb) @@ -315,16 +330,29 @@ static int ntb_epf_link_disable(struct ntb_dev *ntb) static irqreturn_t ntb_epf_vec_isr(int irq, void *dev) { - struct ntb_epf_dev *ndev = dev; - int irq_no; - - irq_no = irq - pci_irq_vector(ndev->ntb.pdev, 0); - ndev->db_val = irq_no + 1; + struct ntb_epf_irq_ctx *ctx = dev; + struct ntb_epf_dev *ndev = ctx->ndev; + unsigned int db_vector; + unsigned int irq_no = ctx->irq_no; - if (irq_no == 0) + if (irq_no == EPF_IRQ_LINK) { ntb_link_event(&ndev->ntb); - else - ntb_db_event(&ndev->ntb, irq_no); + } else if (irq_no == EPF_IRQ_RESERVED_DB) { + dev_warn_ratelimited(ndev->dev, + "Unexpected reserved doorbell slot IRQ received\n"); + } else { + db_vector = irq_no - EPF_IRQ_DB_START; + if (ndev->db_count < NTB_EPF_MIN_DB_COUNT || + db_vector >= ndev->db_count - 1) { + dev_warn_ratelimited(ndev->dev, + "Unexpected doorbell vector %u (db_count %u)\n", + db_vector, ndev->db_count); + return IRQ_HANDLED; + } + + atomic64_or(BIT_ULL(db_vector), &ndev->db_val); + ntb_db_event(&ndev->ntb, db_vector); + } return IRQ_HANDLED; } @@ -350,31 +378,30 @@ static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max) argument &= ~MSIX_ENABLE; } + ndev->db_count = irq - 1; for (i = 0; i < irq; i++) { + ndev->irq_ctx[i].ndev = ndev; + ndev->irq_ctx[i].irq_no = i; ret = request_irq(pci_irq_vector(pdev, i), ntb_epf_vec_isr, - 0, "ntb_epf", ndev); + 0, "ntb_epf", &ndev->irq_ctx[i]); if (ret) { dev_err(dev, "Failed to request irq\n"); - goto err_request_irq; + goto err_free_irq; } } - ndev->db_count = irq - 1; - ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL, argument | irq); if (ret) { dev_err(dev, "Failed to configure doorbell\n"); - goto err_configure_db; + goto err_free_irq; } return 0; -err_configure_db: - for (i = 0; i < ndev->db_count + 1; i++) - free_irq(pci_irq_vector(pdev, i), ndev); - -err_request_irq: +err_free_irq: + while (i--) + free_irq(pci_irq_vector(pdev, i), &ndev->irq_ctx[i]); pci_free_irq_vectors(pdev); return ret; @@ -499,7 +526,7 @@ static u64 ntb_epf_db_read(struct ntb_dev *ntb) { struct ntb_epf_dev *ndev = ntb_ndev(ntb); - return ndev->db_val; + return atomic64_read(&ndev->db_val); } static int ntb_epf_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) @@ -511,7 +538,7 @@ static int ntb_epf_db_clear(struct ntb_dev *ntb, u64 db_bits) { struct ntb_epf_dev *ndev = ntb_ndev(ntb); - ndev->db_val = 0; + atomic64_and(~db_bits, &ndev->db_val); return 0; } @@ -552,6 +579,12 @@ static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) struct device *dev = ndev->dev; int ret; + ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT); + if (ndev->mw_count > NTB_EPF_MAX_MW_COUNT) { + dev_err(dev, "Unsupported MW count: %u\n", ndev->mw_count); + return -EINVAL; + } + /* One Link interrupt and rest doorbell interrupt */ ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1, NTB_EPF_MAX_DB_COUNT + 1); @@ -560,15 +593,13 @@ static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) return ret; } - ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; - ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT); + /* + * ndev->db_count includes an extra skipped slot due to the legacy + * doorbell layout, hence -1. + */ + ndev->db_valid_mask = BIT_ULL(ndev->db_count - 1) - 1; ndev->spad_count = readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); - if (ndev->mw_count > NTB_EPF_MAX_MW_COUNT) { - dev_err(dev, "Unsupported MW count: %u\n", ndev->mw_count); - return -EINVAL; - } - return 0; } @@ -646,7 +677,8 @@ static void ntb_epf_deinit_pci(struct ntb_epf_dev *ndev) struct pci_dev *pdev = ndev->ntb.pdev; pci_iounmap(pdev, ndev->ctrl_reg); - pci_iounmap(pdev, ndev->peer_spad_reg); + if (ndev->barno_map[BAR_PEER_SPAD] != ndev->barno_map[BAR_CONFIG]) + pci_iounmap(pdev, ndev->peer_spad_reg); pci_iounmap(pdev, ndev->db_reg); pci_release_regions(pdev); @@ -662,7 +694,7 @@ static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev) ntb_epf_send_command(ndev, CMD_TEARDOWN_DOORBELL, ndev->db_count + 1); for (i = 0; i < ndev->db_count + 1; i++) - free_irq(pci_irq_vector(pdev, i), ndev); + free_irq(pci_irq_vector(pdev, i), &ndev->irq_ctx[i]); pci_free_irq_vectors(pdev); } diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 7cabc82305d6b..f59f926d4bfaa 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -223,6 +223,7 @@ struct ntb_transport_mw { void *alloc_addr; void *virt_addr; dma_addr_t dma_addr; + dma_addr_t original_dma_addr; }; struct ntb_transport_client_dev { @@ -765,8 +766,8 @@ static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) return; ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); - dma_free_coherent(dma_dev, mw->alloc_size, - mw->alloc_addr, mw->dma_addr); + dma_free_attrs(dma_dev, mw->alloc_size, mw->alloc_addr, + mw->original_dma_addr, DMA_ATTR_FORCE_CONTIGUOUS); mw->xlat_size = 0; mw->buff_size = 0; mw->alloc_size = 0; @@ -802,6 +803,7 @@ static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, return -ENOMEM; } virt_addr = alloc_addr; + mw->original_dma_addr = dma_addr; /* * we must ensure that the memory address allocated is BAR size @@ -826,8 +828,8 @@ static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, return 0; err: - dma_free_coherent(ntb_dev, mw->alloc_size, alloc_addr, dma_addr); - + dma_free_attrs(ntb_dev, mw->alloc_size, alloc_addr, dma_addr, + DMA_ATTR_FORCE_CONTIGUOUS); return rc; } diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index fdcb080a4314f..7e1112960d7f8 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -919,6 +919,9 @@ static int discover_arenas(struct btt *btt) return ret; out: + kfree(arena->freelist); + kfree(arena->rtt); + kfree(arena->map_locks); kfree(arena); free_arenas(btt); return ret; @@ -1589,7 +1592,7 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize, if (btt->init_state != INIT_READY && nd_region->ro) { dev_warn(dev, "%s is read-only, unable to init btt metadata\n", dev_name(&nd_region->dev)); - return NULL; + goto err; } else if (btt->init_state != INIT_READY) { btt->num_arenas = (rawsize / ARENA_MAX_SIZE) + ((rawsize % ARENA_MAX_SIZE) ? 1 : 0); @@ -1599,25 +1602,28 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize, ret = create_arenas(btt); if (ret) { dev_info(dev, "init: create_arenas: %d\n", ret); - return NULL; + goto err; } ret = btt_meta_init(btt); if (ret) { dev_err(dev, "init: error in meta_init: %d\n", ret); - return NULL; + goto err; } } ret = btt_blk_init(btt); if (ret) { dev_err(dev, "init: error in blk_init: %d\n", ret); - return NULL; + goto err; } btt_debugfs_init(btt); return btt; +err: + free_arenas(btt); + return NULL; } /** diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index b199eea3260ef..197e5368c0a46 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -365,11 +365,6 @@ unsigned sizeof_namespace_label(struct nvdimm_drvdata *ndd); for (res = (ndd)->dpa.child, next = res ? res->sibling : NULL; \ res; res = next, next = next ? next->sibling : NULL) -struct nd_percpu_lane { - int count; - spinlock_t lock; -}; - enum nd_label_flags { ND_LABEL_REAP, }; @@ -400,6 +395,10 @@ struct nd_mapping { struct nvdimm_drvdata *ndd; }; +struct nd_lane { + struct mutex lock; /* serialize lane access */ +} ____cacheline_aligned_in_smp; + struct nd_region { struct device dev; struct ida ns_ida; @@ -420,7 +419,7 @@ struct nd_region { struct kernfs_node *bb_state; struct badblocks bb; struct nd_interleave_set *nd_set; - struct nd_percpu_lane __percpu *lane; + struct nd_lane *lane; int (*flush)(struct nd_region *nd_region, struct bio *bio); struct nd_mapping mapping[] __counted_by(ndr_mappings); }; diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index e35c2e18518f0..5e079d61cbaa3 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -192,7 +192,9 @@ static void nd_region_release(struct device *dev) put_device(&nvdimm->dev); } - free_percpu(nd_region->lane); + for (i = 0; i < nd_region->num_lanes; i++) + mutex_destroy(&nd_region->lane[i].lock); + kfree(nd_region->lane); if (!test_bit(ND_REGION_CXL, &nd_region->flags)) memregion_free(nd_region->id); kfree(nd_region); @@ -904,52 +906,30 @@ void nd_region_advance_seeds(struct nd_region *nd_region, struct device *dev) * nd_region_acquire_lane - allocate and lock a lane * @nd_region: region id and number of lanes possible * - * A lane correlates to a BLK-data-window and/or a log slot in the BTT. - * We optimize for the common case where there are 256 lanes, one - * per-cpu. For larger systems we need to lock to share lanes. For now - * this implementation assumes the cost of maintaining an allocator for - * free lanes is on the order of the lock hold time, so it implements a - * static lane = cpu % num_lanes mapping. + * A lane correlates to a log slot in the BTT. Lanes are shared across + * CPUs using a static lane = cpu % num_lanes mapping, with a per-lane + * mutex to serialize access. * - * In the case of a BTT instance on top of a BLK namespace a lane may be - * acquired recursively. We lock on the first instance. - * - * In the case of a BTT instance on top of PMEM, we only acquire a lane - * for the BTT metadata updates. + * Callers must be in sleepable context. The only in-tree caller is + * BTT's ->submit_bio handler (btt_read_pg / btt_write_pg). */ unsigned int nd_region_acquire_lane(struct nd_region *nd_region) + __acquires(&nd_region->lane[lane].lock) { - unsigned int cpu, lane; - - migrate_disable(); - cpu = smp_processor_id(); - if (nd_region->num_lanes < nr_cpu_ids) { - struct nd_percpu_lane *ndl_lock, *ndl_count; + unsigned int lane; - lane = cpu % nd_region->num_lanes; - ndl_count = per_cpu_ptr(nd_region->lane, cpu); - ndl_lock = per_cpu_ptr(nd_region->lane, lane); - if (ndl_count->count++ == 0) - spin_lock(&ndl_lock->lock); - } else - lane = cpu; + might_sleep(); + lane = raw_smp_processor_id() % nd_region->num_lanes; + mutex_lock(&nd_region->lane[lane].lock); return lane; } EXPORT_SYMBOL(nd_region_acquire_lane); void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane) + __releases(&nd_region->lane[lane].lock) { - if (nd_region->num_lanes < nr_cpu_ids) { - unsigned int cpu = smp_processor_id(); - struct nd_percpu_lane *ndl_lock, *ndl_count; - - ndl_count = per_cpu_ptr(nd_region->lane, cpu); - ndl_lock = per_cpu_ptr(nd_region->lane, lane); - if (--ndl_count->count == 0) - spin_unlock(&ndl_lock->lock); - } - migrate_enable(); + mutex_unlock(&nd_region->lane[lane].lock); } EXPORT_SYMBOL(nd_region_release_lane); @@ -1019,17 +999,16 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, goto err_id; } - nd_region->lane = alloc_percpu(struct nd_percpu_lane); + nd_region->num_lanes = ndr_desc->num_lanes; + if (!nd_region->num_lanes) + goto err_percpu; + nd_region->lane = kcalloc(nd_region->num_lanes, + sizeof(*nd_region->lane), GFP_KERNEL); if (!nd_region->lane) goto err_percpu; - for (i = 0; i < nr_cpu_ids; i++) { - struct nd_percpu_lane *ndl; - - ndl = per_cpu_ptr(nd_region->lane, i); - spin_lock_init(&ndl->lock); - ndl->count = 0; - } + for (i = 0; i < nd_region->num_lanes; i++) + mutex_init(&nd_region->lane[i].lock); for (i = 0; i < ndr_desc->num_mappings; i++) { struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; @@ -1046,7 +1025,6 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, } nd_region->provider_data = ndr_desc->provider_data; nd_region->nd_set = ndr_desc->nd_set; - nd_region->num_lanes = ndr_desc->num_lanes; nd_region->flags = ndr_desc->flags; nd_region->ro = ro; nd_region->numa_node = ndr_desc->numa_node; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index c692fc73babfe..da6e983e20058 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -225,7 +225,7 @@ static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q) { struct apple_nvme *anv = queue_to_apple_nvme(q); - if (q->is_adminq && anv->hw->has_lsq_nvmmu) + if (q->is_adminq) return APPLE_NVME_AQ_DEPTH; return anv->hw->max_queue_depth; @@ -303,7 +303,7 @@ static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, memcpy((void *)q->sqes + (q->sq_tail << APPLE_NVME_IOSQES), cmd, sizeof(*cmd)); - if (++q->sq_tail == anv->hw->max_queue_depth) + if (++q->sq_tail == apple_nvme_queue_depth(q)) q->sq_tail = 0; writel(q->sq_tail, q->sq_db); @@ -1139,10 +1139,7 @@ static void apple_nvme_reset_work(struct work_struct *work) } /* Setup the admin queue */ - if (anv->hw->has_lsq_nvmmu) - aqa = APPLE_NVME_AQ_DEPTH - 1; - else - aqa = anv->hw->max_queue_depth - 1; + aqa = APPLE_NVME_AQ_DEPTH - 1; aqa |= aqa << 16; writel(aqa, anv->mmio_nvme + NVME_REG_AQA); writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); @@ -1325,8 +1322,7 @@ static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) * both queues. The admin queue gets the first APPLE_NVME_AQ_DEPTH which * must be marked as reserved in the IO queue. */ - if (anv->hw->has_lsq_nvmmu) - anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; + anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; anv->tagset.queue_depth = anv->hw->max_queue_depth - 1; anv->tagset.timeout = NVME_IO_TIMEOUT; anv->tagset.numa_node = NUMA_NO_NODE; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index c3032d6ad6b1e..a231da6865853 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2263,7 +2263,7 @@ static int nvme_query_fdp_granularity(struct nvme_ctrl *ctrl, } n = le16_to_cpu(h->numfdpc) + 1; - if (fdp_idx > n) { + if (fdp_idx >= n) { dev_warn(ctrl->device, "FDP index:%d out of range:%d\n", fdp_idx, n); /* Proceed without registering FDP streams */ @@ -3926,7 +3926,7 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, int ret = -ENOMEM; #ifdef CONFIG_NVME_MULTIPATH - size += num_possible_nodes() * sizeof(struct nvme_ns *); + size += nr_node_ids * sizeof(struct nvme_ns *); #endif head = kzalloc(size, GFP_KERNEL); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 263161cb8ac06..9d3ac937f5949 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -511,6 +511,12 @@ static void nvme_ns_head_submit_bio(struct bio *bio) ns = nvme_find_path(head); if (likely(ns)) { bio_set_dev(bio, ns->disk->part0); + /* + * Use BIO_REMAPPED to skip bio_check_eod() when this bio + * enters submit_bio_noacct() for the per-path device. The EOD + * check already passed on the multipath head. + */ + bio_set_flag(bio, BIO_REMAPPED); bio->bi_opf |= REQ_NVME_MPATH; trace_block_bio_remap(bio, disk_devt(ns->head->disk), bio->bi_iter.bi_sector); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index b5f8462006789..0de6844ca56c0 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -587,11 +587,16 @@ static bool nvme_dbbuf_update_and_check_event(u16 value, __le32 *dbbuf_db, } static struct nvme_descriptor_pools * -nvme_setup_descriptor_pools(struct nvme_dev *dev, unsigned numa_node) +nvme_setup_descriptor_pools(struct nvme_dev *dev, int numa_node) { - struct nvme_descriptor_pools *pools = &dev->descriptor_pools[numa_node]; + struct nvme_descriptor_pools *pools; size_t small_align = NVME_SMALL_POOL_SIZE; + if (numa_node == NUMA_NO_NODE) + numa_node = 0; + + pools = &dev->descriptor_pools[numa_node]; + if (pools->small) return pools; /* already initialized */ diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 62dd59b9aa4f1..4477c4d6b1eed 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -944,7 +944,7 @@ u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, status = nvmet_check_sqid(ctrl, sqid, true); if (status != NVME_SC_SUCCESS) - return status; + goto ctrl_put; ret = nvmet_sq_init(sq, cq); if (ret) { diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c index e9b35549e254f..114869d16a1f3 100644 --- a/drivers/nvme/target/discovery.c +++ b/drivers/nvme/target/discovery.c @@ -166,6 +166,7 @@ static void nvmet_execute_disc_get_log_page(struct nvmet_req *req) u64 offset = nvmet_get_log_page_offset(req->cmd); size_t data_len = nvmet_get_log_page_len(req->cmd); size_t alloc_len; + size_t copy_len; struct nvmet_subsys_link *p; struct nvmet_port *r; u32 numrec = 0; @@ -242,7 +243,27 @@ static void nvmet_execute_disc_get_log_page(struct nvmet_req *req) up_read(&nvmet_config_sem); - status = nvmet_copy_to_sgl(req, 0, buffer + offset, data_len); + /* + * Validate the host-supplied log page offset before copying out. + * Without this check, the host controls a 64-bit byte offset into + * a small kzalloc'd buffer: a value past the log page lets the + * subsequent memcpy read adjacent kernel heap, and a value aimed + * at unmapped kernel memory faults the in-kernel copy and crashes + * the target host. The Discovery controller is unauthenticated, + * so the bug is reachable from any reachable fabric peer. + */ + if (offset > alloc_len) { + req->error_loc = + offsetof(struct nvme_get_log_page_command, lpo); + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto out_free_buffer; + } + + copy_len = min_t(size_t, data_len, alloc_len - offset); + status = nvmet_copy_to_sgl(req, 0, buffer + offset, copy_len); + if (!status && copy_len < data_len) + status = nvmet_zero_sgl(req, copy_len, data_len - copy_len); +out_free_buffer: kfree(buffer); out: nvmet_req_complete(req, status); diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index f1e613e7c63e5..45820a12750de 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -132,13 +132,22 @@ static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) return 0; } -static u8 nvmet_auth_reply(struct nvmet_req *req, void *d) +static u8 nvmet_auth_reply(struct nvmet_req *req, void *d, u32 tl) { struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmf_auth_dhchap_reply_data *data = d; - u16 dhvlen = le16_to_cpu(data->dhvlen); + u16 dhvlen; u8 *response; + if (tl < sizeof(*data)) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + + dhvlen = le16_to_cpu(data->dhvlen); + + /* Validate that hl and dhvlen fit within the transfer length */ + if (sizeof(*data) + 2 * (size_t)data->hl + dhvlen > tl) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + pr_debug("%s: ctrl %d qid %d: data hl %d cvalid %d dhvlen %u\n", __func__, ctrl->cntlid, req->sq->qid, data->hl, data->cvalid, dhvlen); @@ -338,7 +347,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) switch (data->auth_id) { case NVME_AUTH_DHCHAP_MESSAGE_REPLY: - dhchap_status = nvmet_auth_reply(req, d); + dhchap_status = nvmet_auth_reply(req, d, tl); if (dhchap_status == 0) req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1; @@ -484,7 +493,31 @@ static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al) u32 nvmet_auth_receive_data_len(struct nvmet_req *req) { - return le32_to_cpu(req->cmd->auth_receive.al); + struct nvmet_ctrl *ctrl = req->sq->ctrl; + u32 al = le32_to_cpu(req->cmd->auth_receive.al); + u32 min_len; + + /* + * Reject too-short al before kmalloc(al), since the SUCCESS1 and + * FAILURE1/default builders write fixed response headers into it. + */ + switch (req->sq->dhchap_step) { + case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE: + return al; + case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1: + min_len = sizeof(struct nvmf_auth_dhchap_success1_data); + if (req->sq->dhchap_c2) + min_len += nvme_auth_hmac_hash_len(ctrl->shash_id); + break; + default: + min_len = sizeof(struct nvmf_auth_dhchap_failure_data); + break; + } + + if (al < min_len) + return 0; + + return al; } void nvmet_execute_auth_receive(struct nvmet_req *req) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index e6e2c3f9afdf5..ea1185b8267ef 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -666,7 +666,8 @@ static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) if (rsp->n_rdma) nvmet_rdma_rw_ctx_destroy(rsp); - if (rsp->req.sg != rsp->cmd->inline_sg) + if (rsp->req.sg < rsp->cmd->inline_sg || + rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) nvmet_req_free_sgls(&rsp->req); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) @@ -821,24 +822,25 @@ static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc) static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len, u64 off) { - int sg_count = num_pages(len); + u64 page_off = off % PAGE_SIZE; + u64 page_idx = off / PAGE_SIZE; + int sg_count = num_pages(page_off + len); struct scatterlist *sg; int i; - sg = rsp->cmd->inline_sg; + sg = &rsp->cmd->inline_sg[page_idx]; for (i = 0; i < sg_count; i++, sg++) { if (i < sg_count - 1) sg_unmark_end(sg); else sg_mark_end(sg); - sg->offset = off; - sg->length = min_t(int, len, PAGE_SIZE - off); + sg->offset = page_off; + sg->length = min_t(u64, len, PAGE_SIZE - page_off); len -= sg->length; - if (!i) - off = 0; + page_off = 0; } - rsp->req.sg = rsp->cmd->inline_sg; + rsp->req.sg = &rsp->cmd->inline_sg[page_idx]; rsp->req.sg_cnt = sg_count; } @@ -1598,8 +1600,10 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, pending++; } mutex_unlock(&nvmet_rdma_queue_mutex); - if (pending > NVMET_RDMA_BACKLOG) - return NVME_SC_CONNECT_CTRL_BUSY; + if (pending > NVMET_RDMA_BACKLOG) { + ret = NVME_SC_CONNECT_CTRL_BUSY; + goto put_device; + } } ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn); diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 20f150d17a962..15c52f1f95f1b 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1844,10 +1844,11 @@ static void nvmet_tcp_tls_handshake_done(void *data, int status, if (!status) status = nvmet_tcp_tls_key_lookup(queue, peerid); + if (!status) + status = nvmet_tcp_set_queue_sock(queue); + if (status) nvmet_tcp_schedule_release_queue(queue); - else - nvmet_tcp_set_queue_sock(queue); kref_put(&queue->kref, nvmet_tcp_release_queue); } @@ -1999,6 +2000,12 @@ static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, nvmet_tcp_free_cmd(&queue->connect); out_ida_remove: ida_free(&nvmet_tcp_queue_ida, queue->idx); + /* + * Drain the page fragment cache if any allocations were done. + * The first allocation using pf_cache is nvmet_tcp_alloc_cmd() + * for queue->connect after ida_alloc(). + */ + page_frag_cache_drain(&queue->pf_cache); out_sock: fput(queue->sock->file); out_free_queue: diff --git a/drivers/of/cpu.c b/drivers/of/cpu.c index 5214dc3d05ae1..bd0e918d6f290 100644 --- a/drivers/of/cpu.c +++ b/drivers/of/cpu.c @@ -60,7 +60,7 @@ static bool __of_find_n_match_cpu_property(struct device_node *cpun, cell = of_get_property(cpun, prop_name, &prop_len); if (!cell && !ac && arch_match_cpu_phys_id(cpu, 0)) return true; - if (!cell || !ac) + if (!cell || !ac || ac > 2) return false; prop_len /= sizeof(*cell) * ac; for (tid = 0; tid < prop_len; tid++) { diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index ade288372101b..aa450425ec1e6 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -225,7 +225,6 @@ static void __of_attach_node(struct device_node *np) np->sibling = np->parent->child; np->parent->child = np; of_node_clear_flag(np, OF_DETACHED); - fwnode_set_flag(&np->fwnode, FWNODE_FLAG_NOT_DEVICE); raw_spin_unlock_irqrestore(&devtree_lock, flags); diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 8d5777cb5d1b3..deaea58c74f2a 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -69,29 +69,32 @@ static int __init early_init_dt_alloc_reserved_memory_arch(phys_addr_t size, * the initial static array is copied over to this new array and * the new array is used from this point on. */ -static void __init alloc_reserved_mem_array(void) +static int __init alloc_reserved_mem_array(void) { struct reserved_mem *new_array; size_t alloc_size, copy_size, memset_size; + int ret; + + if (!total_reserved_mem_cnt) + return 0; alloc_size = array_size(total_reserved_mem_cnt, sizeof(*new_array)); if (alloc_size == SIZE_MAX) { - pr_err("Failed to allocate memory for reserved_mem array with err: %d", -EOVERFLOW); - return; + ret = -EOVERFLOW; + goto fail; } new_array = memblock_alloc(alloc_size, SMP_CACHE_BYTES); if (!new_array) { - pr_err("Failed to allocate memory for reserved_mem array with err: %d", -ENOMEM); - return; + ret = -ENOMEM; + goto fail; } copy_size = array_size(reserved_mem_count, sizeof(*new_array)); if (copy_size == SIZE_MAX) { memblock_free(new_array, alloc_size); - total_reserved_mem_cnt = MAX_RESERVED_REGIONS; - pr_err("Failed to allocate memory for reserved_mem array with err: %d", -EOVERFLOW); - return; + ret = -EOVERFLOW; + goto fail; } memset_size = alloc_size - copy_size; @@ -100,6 +103,12 @@ static void __init alloc_reserved_mem_array(void) memset(new_array + reserved_mem_count, 0, memset_size); reserved_mem = new_array; + return 0; + +fail: + pr_err("Failed to allocate memory for reserved_mem array with err: %d", ret); + reserved_mem_count = 0; + return ret; } static void fdt_init_reserved_mem_node(unsigned long node, const char *uname, @@ -266,7 +275,8 @@ void __init fdt_scan_reserved_mem_late(void) } /* Attempt dynamic allocation of a new reserved_mem array */ - alloc_reserved_mem_array(); + if (alloc_reserved_mem_array()) + return; if (__reserved_mem_check_root(node)) { pr_err("Reserved memory: unsupported node format, ignoring\n"); diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index c1c5686fc7b19..4e45f3414c2c1 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -185,6 +185,15 @@ static int overlay_notify(struct overlay_changeset *ovcs, return 0; } +static void overlay_fw_devlink_refresh(struct overlay_changeset *ovcs) +{ + for (int i = 0; i < ovcs->count; i++) { + struct device_node *np = ovcs->fragments[i].target; + + fw_devlink_refresh_fwnode(of_fwnode_handle(np)); + } +} + /* * The values of properties in the "/__symbols__" node are paths in * the ovcs->overlay_root. When duplicating the properties, the paths @@ -951,6 +960,12 @@ static int of_overlay_apply(struct overlay_changeset *ovcs, pr_err("overlay apply changeset entry notify error %d\n", ret); /* notify failure is not fatal, continue */ + /* + * Needs to happen after changeset notify to give the listeners a chance + * to finish creating all the devices they need to create. + */ + overlay_fw_devlink_refresh(ovcs); + ret_tmp = overlay_notify(ovcs, OF_OVERLAY_POST_APPLY); if (ret_tmp) if (!ret) diff --git a/drivers/of/platform.c b/drivers/of/platform.c index a42224f9d1a88..53bca8c6f7810 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -744,11 +744,6 @@ static int of_platform_notify(struct notifier_block *nb, if (of_node_check_flag(rd->dn, OF_POPULATED)) return NOTIFY_OK; - /* - * Clear the flag before adding the device so that fw_devlink - * doesn't skip adding consumers to this device. - */ - fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); /* pdev_parent may be NULL when no bus platform device */ pdev_parent = of_find_device_by_node(parent); pdev = of_platform_device_create(rd->dn, NULL, diff --git a/drivers/opp/core.c b/drivers/opp/core.c index da3f5eba43419..ab0b0a2f85a17 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2088,11 +2088,10 @@ int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, return ret; list_add(&new_opp->node, head); + new_opp->opp_table = opp_table; + kref_init(&new_opp->kref); } - new_opp->opp_table = opp_table; - kref_init(&new_opp->kref); - opp_debug_create_one(new_opp, opp_table); if (!_opp_supported_by_regulators(new_opp, opp_table)) { diff --git a/drivers/opp/of.c b/drivers/opp/of.c index f96adfd5b2197..c02e20632fa64 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -673,7 +673,7 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, */ if (unlikely(opp_table->regulator_count == -1)) { opp_table->regulator_count = 0; - return 0; + goto free_microwatt; } for (i = 0, j = 0; i < opp_table->regulator_count; i++) { @@ -696,6 +696,7 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, opp->supplies[i].u_watt = microwatt[i]; } +free_microwatt: kfree(microwatt); free_microamp: kfree(microamp); diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index e35044cc52185..38b2d8262e516 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -137,6 +137,7 @@ struct imx_pcie_drvdata { const u32 mode_off[IMX_PCIE_MAX_INSTANCES]; const u32 mode_mask[IMX_PCIE_MAX_INSTANCES]; const struct pci_epc_features *epc_features; + int (*select_ref_clk_src)(struct imx_pcie *pcie); int (*init_phy)(struct imx_pcie *pcie); int (*enable_ref_clk)(struct imx_pcie *pcie, bool enable); int (*core_reset)(struct imx_pcie *pcie, bool assert); @@ -247,10 +248,26 @@ static unsigned int imx_pcie_grp_offset(const struct imx_pcie *imx_pcie) return imx_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14; } -static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie) +static int imx95_pcie_select_ref_clk_src(struct imx_pcie *imx_pcie) { bool ext = imx_pcie->enable_ext_refclk; + /* + * Regarding the Signal Descriptions of i.MX95 PCIe PHY, ref_use_pad is + * used to select reference clock connected to a pair of pads. + * + * Any change in this input must be followed by phy_reset assertion. + */ + + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_GEN_CTRL, + IMX95_PCIE_REF_USE_PAD, + ext ? IMX95_PCIE_REF_USE_PAD : 0); + + return 0; +} + +static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie) +{ /* * ERR051624: The Controller Without Vaux Cannot Exit L23 Ready * Through Beacon or PERST# De-assertion @@ -269,13 +286,6 @@ static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie) IMX95_PCIE_PHY_CR_PARA_SEL, IMX95_PCIE_PHY_CR_PARA_SEL); - regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_GEN_CTRL, - IMX95_PCIE_REF_USE_PAD, - ext ? IMX95_PCIE_REF_USE_PAD : 0); - regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0, - IMX95_PCIE_REF_CLKEN, - ext ? 0 : IMX95_PCIE_REF_CLKEN); - return 0; } @@ -665,14 +675,6 @@ static int imx_pcie_attach_pd(struct device *dev) return 0; } -static int imx6sx_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) -{ - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, - IMX6SX_GPR12_PCIE_TEST_POWERDOWN, - enable ? 0 : IMX6SX_GPR12_PCIE_TEST_POWERDOWN); - return 0; -} - static int imx6q_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) { if (enable) { @@ -732,7 +734,29 @@ static void imx95_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable) static int imx95_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) { + bool ext = imx_pcie->enable_ext_refclk; + imx95_pcie_clkreq_override(imx_pcie, enable); + /* + * The ref_clk_en signal must remain de-asserted until the + * reference clock is running at appropriate frequency, at which + * point this bit can be asserted. For lower power states where + * the reference clock to the PHY is disabled, it may also be + * de-asserted. + * +------------------- -+--------+----------------+ + * | External clock mode | Enable | PCIE_REF_CLKEN | + * +---------------------+--------+----------------+ + * | TRUE | X | 1b'0 | + * +---------------------+--------+----------------+ + * | FALSE | TRUE | 1b'1 | + * +---------------------+--------+----------------+ + * | FALSE | FALSE | 1b'0 | + * +---------------------+--------+----------------+ + */ + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0, + IMX95_PCIE_REF_CLKEN, + ext || !enable ? 0 : IMX95_PCIE_REF_CLKEN); + return 0; } @@ -786,6 +810,9 @@ static int imx6sx_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) if (assert) regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_TEST_POWERDOWN); + else + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6SX_GPR12_PCIE_TEST_POWERDOWN); /* Force PCIe PHY reset */ regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET, @@ -1256,6 +1283,9 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp) pp->bridge->disable_device = imx_pcie_disable_device; } + if (imx_pcie->drvdata->select_ref_clk_src) + imx_pcie->drvdata->select_ref_clk_src(imx_pcie); + imx_pcie_assert_core_reset(imx_pcie); imx_pcie_assert_perst(imx_pcie, true); @@ -1877,7 +1907,6 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx6sx_pcie_init_phy, - .enable_ref_clk = imx6sx_pcie_enable_ref_clk, .core_reset = imx6sx_pcie_core_reset, .ops = &imx_pcie_host_ops, }, @@ -1967,6 +1996,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_mask[0] = IMX95_PCIE_DEVICE_TYPE, .core_reset = imx95_pcie_core_reset, .init_phy = imx95_pcie_init_phy, + .select_ref_clk_src = imx95_pcie_select_ref_clk_src, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, .enable_ref_clk = imx95_pcie_enable_ref_clk, .clr_clkreq_override = imx95_pcie_clr_clkreq_override, @@ -2022,6 +2052,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .ltssm_mask = IMX95_PCIE_LTSSM_EN, .mode_off[0] = IMX95_PE0_GEN_CTRL_1, .mode_mask[0] = IMX95_PCIE_DEVICE_TYPE, + .select_ref_clk_src = imx95_pcie_select_ref_clk_src, .init_phy = imx95_pcie_init_phy, .core_reset = imx95_pcie_core_reset, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c index 0694084f612b7..225d887cd0a3e 100644 --- a/drivers/pci/controller/dwc/pci-meson.c +++ b/drivers/pci/controller/dwc/pci-meson.c @@ -204,7 +204,9 @@ static inline struct clk *meson_pcie_probe_clock(struct device *dev, return ERR_PTR(ret); } - devm_add_action_or_reset(dev, meson_pcie_disable_clock, clk); + ret = devm_add_action_or_reset(dev, meson_pcie_disable_clock, clk); + if (ret) + return ERR_PTR(ret); return clk; } @@ -451,6 +453,14 @@ static int meson_pcie_probe(struct platform_device *pdev) return ret; } +static void meson_pcie_remove(struct platform_device *pdev) +{ + struct meson_pcie *mp = platform_get_drvdata(pdev); + + dw_pcie_host_deinit(&mp->pci.pp); + meson_pcie_power_off(mp); +} + static const struct of_device_id meson_pcie_of_match[] = { { .compatible = "amlogic,axg-pcie", @@ -464,6 +474,7 @@ MODULE_DEVICE_TABLE(of, meson_pcie_of_match); static struct platform_driver meson_pcie_driver = { .probe = meson_pcie_probe, + .remove = meson_pcie_remove, .driver = { .name = "meson-pcie", .of_match_table = meson_pcie_of_match, diff --git a/drivers/pci/controller/dwc/pcie-designware-debugfs.c b/drivers/pci/controller/dwc/pcie-designware-debugfs.c index d0884253be97e..449c67e7930d3 100644 --- a/drivers/pci/controller/dwc/pcie-designware-debugfs.c +++ b/drivers/pci/controller/dwc/pcie-designware-debugfs.c @@ -306,6 +306,7 @@ static ssize_t err_inj_write(struct file *file, const char __user *buf, u32 val, counter, vc_num, err_group, type_mask; int val_diff = 0; char *kern_buf; + int ret; err_group = err_inj_list[pdata->idx].err_inj_group; type_mask = err_inj_type_mask[err_group]; @@ -327,10 +328,10 @@ static ssize_t err_inj_write(struct file *file, const char __user *buf, return -EINVAL; } } else { - val = kstrtou32(kern_buf, 0, &counter); - if (val) { + ret = kstrtou32(kern_buf, 0, &counter); + if (ret) { kfree(kern_buf); - return val; + return ret; } } @@ -557,6 +558,9 @@ static void dwc_pcie_rasdes_debugfs_deinit(struct dw_pcie *pci) { struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + if (!rinfo) + return; + mutex_destroy(&rinfo->reg_event_lock); } diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index c11cf61b8319e..76c9a0a10367b 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -487,13 +487,13 @@ static inline void dw_pcie_writel_atu_ob(struct dw_pcie *pci, u32 index, u32 reg static inline u32 dw_pcie_enable_ecrc(u32 val) { /* - * DWC versions 0x3530302a and 0x3536322a have a design issue where - * the 'TD' bit in the Control register-1 of the ATU outbound - * region acts like an override for the ECRC setting, i.e., the - * presence of TLP Digest (ECRC) in the outgoing TLPs is solely - * determined by this bit. This is contrary to the PCIe spec which - * says that the enablement of the ECRC is solely determined by the - * AER registers. + * DesignWare core versions prior to 5.10A have a design issue where the + * 'TD' bit in the Control register-1 of the ATU outbound region acts + * like an override for the ECRC setting, i.e., the presence of TLP + * Digest (ECRC) in the outgoing TLPs is solely determined by this + * bit. This is contrary to the PCIe spec which says that the + * enablement of the ECRC is solely determined by the AER + * registers. * * Because of this, even when the ECRC is enabled through AER * registers, the transactions going through ATU won't have TLP @@ -563,7 +563,7 @@ int dw_pcie_prog_outbound_atu(struct dw_pcie *pci, if (upper_32_bits(limit_addr) > upper_32_bits(parent_bus_addr) && dw_pcie_ver_is_ge(pci, 460A)) val |= PCIE_ATU_INCREASE_REGION_SIZE; - if (dw_pcie_ver_is(pci, 490A) || dw_pcie_ver_is(pci, 500A)) + if (!dw_pcie_ver_is_ge(pci, 510A)) val = dw_pcie_enable_ecrc(val); dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL1, val); diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 3e69ef60165b0..a07b7abda41f3 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -35,6 +35,7 @@ #define DW_PCIE_VER_480A 0x3438302a #define DW_PCIE_VER_490A 0x3439302a #define DW_PCIE_VER_500A 0x3530302a +#define DW_PCIE_VER_510A 0x3531302a #define DW_PCIE_VER_520A 0x3532302a #define DW_PCIE_VER_540A 0x3534302a #define DW_PCIE_VER_562A 0x3536322a diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c index c21906eced618..f157c716953c2 100644 --- a/drivers/pci/controller/dwc/pcie-intel-gw.c +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -196,6 +196,13 @@ static void intel_pcie_device_rst_deassert(struct intel_pcie *pcie) gpiod_set_value_cansleep(pcie->reset_gpio, 0); } +static void intel_pcie_core_irq_enable(struct intel_pcie *pcie) +{ + pcie_app_wr(pcie, PCIE_APP_IRNEN, 0); + pcie_app_wr(pcie, PCIE_APP_IRNCR, PCIE_APP_IRN_INT); + pcie_app_wr(pcie, PCIE_APP_IRNEN, PCIE_APP_IRN_INT); +} + static void intel_pcie_core_irq_disable(struct intel_pcie *pcie) { pcie_app_wr(pcie, PCIE_APP_IRNEN, 0); @@ -278,6 +285,16 @@ static void intel_pcie_turn_off(struct intel_pcie *pcie) pcie_rc_cfg_wr_mask(pcie, PCI_COMMAND, PCI_COMMAND_MEMORY, 0); } +static int intel_pcie_start_link(struct dw_pcie *pci) +{ + struct intel_pcie *pcie = dev_get_drvdata(pci->dev); + + intel_pcie_device_rst_deassert(pcie); + intel_pcie_ltssm_enable(pcie); + + return 0; +} + static int intel_pcie_host_setup(struct intel_pcie *pcie) { int ret; @@ -285,13 +302,9 @@ static int intel_pcie_host_setup(struct intel_pcie *pcie) intel_pcie_core_rst_assert(pcie); intel_pcie_device_rst_assert(pcie); - - ret = phy_init(pcie->phy); - if (ret) - return ret; - intel_pcie_core_rst_deassert(pcie); + /* Controller clock must be provided earlier than PHY */ ret = clk_prepare_enable(pcie->core_clk); if (ret) { dev_err(pcie->pci.dev, "Core clock enable failed: %d\n", ret); @@ -300,34 +313,24 @@ static int intel_pcie_host_setup(struct intel_pcie *pcie) pci->atu_base = pci->dbi_base + 0xC0000; + ret = phy_init(pcie->phy); + if (ret) + goto phy_err; + intel_pcie_ltssm_disable(pcie); intel_pcie_link_setup(pcie); intel_pcie_init_n_fts(pci); - ret = dw_pcie_setup_rc(&pci->pp); - if (ret) - goto app_init_err; - dw_pcie_upconfig_setup(pci); - intel_pcie_device_rst_deassert(pcie); - intel_pcie_ltssm_enable(pcie); - - ret = dw_pcie_wait_for_link(pci); - if (ret) - goto app_init_err; - - /* Enable integrated interrupts */ - pcie_app_wr_mask(pcie, PCIE_APP_IRNEN, PCIE_APP_IRN_INT, - PCIE_APP_IRN_INT); + intel_pcie_core_irq_enable(pcie); return 0; -app_init_err: +phy_err: clk_disable_unprepare(pcie->core_clk); clk_err: intel_pcie_core_rst_assert(pcie); - phy_exit(pcie->phy); return ret; } @@ -381,6 +384,7 @@ static int intel_pcie_rc_init(struct dw_pcie_rp *pp) } static const struct dw_pcie_ops intel_pcie_ops = { + .start_link = intel_pcie_start_link, }; static const struct dw_pcie_host_ops intel_pcie_dw_ops = { diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index af6bf5cce65be..2af7573979751 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -1473,6 +1473,7 @@ static const struct qcom_pcie_cfg cfg_1_9_0 = { static const struct qcom_pcie_cfg cfg_1_34_0 = { .ops = &ops_1_9_0, .override_no_snoop = true, + .no_l0s = true, }; static const struct qcom_pcie_cfg cfg_2_1_0 = { @@ -1613,6 +1614,22 @@ static void qcom_pcie_icc_opp_update(struct qcom_pcie *pcie) } } +static int qcom_pcie_set_max_opp(struct device *dev) +{ + unsigned long max_freq = ULONG_MAX; + struct dev_pm_opp *opp; + int ret; + + opp = dev_pm_opp_find_freq_floor(dev, &max_freq); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(dev, opp); + dev_pm_opp_put(opp); + + return ret; +} + static int qcom_pcie_link_transition_count(struct seq_file *s, void *data) { struct qcom_pcie *pcie = (struct qcom_pcie *)dev_get_drvdata(s->private); @@ -1674,6 +1691,12 @@ static int qcom_pcie_ecam_host_init(struct pci_config_window *cfg) pci->dbi_base = cfg->win; pp->num_vectors = MSI_DEF_NUM_VECTORS; + /* + * dw_pcie_msi_host_init() is called directly here, bypassing + * dw_pcie_host_init() where pp->lock is normally initialized. + */ + raw_spin_lock_init(&pp->lock); + ret = dw_pcie_msi_host_init(pp); if (ret) return ret; @@ -1845,9 +1868,7 @@ static int qcom_pcie_probe(struct platform_device *pdev) struct qcom_pcie_perst *perst, *tmp_perst; struct qcom_pcie_port *port, *tmp_port; const struct qcom_pcie_cfg *pcie_cfg; - unsigned long max_freq = ULONG_MAX; struct device *dev = &pdev->dev; - struct dev_pm_opp *opp; struct qcom_pcie *pcie; struct dw_pcie_rp *pp; struct resource *res; @@ -1951,21 +1972,9 @@ static int qcom_pcie_probe(struct platform_device *pdev) * probe(), OPP will be updated using qcom_pcie_icc_opp_update(). */ if (!ret) { - opp = dev_pm_opp_find_freq_floor(dev, &max_freq); - if (IS_ERR(opp)) { - ret = PTR_ERR(opp); - dev_err_probe(pci->dev, ret, - "Unable to find max freq OPP\n"); - goto err_pm_runtime_put; - } else { - ret = dev_pm_opp_set_opp(dev, opp); - } - - dev_pm_opp_put(opp); + ret = qcom_pcie_set_max_opp(dev); if (ret) { - dev_err_probe(pci->dev, ret, - "Failed to set OPP for freq %lu\n", - max_freq); + dev_err_probe(dev, ret, "Failed to set max OPP\n"); goto err_pm_runtime_put; } @@ -2100,6 +2109,14 @@ static int qcom_pcie_resume_noirq(struct device *dev) return 0; if (pm_suspend_target_state != PM_SUSPEND_MEM) { + if (pcie->use_pm_opp) { + ret = qcom_pcie_set_max_opp(dev); + if (ret) { + dev_err(dev, "Failed to set max OPP: %d\n", ret); + return ret; + } + } + ret = icc_enable(pcie->icc_cpu); if (ret) { dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", ret); diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c index d6258c1cffe5e..99952fb7189b1 100644 --- a/drivers/pci/controller/pci-host-common.c +++ b/drivers/pci/controller/pci-host-common.c @@ -68,6 +68,10 @@ int pci_host_common_init(struct platform_device *pdev, if (IS_ERR(cfg)) return PTR_ERR(cfg); + /* Do not reassign bus numbers if probe only */ + if (!pci_has_flag(PCI_PROBE_ONLY)) + pci_add_flags(PCI_REASSIGN_ALL_BUS); + bridge->sysdata = cfg; bridge->ops = (struct pci_ops *)&ops->pci_ops; bridge->enable_device = ops->enable_device; diff --git a/drivers/pci/controller/pci-loongson.c b/drivers/pci/controller/pci-loongson.c index bc630ab8a2831..d0c643996476f 100644 --- a/drivers/pci/controller/pci-loongson.c +++ b/drivers/pci/controller/pci-loongson.c @@ -80,6 +80,18 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, DEV_LS7A_LPC, system_bus_quirk); +static const struct pci_device_id loongson_internal_bridge_devids[] = { + { PCI_VDEVICE(LOONGSON, DEV_LS2K_PCIE_PORT0) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT0) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT1) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT2) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT3) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT4) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT5) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT6) }, + { 0, }, +}; + /* * Some Loongson PCIe ports have hardware limitations on their Maximum Read * Request Size. They can't handle anything larger than this. Sane @@ -92,24 +104,13 @@ static void loongson_set_min_mrrs_quirk(struct pci_dev *pdev) { struct pci_bus *bus = pdev->bus; struct pci_dev *bridge; - static const struct pci_device_id bridge_devids[] = { - { PCI_VDEVICE(LOONGSON, DEV_LS2K_PCIE_PORT0) }, - { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT0) }, - { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT1) }, - { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT2) }, - { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT3) }, - { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT4) }, - { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT5) }, - { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT6) }, - { 0, }, - }; /* look for the matching bridge */ while (!pci_is_root_bus(bus)) { bridge = bus->self; bus = bus->parent; - if (pci_match_id(bridge_devids, bridge)) { + if (pci_match_id(loongson_internal_bridge_devids, bridge)) { if (pcie_get_readrq(pdev) > 256) { pci_info(pdev, "limiting MRRS to 256\n"); pcie_set_readrq(pdev, 256); @@ -176,6 +177,42 @@ static void loongson_pci_msi_quirk(struct pci_dev *dev) } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, DEV_LS7A_PCIE_PORT5, loongson_pci_msi_quirk); +/* + * Older steppings of the Loongson-3C6000 series incorrectly report the + * supported link speeds on their PCIe bridges (device IDs 0x3c19, + * 0x3c29) as only 2.5 GT/s, despite the upstream bus supporting speeds + * from 2.5 GT/s up to 16 GT/s. + */ +static void loongson_pci_bridge_speed_quirk(struct pci_dev *pdev) +{ + u8 old_supported_speeds = pdev->supported_speeds; + + switch (pdev->bus->max_bus_speed) { + case PCIE_SPEED_16_0GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_16_0GB; + fallthrough; + case PCIE_SPEED_8_0GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_8_0GB; + fallthrough; + case PCIE_SPEED_5_0GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_5_0GB; + fallthrough; + case PCIE_SPEED_2_5GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_2_5GB; + break; + default: + pci_warn(pdev, "unexpected max bus speed"); + + return; + } + + if (pdev->supported_speeds != old_supported_speeds) + pci_info(pdev, "fixed up supported link speeds: 0x%x => 0x%x", + old_supported_speeds, pdev->supported_speeds); +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, 0x3c19, loongson_pci_bridge_speed_quirk); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, 0x3c29, loongson_pci_bridge_speed_quirk); + static struct loongson_pci *pci_bus_to_loongson_pci(struct pci_bus *bus) { struct pci_config_window *cfg; @@ -230,11 +267,11 @@ static void __iomem *pci_loongson_map_bus(struct pci_bus *bus, struct loongson_pci *priv = pci_bus_to_loongson_pci(bus); /* - * Do not read more than one device on the bus other than - * the host bus. + * Do not read more than one device on the internal bridges. */ if ((priv->data->flags & FLAG_DEV_FIX) && bus->self) { - if (!pci_is_root_bus(bus) && (device > 0)) + if (!pci_is_root_bus(bus) && (device > 0) && + pci_match_id(loongson_internal_bridge_devids, bus->self)) return NULL; } diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c index 3dbb7adc421c4..76f3823d96139 100644 --- a/drivers/pci/controller/pcie-altera.c +++ b/drivers/pci/controller/pcie-altera.c @@ -864,11 +864,25 @@ static int altera_pcie_init_irq_domain(struct altera_pcie *pcie) return 0; } +static void altera_pcie_disable_irq(struct altera_pcie *pcie) +{ + if (pcie->pcie_data->version == ALTERA_PCIE_V1 || + pcie->pcie_data->version == ALTERA_PCIE_V2) { + /* Disable all P2A interrupts */ + cra_writel(pcie, 0, P2A_INT_ENABLE); + } else if (pcie->pcie_data->version == ALTERA_PCIE_V3) { + /* Disable port-level interrupts (CFG_AER, etc.) */ + writel(0, pcie->hip_base + + pcie->pcie_data->port_conf_offset + + pcie->pcie_data->port_irq_enable_offset); + } +} + static void altera_pcie_irq_teardown(struct altera_pcie *pcie) { + altera_pcie_disable_irq(pcie); irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); irq_domain_remove(pcie->irq_domain); - irq_dispose_mapping(pcie->irq); } static int altera_pcie_parse_dt(struct altera_pcie *pcie) @@ -891,7 +905,6 @@ static int altera_pcie_parse_dt(struct altera_pcie *pcie) if (pcie->irq < 0) return pcie->irq; - irq_set_chained_handler_and_data(pcie->irq, pcie->pcie_data->ops->rp_isr, pcie); return 0; } @@ -1020,6 +1033,14 @@ static int altera_pcie_probe(struct platform_device *pdev) return ret; } + /* + * The chained handler uses pcie->irq_domain, so set it only after the + * INTx domain has been created. + */ + irq_set_chained_handler_and_data(pcie->irq, + pcie->pcie_data->ops->rp_isr, + pcie); + if (pcie->pcie_data->version == ALTERA_PCIE_V1 || pcie->pcie_data->version == ALTERA_PCIE_V2) { /* clear all interrupts */ @@ -1037,7 +1058,16 @@ static int altera_pcie_probe(struct platform_device *pdev) bridge->busnr = pcie->root_bus_nr; bridge->ops = &altera_pcie_ops; - return pci_host_probe(bridge); + ret = pci_host_probe(bridge); + if (ret) + goto err_teardown_irq; + + return 0; + +err_teardown_irq: + altera_pcie_irq_teardown(pcie); + + return ret; } static void altera_pcie_remove(struct platform_device *pdev) diff --git a/drivers/pci/controller/pcie-iproc-bcma.c b/drivers/pci/controller/pcie-iproc-bcma.c index 99a99900444de..593418c2bc3aa 100644 --- a/drivers/pci/controller/pcie-iproc-bcma.c +++ b/drivers/pci/controller/pcie-iproc-bcma.c @@ -64,7 +64,7 @@ static int iproc_bcma_pcie_probe(struct bcma_device *bdev) if (ret) return ret; - pcie->map_irq = iproc_bcma_pcie_map_irq; + bridge->map_irq = iproc_bcma_pcie_map_irq; bcma_set_drvdata(bdev, pcie); diff --git a/drivers/pci/controller/pcie-iproc-platform.c b/drivers/pci/controller/pcie-iproc-platform.c index 0cb78c583c7ea..4c9a0c4bb923e 100644 --- a/drivers/pci/controller/pcie-iproc-platform.c +++ b/drivers/pci/controller/pcie-iproc-platform.c @@ -98,7 +98,7 @@ static int iproc_pltfm_pcie_probe(struct platform_device *pdev) switch (pcie->type) { case IPROC_PCIE_PAXC: case IPROC_PCIE_PAXC_V2: - pcie->map_irq = NULL; + bridge->map_irq = NULL; break; default: break; diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index ccf71993ea359..c22d0aecaaac5 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -1502,7 +1502,6 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res) host->ops = &iproc_pcie_ops; host->sysdata = pcie; - host->map_irq = pcie->map_irq; ret = pci_host_probe(host); if (ret < 0) { diff --git a/drivers/pci/controller/pcie-iproc.h b/drivers/pci/controller/pcie-iproc.h index 969ded03b8c2d..c4443f236ca3b 100644 --- a/drivers/pci/controller/pcie-iproc.h +++ b/drivers/pci/controller/pcie-iproc.h @@ -61,7 +61,6 @@ struct iproc_msi; * @base_addr: PCIe host controller register base physical address * @mem: host bridge memory window resource * @phy: optional PHY device that controls the Serdes - * @map_irq: function callback to map interrupts * @ep_is_internal: indicates an internal emulated endpoint device is connected * @iproc_cfg_read: indicates the iProc config read function should be used * @rej_unconfig_pf: indicates the root complex needs to detect and reject @@ -91,7 +90,6 @@ struct iproc_pcie { phys_addr_t base_addr; struct resource mem; struct phy *phy; - int (*map_irq)(const struct pci_dev *, u8, u8); bool ep_is_internal; bool iproc_cfg_read; bool rej_unconfig_pf; diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index b0accd8285892..88e3fdcb74c06 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -1222,8 +1222,8 @@ static int mtk_pcie_probe(struct platform_device *pdev) err = pci_pwrctrl_create_devices(pcie->dev); if (err) { - goto err_tear_down_irq; dev_err_probe(dev, err, "failed to create pwrctrl devices\n"); + goto err_tear_down_irq; } err = mtk_pcie_setup(pcie); @@ -1260,7 +1260,7 @@ static void mtk_pcie_remove(struct platform_device *pdev) pci_remove_root_bus(host->bus); pci_unlock_rescan_remove(); - pci_pwrctrl_power_off_devices(pcie->dev); + mtk_pcie_devices_power_down(pcie); mtk_pcie_power_down(pcie); pci_pwrctrl_destroy_devices(pcie->dev); mtk_pcie_irq_teardown(pcie); diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 75722524fe744..1bb8839c3cb0f 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -7,6 +7,7 @@ * Honghui Zhang */ +#include #include #include #include @@ -61,7 +62,7 @@ /* MediaTek specific configuration registers */ #define PCIE_FTS_NUM 0x70c #define PCIE_FTS_NUM_MASK GENMASK(15, 8) -#define PCIE_FTS_NUM_L0(x) ((x) & 0xff << 8) +#define PCIE_FTS_NUM_L0(x) FIELD_PREP(PCIE_FTS_NUM_MASK, x) #define PCIE_FC_CREDIT 0x73c #define PCIE_FC_CREDIT_MASK (GENMASK(31, 31) | GENMASK(28, 16)) @@ -175,6 +176,7 @@ struct mtk_pcie_soc { /** * struct mtk_pcie_port - PCIe port information * @base: IO mapped register base + * @phys_base: Physical address of the I/O register base region * @list: port list * @pcie: pointer to PCIe host info * @reset: pointer to port reset control @@ -196,6 +198,7 @@ struct mtk_pcie_soc { */ struct mtk_pcie_port { void __iomem *base; + phys_addr_t phys_base; struct list_head list; struct mtk_pcie *pcie; struct reset_control *reset; @@ -405,7 +408,7 @@ static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) phys_addr_t addr; /* MT2712/MT7622 only support 32-bit MSI addresses */ - addr = virt_to_phys(port->base + PCIE_MSI_VECTOR); + addr = port->phys_base + PCIE_MSI_VECTOR; msg->address_hi = 0; msg->address_lo = lower_32_bits(addr); @@ -520,7 +523,7 @@ static void mtk_pcie_enable_msi(struct mtk_pcie_port *port) u32 val; phys_addr_t msg_addr; - msg_addr = virt_to_phys(port->base + PCIE_MSI_VECTOR); + msg_addr = port->phys_base + PCIE_MSI_VECTOR; val = lower_32_bits(msg_addr); writel(val, port->base + PCIE_IMSI_ADDR); @@ -529,23 +532,27 @@ static void mtk_pcie_enable_msi(struct mtk_pcie_port *port) writel(val, port->base + PCIE_INT_MASK); } -static void mtk_pcie_irq_teardown(struct mtk_pcie *pcie) +static void mtk_pcie_irq_teardown_port(struct mtk_pcie_port *port) { - struct mtk_pcie_port *port, *tmp; + irq_set_chained_handler_and_data(port->irq, NULL, NULL); - list_for_each_entry_safe(port, tmp, &pcie->ports, list) { - irq_set_chained_handler_and_data(port->irq, NULL, NULL); + if (port->irq_domain) + irq_domain_remove(port->irq_domain); - if (port->irq_domain) - irq_domain_remove(port->irq_domain); + if (IS_ENABLED(CONFIG_PCI_MSI)) { + if (port->inner_domain) + irq_domain_remove(port->inner_domain); + } - if (IS_ENABLED(CONFIG_PCI_MSI)) { - if (port->inner_domain) - irq_domain_remove(port->inner_domain); - } + irq_dispose_mapping(port->irq); +} - irq_dispose_mapping(port->irq); - } +static void mtk_pcie_irq_teardown(struct mtk_pcie *pcie) +{ + struct mtk_pcie_port *port, *tmp; + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) + mtk_pcie_irq_teardown_port(port); } static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq, @@ -865,7 +872,7 @@ static int mtk_pcie_startup_port_an7583(struct mtk_pcie_port *port) return mtk_pcie_startup_port_v2(port); } -static void mtk_pcie_enable_port(struct mtk_pcie_port *port) +static int mtk_pcie_enable_port(struct mtk_pcie_port *port) { struct mtk_pcie *pcie = port->pcie; struct device *dev = pcie->dev; @@ -874,7 +881,7 @@ static void mtk_pcie_enable_port(struct mtk_pcie_port *port) err = clk_prepare_enable(port->sys_ck); if (err) { dev_err(dev, "failed to enable sys_ck%d clock\n", port->slot); - goto err_sys_clk; + return err; } err = clk_prepare_enable(port->ahb_ck); @@ -922,11 +929,15 @@ static void mtk_pcie_enable_port(struct mtk_pcie_port *port) goto err_phy_on; } - if (!pcie->soc->startup(port)) - return; + err = pcie->soc->startup(port); + if (err) { + dev_info(dev, "Port%d link down\n", port->slot); + goto err_soc_startup; + } - dev_info(dev, "Port%d link down\n", port->slot); + return 0; +err_soc_startup: phy_power_off(port->phy); err_phy_on: phy_exit(port->phy); @@ -942,8 +953,8 @@ static void mtk_pcie_enable_port(struct mtk_pcie_port *port) clk_disable_unprepare(port->ahb_ck); err_ahb_clk: clk_disable_unprepare(port->sys_ck); -err_sys_clk: - mtk_pcie_port_free(port); + + return err; } static int mtk_pcie_parse_port(struct mtk_pcie *pcie, @@ -953,6 +964,7 @@ static int mtk_pcie_parse_port(struct mtk_pcie *pcie, struct mtk_pcie_port *port; struct device *dev = pcie->dev; struct platform_device *pdev = to_platform_device(dev); + struct resource *res; char name[20]; int err; @@ -961,7 +973,14 @@ static int mtk_pcie_parse_port(struct mtk_pcie *pcie, return -ENOMEM; snprintf(name, sizeof(name), "port%d", slot); - port->base = devm_platform_ioremap_resource_byname(pdev, name); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + if (!res) { + dev_err(dev, "failed to get port%d base\n", slot); + return -EINVAL; + } + + port->phys_base = res->start; + port->base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(port->base)) { dev_err(dev, "failed to map port%d base\n", slot); return PTR_ERR(port->base); @@ -1109,8 +1128,13 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie) return err; /* enable each port, and then check link status */ - list_for_each_entry_safe(port, tmp, &pcie->ports, list) - mtk_pcie_enable_port(port); + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + err = mtk_pcie_enable_port(port); + if (err) { + mtk_pcie_irq_teardown_port(port); + mtk_pcie_port_free(port); + } + } /* power down PCIe subsys if slots are all empty (link down) */ if (list_empty(&pcie->ports)) @@ -1209,14 +1233,18 @@ static int mtk_pcie_resume_noirq(struct device *dev) { struct mtk_pcie *pcie = dev_get_drvdata(dev); struct mtk_pcie_port *port, *tmp; + int err; if (list_empty(&pcie->ports)) return 0; clk_prepare_enable(pcie->free_ck); - list_for_each_entry_safe(port, tmp, &pcie->ports, list) - mtk_pcie_enable_port(port); + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + err = mtk_pcie_enable_port(port); + if (err) + mtk_pcie_port_free(port); + } /* In case of EP was removed while system suspend. */ if (list_empty(&pcie->ports)) diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c index 213028052aa58..cd9171eebc289 100644 --- a/drivers/pci/controller/pcie-rcar-host.c +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -346,7 +346,6 @@ static void rcar_pcie_hw_enable(struct rcar_pcie_host *host) struct rcar_pcie *pcie = &host->pcie; struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); struct resource_entry *win; - LIST_HEAD(res); int i = 0; /* Try setting 5 GT/s link speed */ diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c index 2bdcc35b652cf..5314aca2188a0 100644 --- a/drivers/pci/endpoint/functions/pci-epf-ntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -559,12 +559,15 @@ static int epf_ntb_configure_db(struct epf_ntb *ntb, struct pci_epc *epc; int ret; - if (db_count > MAX_DB_COUNT) - return -EINVAL; - ntb_epc = ntb->epc[type]; epc = ntb_epc->epc; + if (!db_count || db_count > MAX_DB_COUNT) { + dev_err(&epc->dev, "DB count %d out of range (1 - %d)\n", + db_count, MAX_DB_COUNT); + return -EINVAL; + } + if (msix) ret = epf_ntb_configure_msix(ntb, type, db_count); else @@ -1278,7 +1281,6 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, u8 func_no, vfunc_no; struct pci_epc *epc; struct device *dev; - u32 db_count; int ret; ntb_epc = ntb->epc[type]; @@ -1296,17 +1298,16 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, func_no = ntb_epc->func_no; vfunc_no = ntb_epc->vfunc_no; - db_count = ntb->db_count; - if (db_count > MAX_DB_COUNT) { - dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); + if (!ntb->db_count || ntb->db_count > MAX_DB_COUNT) { + dev_err(dev, "DB count %d out of range (1 - %d)\n", + ntb->db_count, MAX_DB_COUNT); return -EINVAL; } - ntb->db_count = db_count; epc = ntb_epc->epc; if (msi_capable) { - ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count); + ret = pci_epc_set_msi(epc, func_no, vfunc_no, ntb->db_count); if (ret) { dev_err(dev, "%s intf: MSI configuration failed\n", pci_epc_interface_string(type)); @@ -1315,7 +1316,7 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, } if (msix_capable) { - ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count, + ret = pci_epc_set_msix(epc, func_no, vfunc_no, ntb->db_count, ntb_epc->msix_bar, ntb_epc->msix_table_offset); if (ret) { diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 2256c3062b1ae..ff9db8f980af5 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -37,6 +37,7 @@ */ #include +#include #include #include #include @@ -69,6 +70,9 @@ static struct workqueue_struct *kpcintb_workqueue; #define MAX_DB_COUNT 32 #define MAX_MW 4 +/* Limit per-work execution to avoid monopolizing kworker on doorbell storms. */ +#define VNTB_PEER_DB_WORK_BUDGET 5 + enum epf_ntb_bar { BAR_CONFIG, BAR_DB, @@ -79,6 +83,12 @@ enum epf_ntb_bar { VNTB_BAR_NUM, }; +enum epf_irq_slot { + EPF_IRQ_LINK = 0, + EPF_IRQ_RESERVED_DB, /* Historically skipped slot */ + EPF_IRQ_DB_START, +}; + /* * +--------------------------------------------------+ Base * | | @@ -129,6 +139,8 @@ struct epf_ntb { u32 spad_count; u64 mws_size[MAX_MW]; atomic64_t db; + atomic64_t peer_db_pending; + struct work_struct peer_db_work; u32 vbus_number; u16 vntb_pid; u16 vntb_vid; @@ -261,10 +273,11 @@ static void epf_ntb_cmd_handler(struct work_struct *work) ntb = container_of(work, struct epf_ntb, cmd_handler.work); - for (i = 1; i < ntb->db_count && !ntb->msi_doorbell; i++) { + for (i = EPF_IRQ_DB_START; i < ntb->db_count && !ntb->msi_doorbell; + i++) { if (ntb->epf_db[i]) { - atomic64_or(1 << (i - 1), &ntb->db); - ntb_db_event(&ntb->ntb, i); + atomic64_or(1 << (i - EPF_IRQ_DB_START), &ntb->db); + ntb_db_event(&ntb->ntb, i - EPF_IRQ_DB_START); ntb->epf_db[i] = 0; } } @@ -330,10 +343,10 @@ static irqreturn_t epf_ntb_doorbell_handler(int irq, void *data) struct epf_ntb *ntb = data; int i; - for (i = 1; i < ntb->db_count; i++) + for (i = EPF_IRQ_DB_START; i < ntb->db_count; i++) if (irq == ntb->epf->db_msg[i].virq) { - atomic64_or(1 << (i - 1), &ntb->db); - ntb_db_event(&ntb->ntb, i); + atomic64_or(1 << (i - EPF_IRQ_DB_START), &ntb->db); + ntb_db_event(&ntb->ntb, i - EPF_IRQ_DB_START); } return IRQ_HANDLED; @@ -483,7 +496,6 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb) { const struct pci_epc_features *epc_features; struct device *dev; - u32 db_count; int ret; dev = &ntb->epf->dev; @@ -495,14 +507,12 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb) return -EINVAL; } - db_count = ntb->db_count; - if (db_count > MAX_DB_COUNT) { - dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); + if (!ntb->db_count || ntb->db_count > MAX_DB_COUNT) { + dev_err(dev, "DB count %d out of range (1 - %d)\n", + ntb->db_count, MAX_DB_COUNT); return -EINVAL; } - ntb->db_count = db_count; - if (epc_features->msi_capable) { ret = pci_epc_set_msi(ntb->epf->epc, ntb->epf->func_no, @@ -920,6 +930,9 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb) INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler); queue_work(kpcintb_workqueue, &ntb->cmd_handler.work); + atomic64_set(&ntb->peer_db_pending, 0); + enable_work(&ntb->peer_db_work); + return 0; err_write_header: @@ -943,6 +956,8 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb) static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) { disable_delayed_work_sync(&ntb->cmd_handler); + disable_work_sync(&ntb->peer_db_work); + atomic64_set(&ntb->peer_db_pending, 0); epf_ntb_mw_bar_clear(ntb, ntb->num_mws); epf_ntb_db_bar_clear(ntb); epf_ntb_config_sspad_bar_clear(ntb); @@ -1253,7 +1268,10 @@ static int vntb_epf_peer_mw_count(struct ntb_dev *ntb) static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb) { - return BIT_ULL(ntb_ndev(ntb)->db_count) - 1; + if (ntb_ndev(ntb)->db_count < EPF_IRQ_DB_START) + return 0; + + return BIT_ULL(ntb_ndev(ntb)->db_count - EPF_IRQ_DB_START) - 1; } static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) @@ -1357,22 +1375,84 @@ static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 return 0; } -static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits) +static void vntb_epf_peer_db_work(struct work_struct *work) { - u32 interrupt_num = ffs(db_bits) + 1; - struct epf_ntb *ntb = ntb_ndev(ndev); + struct epf_ntb *ntb = container_of(work, struct epf_ntb, peer_db_work); + struct pci_epf *epf = ntb->epf; + unsigned int budget = VNTB_PEER_DB_WORK_BUDGET; u8 func_no, vfunc_no; + unsigned int db_bit; + u32 interrupt_num; + u64 db_bits; int ret; - func_no = ntb->epf->func_no; - vfunc_no = ntb->epf->vfunc_no; + if (!epf || !epf->epc) + return; + + func_no = epf->func_no; + vfunc_no = epf->vfunc_no; + + /* + * Drain doorbells from peer_db_pending in snapshots (atomic64_xchg()). + * Limit the number of snapshots handled per run so we don't monopolize + * the workqueue under a doorbell storm. + */ + while (budget--) { + db_bits = atomic64_xchg(&ntb->peer_db_pending, 0); + if (!db_bits) + return; + + while (db_bits) { + /* + * pci_epc_raise_irq() for MSI expects a 1-based + * interrupt number. The first usable doorbell starts + * at EPF_IRQ_DB_START in the legacy slot layout. + * + * Legacy mapping (kept for compatibility): + * + * MSI #1 : link event (reserved) + * MSI #2 : unused (historical offset) + * MSI #3 : doorbell bit 0 (DB#0) + * MSI #4 : doorbell bit 1 (DB#1) + * ... + * + * Do not change this mapping to avoid breaking + * interoperability with older peers. + */ + db_bit = __ffs64(db_bits); + interrupt_num = db_bit + EPF_IRQ_DB_START + 1; + db_bits &= ~BIT_ULL(db_bit); + + ret = pci_epc_raise_irq(epf->epc, func_no, vfunc_no, + PCI_IRQ_MSI, interrupt_num); + if (ret) + dev_err(&ntb->ntb.dev, + "Failed to raise IRQ for interrupt_num %u: %d\n", + interrupt_num, ret); + } + } - ret = pci_epc_raise_irq(ntb->epf->epc, func_no, vfunc_no, - PCI_IRQ_MSI, interrupt_num + 1); - if (ret) - dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n"); + if (atomic64_read(&ntb->peer_db_pending)) + queue_work(kpcintb_workqueue, &ntb->peer_db_work); +} - return ret; +static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + + db_bits &= vntb_epf_db_valid_mask(ndev); + if (!db_bits) + return 0; + + /* + * .peer_db_set() may be called from atomic context. pci_epc_raise_irq() + * can sleep (it takes epc->lock), so defer MSI raising to process + * context. Doorbell requests are coalesced in peer_db_pending. + */ + atomic64_or(db_bits, &ntb->peer_db_pending); + queue_work(kpcintb_workqueue, &ntb->peer_db_work); + + return 0; } static u64 vntb_epf_db_read(struct ntb_dev *ndev) @@ -1619,6 +1699,10 @@ static int epf_ntb_probe(struct pci_epf *epf, ntb->epf = epf; ntb->vbus_number = 0xff; + INIT_WORK(&ntb->peer_db_work, vntb_epf_peer_db_work); + disable_work(&ntb->peer_db_work); + atomic64_set(&ntb->peer_db_pending, 0); + /* Initially, no bar is assigned */ for (i = 0; i < VNTB_BAR_NUM; i++) ntb->epf_ntb_bar[i] = NO_BAR; diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 91ac4e37ecb9c..08df9bace13d1 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -938,12 +938,18 @@ static void sriov_restore_vf_rebar_state(struct pci_dev *dev) return; pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl); + if (PCI_POSSIBLE_ERROR(ctrl)) + return; + nbars = FIELD_GET(PCI_VF_REBAR_CTRL_NBAR_MASK, ctrl); for (i = 0; i < nbars; i++, pos += 8) { int bar_idx, size; pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl); + if (PCI_POSSIBLE_ERROR(ctrl)) + return; + bar_idx = FIELD_GET(PCI_VF_REBAR_CTRL_BAR_IDX, ctrl); size = pci_rebar_bytes_to_size(dev->sriov->barsz[bar_idx]); ctrl &= ~PCI_VF_REBAR_CTRL_BAR_SIZE; diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 81d24a270a795..a3e0daff09880 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -749,7 +749,7 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) - goto out_unmap; + goto out_disable; /* Disable INTX */ pci_intx_for_msi(dev, 0); @@ -770,8 +770,6 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, pcibios_free_irq(dev); return 0; -out_unmap: - iounmap(dev->msix_base); out_disable: dev->msix_enabled = 0; pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0); diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 7c898542af8d5..206e3feb9636e 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -548,6 +548,16 @@ static const struct pci_p2pdma_whitelist_entry { {PCI_VENDOR_ID_INTEL, 0x2033, 0}, {PCI_VENDOR_ID_INTEL, 0x2020, 0}, {PCI_VENDOR_ID_INTEL, 0x09a2, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_DSA_GNRD, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_DSA_DMR, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IAA_DMR, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_QAT_4XXX, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_QAT_401XX, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_QAT_402XX, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_QAT_420XX, 0}, + {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_QAT_6XXX, 0}, /* Google SoCs. */ {PCI_VENDOR_ID_GOOGLE, PCI_ANY_ID, 0}, {} diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index d34266651ad09..0ee01d99e3275 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1514,9 +1514,6 @@ static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool pci_power_name(dev->current_state), pci_power_name(state)); - if (dev->bus->self) - pcie_aspm_pm_state_change(dev->bus->self, locked); - return 0; } diff --git a/drivers/pci/pwrctrl/core.c b/drivers/pci/pwrctrl/core.c index 97cff5b8ca885..cd08d590483b8 100644 --- a/drivers/pci/pwrctrl/core.c +++ b/drivers/pci/pwrctrl/core.c @@ -161,10 +161,12 @@ static void pci_pwrctrl_power_off_device(struct device_node *np) if (!pdev) return; - if (device_is_bound(&pdev->dev)) { - ret = __pci_pwrctrl_power_off_device(&pdev->dev); - if (ret) - dev_err(&pdev->dev, "Failed to power off device: %d", ret); + scoped_guard(device, &pdev->dev) { + if (device_is_bound(&pdev->dev)) { + ret = __pci_pwrctrl_power_off_device(&pdev->dev); + if (ret) + dev_err(&pdev->dev, "Failed to power off device: %d", ret); + } } platform_device_put(pdev); @@ -205,7 +207,7 @@ static int __pci_pwrctrl_power_on_device(struct device *dev) static int pci_pwrctrl_power_on_device(struct device_node *np) { struct platform_device *pdev; - int ret; + int ret = 0; for_each_available_child_of_node_scoped(np, child) { ret = pci_pwrctrl_power_on_device(child); @@ -217,12 +219,14 @@ static int pci_pwrctrl_power_on_device(struct device_node *np) if (!pdev) return 0; - if (device_is_bound(&pdev->dev)) { - ret = __pci_pwrctrl_power_on_device(&pdev->dev); - } else { - /* FIXME: Use blocking wait instead of probe deferral */ - dev_dbg(&pdev->dev, "driver is not bound\n"); - ret = -EPROBE_DEFER; + scoped_guard(device, &pdev->dev) { + if (device_is_bound(&pdev->dev)) { + ret = __pci_pwrctrl_power_on_device(&pdev->dev); + } else { + /* FIXME: Use blocking wait instead of probe deferral */ + dev_dbg(&pdev->dev, "driver is not bound\n"); + ret = -EPROBE_DEFER; + } } platform_device_put(pdev); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index caaed1a01dc02..dd0025d3914e6 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -80,11 +80,10 @@ static bool pcie_lbms_seen(struct pci_dev *dev, u16 lnksta) * Restrict the speed to 2.5GT/s then with the Target Link Speed field, * request a retrain and check the result. * - * If this turns out successful and we know by the Vendor:Device ID it is - * safe to do so, then lift the restriction, letting the devices negotiate - * a higher speed. Also check for a similar 2.5GT/s speed restriction the - * firmware may have already arranged and lift it with ports that already - * report their data link being up. + * If this turns out successful, or where a 2.5GT/s speed restriction has + * been previously arranged by the firmware and the port reports its link + * already being up, lift the restriction, in a hope it is safe to do so, + * letting the devices negotiate a higher speed. * * Otherwise revert the speed to the original setting and request a retrain * again to remove any residual state, ignoring the result as it's supposed @@ -95,51 +94,37 @@ static bool pcie_lbms_seen(struct pci_dev *dev, u16 lnksta) */ int pcie_failed_link_retrain(struct pci_dev *dev) { - static const struct pci_device_id ids[] = { - { PCI_VDEVICE(ASMEDIA, 0x2824) }, /* ASMedia ASM2824 */ - {} - }; - u16 lnksta, lnkctl2; + u16 lnksta, lnkctl2, oldlnkctl2; int ret = -ENOTTY; + u32 lnkcap; if (!pci_is_pcie(dev) || !pcie_downstream_port(dev) || !pcie_cap_has_lnkctl2(dev) || !dev->link_active_reporting) return ret; pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); + pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &oldlnkctl2); if (!(lnksta & PCI_EXP_LNKSTA_DLLLA) && pcie_lbms_seen(dev, lnksta)) { - u16 oldlnkctl2; - pci_info(dev, "broken device, retraining non-functional downstream link at 2.5GT/s\n"); - - pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &oldlnkctl2); ret = pcie_set_target_speed(dev, PCIE_SPEED_2_5GT, false); - if (ret) { - pci_info(dev, "retraining failed\n"); - pcie_set_target_speed(dev, PCIE_LNKCTL2_TLS2SPEED(oldlnkctl2), - true); - return ret; - } - - pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); + if (ret) + goto err; } pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &lnkctl2); - - if ((lnksta & PCI_EXP_LNKSTA_DLLLA) && - (lnkctl2 & PCI_EXP_LNKCTL2_TLS) == PCI_EXP_LNKCTL2_TLS_2_5GT && - pci_match_id(ids, dev)) { - u32 lnkcap; - + pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); + if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) == PCI_EXP_LNKCTL2_TLS_2_5GT && + (lnkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_2_5GB) { pci_info(dev, "removing 2.5GT/s downstream link speed restriction\n"); - pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); ret = pcie_set_target_speed(dev, PCIE_LNKCAP_SLS2SPEED(lnkcap), false); - if (ret) { - pci_info(dev, "retraining failed\n"); - return ret; - } + if (ret) + goto err; } + return ret; +err: + pci_info(dev, "retraining failed\n"); + pcie_set_target_speed(dev, PCIE_LNKCTL2_TLS2SPEED(oldlnkctl2), true); return ret; } diff --git a/drivers/pci/rebar.c b/drivers/pci/rebar.c index 39f8cf3b70d57..11965947c4cb5 100644 --- a/drivers/pci/rebar.c +++ b/drivers/pci/rebar.c @@ -231,6 +231,9 @@ void pci_restore_rebar_state(struct pci_dev *pdev) return; pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + if (PCI_POSSIBLE_ERROR(ctrl)) + return; + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl); for (i = 0; i < nbars; i++, pos += 8) { @@ -238,6 +241,9 @@ void pci_restore_rebar_state(struct pci_dev *pdev) int bar_idx, size; pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + if (PCI_POSSIBLE_ERROR(ctrl)) + return; + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; res = pci_resource_n(pdev, bar_idx); size = pci_rebar_bytes_to_size(resource_size(res)); diff --git a/drivers/pci/rom.c b/drivers/pci/rom.c index e18d3a4383ba6..f2105c6ceef52 100644 --- a/drivers/pci/rom.c +++ b/drivers/pci/rom.c @@ -5,13 +5,40 @@ * (C) Copyright 2004 Jon Smirl * (C) Copyright 2004 Silicon Graphics, Inc. Jesse Barnes */ + +#include +#include #include #include +#include +#include #include +#include #include #include "pci.h" +#define PCI_ROM_HEADER_SIZE 0x1A +#define PCI_ROM_POINTER_TO_DATA_STRUCT 0x18 +#define PCI_ROM_LAST_IMAGE_INDICATOR 0x15 +#define PCI_ROM_LAST_IMAGE_INDICATOR_BIT BIT(7) +#define PCI_ROM_IMAGE_LEN 0x10 +#define PCI_ROM_IMAGE_SECTOR_SIZE SZ_512 +#define PCI_ROM_IMAGE_SIGNATURE 0xAA55 + +/* Data structure signature is "PCIR" in ASCII representation */ +#define PCI_ROM_DATA_STRUCT_SIGNATURE 0x52494350 +#define PCI_ROM_DATA_STRUCT_LEN 0x0A + +/* + * Per PCI Firmware r3.3, sec 5.1.3, a conformant PCI Data Structure is at + * least 24 bytes (0x18), large enough to cover every fixed field this + * driver reads (up to the Indicator byte at offset 0x15). Reject smaller + * device-claimed lengths so the follow-up readers in pci_get_rom_size() + * cannot escape the mapped ROM window. + */ +#define PCI_ROM_DATA_STRUCT_MIN_LEN 0x18 + /** * pci_enable_rom - enable ROM decoding for a PCI device * @pdev: PCI device to enable @@ -69,6 +96,91 @@ void pci_disable_rom(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(pci_disable_rom); +static bool pci_rom_header_valid(struct pci_dev *pdev, void __iomem *image, + void __iomem *rom, size_t size, + bool expect_valid) +{ + unsigned long rom_end = (unsigned long)rom + size - 1; + unsigned long header_end; + u16 signature; + + /* + * Per PCI Firmware r3.3, sec 5.1, each image must start on a + * 512-byte boundary and must contain the PCI Expansion ROM header. + * Because @rom is page-aligned (returned by ioremap()), checking + * 512-byte alignment of @image is equivalent to enforcing the + * spec's sector-aligned layout within the ROM. This also + * satisfies the natural-alignment requirement of readw() on archs + * such as arm64 that disallow unaligned IOMEM access. + */ + if (!IS_ALIGNED((unsigned long)image, PCI_ROM_IMAGE_SECTOR_SIZE)) + return false; + + if (check_add_overflow((unsigned long)image, PCI_ROM_HEADER_SIZE - 1, + &header_end)) + return false; + + if (image < rom || header_end > rom_end) + return false; + + /* Standard PCI ROMs start out with these bytes 55 AA */ + signature = readw(image); + if (signature != PCI_ROM_IMAGE_SIGNATURE) { + if (expect_valid) { + pci_info(pdev, "Invalid PCI ROM header signature: expecting %#06x, got %#06x\n", + PCI_ROM_IMAGE_SIGNATURE, signature); + } else { + pci_info(pdev, "No more images in PCI ROM\n"); + } + return false; + } + + return true; +} + +static bool pci_rom_data_struct_valid(struct pci_dev *pdev, void __iomem *pds, + void __iomem *rom, size_t size) +{ + unsigned long rom_end = (unsigned long)rom + size - 1; + unsigned long end; + u32 signature; + u16 data_len; + + /* + * Some CPU architectures require IOMEM access addresses to be + * aligned, for example arm64, so since we're about to call + * readl(), check here for 4-byte alignment. + */ + if (!IS_ALIGNED((unsigned long)pds, 4)) + return false; + + if (check_add_overflow((unsigned long)pds, PCI_ROM_DATA_STRUCT_LEN + 1, + &end)) + return false; + + if (pds < rom || end > rom_end) + return false; + + signature = readl(pds); + if (signature != PCI_ROM_DATA_STRUCT_SIGNATURE) { + pci_info(pdev, "Invalid PCI ROM data signature: expecting %#010x, got %#010x\n", + PCI_ROM_DATA_STRUCT_SIGNATURE, signature); + return false; + } + + data_len = readw(pds + PCI_ROM_DATA_STRUCT_LEN); + if (data_len < PCI_ROM_DATA_STRUCT_MIN_LEN || data_len == U16_MAX) + return false; + + if (check_add_overflow((unsigned long)pds, data_len - 1, &end)) + return false; + + if (end > rom_end) + return false; + + return true; +} + /** * pci_get_rom_size - obtain the actual size of the ROM image * @pdev: target PCI device @@ -84,37 +196,28 @@ static size_t pci_get_rom_size(struct pci_dev *pdev, void __iomem *rom, size_t size) { void __iomem *image; - int last_image; unsigned int length; + bool last_image; image = rom; do { void __iomem *pds; - /* Standard PCI ROMs start out with these bytes 55 AA */ - if (readw(image) != 0xAA55) { - pci_info(pdev, "Invalid PCI ROM header signature: expecting 0xaa55, got %#06x\n", - readw(image)); + if (!pci_rom_header_valid(pdev, image, rom, size, true)) break; - } - /* get the PCI data structure and check its "PCIR" signature */ - pds = image + readw(image + 24); - if (readl(pds) != 0x52494350) { - pci_info(pdev, "Invalid PCI ROM data signature: expecting 0x52494350, got %#010x\n", - readl(pds)); + + /* Get the PCI data structure and check its "PCIR" signature */ + pds = image + readw(image + PCI_ROM_POINTER_TO_DATA_STRUCT); + if (!pci_rom_data_struct_valid(pdev, pds, rom, size)) break; - } - last_image = readb(pds + 21) & 0x80; - length = readw(pds + 16); - image += length * 512; - /* Avoid iterating through memory outside the resource window */ - if (image >= rom + size) + + last_image = readb(pds + PCI_ROM_LAST_IMAGE_INDICATOR) & + PCI_ROM_LAST_IMAGE_INDICATOR_BIT; + length = readw(pds + PCI_ROM_IMAGE_LEN); + image += length * PCI_ROM_IMAGE_SECTOR_SIZE; + + if (!last_image && + !pci_rom_header_valid(pdev, image, rom, size, false)) break; - if (!last_image) { - if (readw(image) != 0xAA55) { - pci_info(pdev, "No more image in the PCI ROM\n"); - break; - } - } } while (length && !last_image); /* never return a size larger than the PCI resource window */ diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index f5305c8fdca43..6e5cc4086a9e2 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -197,13 +197,14 @@ enum cmn_model { CMN600 = 1, CMN650 = 2, - CMN700 = 4, - CI700 = 8, + CI700 = 4, + CMN700 = 8, CMNS3 = 16, /* ...and then we can use bitmap tricks for commonality */ CMN_ANY = -1, NOT_CMN600 = -2, - CMN_650ON = CMN650 | CMN700 | CMNS3, + CMN_700ON = ~(CMN700 - 1), + CMN_650ON = CMN_700ON | CMN650, }; /* Actual part numbers and revision IDs defined by the hardware */ @@ -919,14 +920,14 @@ static struct attribute *arm_cmn_event_attrs[] = { CMN_EVENT_DVM(NOT_CMN600, txsnp_stall, 0x0a), CMN_EVENT_DVM(NOT_CMN600, trkfull, 0x0b), CMN_EVENT_DVM_OCC(NOT_CMN600, trk_occupancy, 0x0c), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_cxha, 0x0d), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_pdn, 0x0e), - CMN_EVENT_DVM(CMN700, trk_alloc, 0x0f), - CMN_EVENT_DVM(CMN700, trk_cxha_alloc, 0x10), - CMN_EVENT_DVM(CMN700, trk_pdn_alloc, 0x11), - CMN_EVENT_DVM(CMN700, txsnp_stall_limit, 0x12), - CMN_EVENT_DVM(CMN700, rxsnp_stall_starv, 0x13), - CMN_EVENT_DVM(CMN700, txsnp_sync_stall_op, 0x14), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_cxha, 0x0d), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_pdn, 0x0e), + CMN_EVENT_DVM(CMN_700ON, trk_alloc, 0x0f), + CMN_EVENT_DVM(CMN_700ON, trk_cxha_alloc, 0x10), + CMN_EVENT_DVM(CMN_700ON, trk_pdn_alloc, 0x11), + CMN_EVENT_DVM(CMN_700ON, txsnp_stall_limit, 0x12), + CMN_EVENT_DVM(CMN_700ON, rxsnp_stall_starv, 0x13), + CMN_EVENT_DVM(CMN_700ON, txsnp_sync_stall_op, 0x14), CMN_EVENT_HNF(CMN_ANY, cache_miss, 0x01), CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access, 0x02), diff --git a/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c b/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c index ece3574435217..368679436d862 100644 --- a/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c +++ b/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c @@ -286,7 +286,7 @@ static int mixel_lvds_phy_reset(struct device *dev) regmap_write(priv->regmap, PHY_CTRL, CTRL_RESET_VAL); - pm_runtime_put(dev); + pm_runtime_put_sync(dev); return 0; } @@ -345,7 +345,9 @@ static int mixel_lvds_phy_probe(struct platform_device *pdev) dev_set_drvdata(dev, priv); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; ret = mixel_lvds_phy_reset(dev); if (ret) { @@ -355,17 +357,15 @@ static int mixel_lvds_phy_probe(struct platform_device *pdev) for (i = 0; i < PHY_NUM; i++) { lvds_phy = devm_kzalloc(dev, sizeof(*lvds_phy), GFP_KERNEL); - if (!lvds_phy) { - ret = -ENOMEM; - goto err; - } + if (!lvds_phy) + return -ENOMEM; phy = devm_phy_create(dev, NULL, &mixel_lvds_phy_ops); if (IS_ERR(phy)) { ret = PTR_ERR(phy); dev_err(dev, "failed to create PHY for channel%d: %d\n", i, ret); - goto err; + return ret; } lvds_phy->phy = phy; @@ -379,19 +379,10 @@ static int mixel_lvds_phy_probe(struct platform_device *pdev) if (IS_ERR(phy_provider)) { ret = PTR_ERR(phy_provider); dev_err(dev, "failed to register PHY provider: %d\n", ret); - goto err; + return ret; } return 0; -err: - pm_runtime_disable(dev); - - return ret; -} - -static void mixel_lvds_phy_remove(struct platform_device *pdev) -{ - pm_runtime_disable(&pdev->dev); } static int __maybe_unused mixel_lvds_phy_runtime_suspend(struct device *dev) @@ -432,7 +423,6 @@ MODULE_DEVICE_TABLE(of, mixel_lvds_phy_of_match); static struct platform_driver mixel_lvds_phy_driver = { .probe = mixel_lvds_phy_probe, - .remove = mixel_lvds_phy_remove, .driver = { .pm = &mixel_lvds_phy_pm_ops, .name = "mixel-lvds-phy", diff --git a/drivers/phy/phy-can-transceiver.c b/drivers/phy/phy-can-transceiver.c index 2b52e47f247a2..1808f903c0570 100644 --- a/drivers/phy/phy-can-transceiver.c +++ b/drivers/phy/phy-can-transceiver.c @@ -162,6 +162,9 @@ static int can_transceiver_phy_probe(struct platform_device *pdev) int err, i, num_ch = 1; match = of_match_node(can_transceiver_phy_ids, pdev->dev.of_node); + if (!match || !match->data) + return -ENODEV; + drvdata = match->data; if (drvdata->flags & CAN_TRANSCEIVER_DUAL_CH) num_ch = 2; diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index 03f2e3ee065f4..75131b6e6eeac 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -548,6 +548,7 @@ config PINCTRL_ST config PINCTRL_STMFX tristate "STMicroelectronics STMFX GPIO expander pinctrl driver" + depends on OF depends on I2C depends on HAS_IOMEM select GENERIC_PINCONF diff --git a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c index 227c37c360e19..305233fc19876 100644 --- a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c +++ b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c @@ -343,7 +343,7 @@ static int cs42l43_pin_set_db(struct cs42l43_pin *priv, unsigned int pin, return regmap_update_bits(priv->regmap, CS42L43_GPIO_CTRL2, CS42L43_GPIO1_DEGLITCH_BYP_MASK << pin, - !!us << pin); + !us << pin); } static int cs42l43_pin_config_get(struct pinctrl_dev *pctldev, @@ -499,12 +499,10 @@ static int cs42l43_gpio_set(struct gpio_chip *chip, unsigned int offset, ret = regmap_update_bits(priv->regmap, CS42L43_GPIO_CTRL1, BIT(shift), value << shift); - if (ret) - return ret; pm_runtime_put(priv->dev); - return 0; + return ret; } static int cs42l43_gpio_direction_out(struct gpio_chip *chip, diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 2a3c04eedc5f3..47ac92ea98c2c 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -246,7 +246,7 @@ static int mtk_eint_irq_set_wake(struct irq_data *d, unsigned int on) } static void mtk_eint_chip_write_mask(const struct mtk_eint *eint, - void __iomem *base, unsigned int **buf) + unsigned int **buf) { int inst, port, port_num; void __iomem *reg; @@ -425,7 +425,7 @@ static void mtk_eint_irq_handler(struct irq_desc *desc) int mtk_eint_do_suspend(struct mtk_eint *eint) { - mtk_eint_chip_write_mask(eint, eint->base, eint->wake_mask); + mtk_eint_chip_write_mask(eint, eint->wake_mask); return 0; } @@ -433,7 +433,7 @@ EXPORT_SYMBOL_GPL(mtk_eint_do_suspend); int mtk_eint_do_resume(struct mtk_eint *eint) { - mtk_eint_chip_write_mask(eint, eint->base, eint->cur_mask); + mtk_eint_chip_write_mask(eint, eint->cur_mask); return 0; } diff --git a/drivers/pinctrl/mediatek/pinctrl-airoha.c b/drivers/pinctrl/mediatek/pinctrl-airoha.c index 995ba6175c950..bf5ebb31e6350 100644 --- a/drivers/pinctrl/mediatek/pinctrl-airoha.c +++ b/drivers/pinctrl/mediatek/pinctrl-airoha.c @@ -539,6 +539,7 @@ static const int en7581_gpio28_pins[] = { 41 }; static const int en7581_gpio29_pins[] = { 42 }; static const int en7581_gpio30_pins[] = { 43 }; static const int en7581_gpio31_pins[] = { 44 }; +static const int en7581_gpio32_pins[] = { 45 }; static const int en7581_gpio33_pins[] = { 46 }; static const int en7581_gpio34_pins[] = { 47 }; static const int en7581_gpio35_pins[] = { 48 }; @@ -623,6 +624,7 @@ static const struct pingroup en7581_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio29", en7581_gpio29), PINCTRL_PIN_GROUP("gpio30", en7581_gpio30), PINCTRL_PIN_GROUP("gpio31", en7581_gpio31), + PINCTRL_PIN_GROUP("gpio32", en7581_gpio32), PINCTRL_PIN_GROUP("gpio33", en7581_gpio33), PINCTRL_PIN_GROUP("gpio34", en7581_gpio34), PINCTRL_PIN_GROUP("gpio35", en7581_gpio35), @@ -746,7 +748,8 @@ static const int an7583_gpio17_pins[] = { 19 }; static const int an7583_gpio18_pins[] = { 20 }; static const int an7583_gpio19_pins[] = { 21 }; static const int an7583_gpio20_pins[] = { 22 }; -static const int an7583_gpio21_pins[] = { 24 }; +static const int an7583_gpio21_pins[] = { 23 }; +static const int an7583_gpio22_pins[] = { 24 }; static const int an7583_gpio23_pins[] = { 25 }; static const int an7583_gpio24_pins[] = { 26 }; static const int an7583_gpio25_pins[] = { 27 }; @@ -756,6 +759,7 @@ static const int an7583_gpio28_pins[] = { 30 }; static const int an7583_gpio29_pins[] = { 31 }; static const int an7583_gpio30_pins[] = { 32 }; static const int an7583_gpio31_pins[] = { 33 }; +static const int an7583_gpio32_pins[] = { 34 }; static const int an7583_gpio33_pins[] = { 35 }; static const int an7583_gpio34_pins[] = { 36 }; static const int an7583_gpio35_pins[] = { 37 }; @@ -825,6 +829,7 @@ static const struct pingroup an7583_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio19", an7583_gpio19), PINCTRL_PIN_GROUP("gpio20", an7583_gpio20), PINCTRL_PIN_GROUP("gpio21", an7583_gpio21), + PINCTRL_PIN_GROUP("gpio22", an7583_gpio22), PINCTRL_PIN_GROUP("gpio23", an7583_gpio23), PINCTRL_PIN_GROUP("gpio24", an7583_gpio24), PINCTRL_PIN_GROUP("gpio25", an7583_gpio25), @@ -834,6 +839,7 @@ static const struct pingroup an7583_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio29", an7583_gpio29), PINCTRL_PIN_GROUP("gpio30", an7583_gpio30), PINCTRL_PIN_GROUP("gpio31", an7583_gpio31), + PINCTRL_PIN_GROUP("gpio32", an7583_gpio32), PINCTRL_PIN_GROUP("gpio33", an7583_gpio33), PINCTRL_PIN_GROUP("gpio34", an7583_gpio34), PINCTRL_PIN_GROUP("gpio35", an7583_gpio35), @@ -871,10 +877,8 @@ static const char *const pcm_spi_groups[] = { "pcm_spi", "pcm_spi_int", "pcm_spi_cs2_p156", "pcm_spi_cs2_p128", "pcm_spi_cs3", "pcm_spi_cs4" }; -static const char *const an7583_pcm_spi_groups[] = { "pcm_spi", "pcm_spi_int", - "pcm_spi_rst", "pcm_spi_cs1", - "pcm_spi_cs2", "pcm_spi_cs3", - "pcm_spi_cs4" }; +static const char *const an7583_pcm_spi_groups[] = { "pcm_spi", + "pcm_spi_rst", "pcm_spi_cs1" }; static const char *const i2s_groups[] = { "i2s" }; static const char *const emmc_groups[] = { "emmc" }; static const char *const pnand_groups[] = { "pnand" }; @@ -902,7 +906,30 @@ static const char *const pwm_groups[] = { "gpio0", "gpio1", "gpio40", "gpio41", "gpio42", "gpio43", "gpio44", "gpio45", - "gpio46", "gpio47" }; + "gpio46" }; +static const char *const an7583_pwm_groups[] = { "gpio0", "gpio1", + "gpio2", "gpio3", + "gpio4", "gpio5", + "gpio6", "gpio7", + "gpio8", "gpio9", + "gpio10", "gpio11", + "gpio12", "gpio13", + "gpio14", "gpio15", + "gpio16", "gpio17", + "gpio18", "gpio19", + "gpio20", "gpio21", + "gpio22", "gpio23", + "gpio24", "gpio25", + "gpio26", "gpio27", + "gpio28", "gpio29", + "gpio30", "gpio31", + "gpio36", "gpio37", + "gpio38", "gpio39", + "gpio40", "gpio41", + "gpio42", "gpio43", + "gpio44", "gpio45", + "gpio46", "gpio47", + "gpio48" }; static const char *const phy1_led0_groups[] = { "gpio33", "gpio34", "gpio35", "gpio42" }; static const char *const phy2_led0_groups[] = { "gpio33", "gpio34", @@ -1500,7 +1527,54 @@ static const struct airoha_pinctrl_func_group pwm_func_group[] = { AIROHA_PINCTRL_PWM_EXT("gpio44", GPIO44_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio45", GPIO45_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio46", GPIO46_FLASH_MODE_CFG), +}; + +static const struct airoha_pinctrl_func_group an7583_pwm_func_group[] = { + AIROHA_PINCTRL_PWM("gpio0", GPIO0_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio1", GPIO1_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio2", GPIO2_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio3", GPIO3_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio4", GPIO4_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio5", GPIO5_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio6", GPIO6_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio7", GPIO7_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio8", GPIO8_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio9", GPIO9_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio10", GPIO10_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio11", GPIO11_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio12", GPIO12_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio13", GPIO13_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio14", GPIO14_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio15", GPIO15_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio16", GPIO16_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio17", GPIO17_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio18", GPIO18_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio19", GPIO19_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio20", GPIO20_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio21", GPIO21_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio22", GPIO22_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio23", GPIO23_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio24", GPIO24_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio25", GPIO25_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio26", GPIO26_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio27", GPIO27_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio28", GPIO28_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio29", GPIO29_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio30", GPIO30_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio31", GPIO31_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio36", GPIO36_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio37", GPIO37_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio38", GPIO38_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio39", GPIO39_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio40", GPIO40_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio41", GPIO41_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio42", GPIO42_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio43", GPIO43_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio44", GPIO44_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio45", GPIO45_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio46", GPIO46_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio47", GPIO47_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio48", GPIO48_FLASH_MODE_CFG), }; #define AIROHA_PINCTRL_PHY_LED0(gpio, mux_val, map_mask, map_val) \ @@ -1618,13 +1692,13 @@ static const struct airoha_pinctrl_func_group phy3_led1_func_group[] = { static const struct airoha_pinctrl_func_group phy4_led1_func_group[] = { AIROHA_PINCTRL_PHY_LED1("gpio43", GPIO_LAN0_LED1_MODE_MASK, - LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)), + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)), AIROHA_PINCTRL_PHY_LED1("gpio44", GPIO_LAN1_LED1_MODE_MASK, - LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)), + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)), AIROHA_PINCTRL_PHY_LED1("gpio45", GPIO_LAN2_LED1_MODE_MASK, - LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)), + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)), AIROHA_PINCTRL_PHY_LED1("gpio46", GPIO_LAN3_LED1_MODE_MASK, - LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)), + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)), }; static const struct airoha_pinctrl_func_group an7583_phy1_led0_func_group[] = { @@ -1678,7 +1752,7 @@ static const struct airoha_pinctrl_func_group an7583_phy1_led1_func_group[] = { LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(0)), AIROHA_PINCTRL_PHY_LED1("gpio10", GPIO_LAN2_LED1_MODE_MASK, LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(0)), - AIROHA_PINCTRL_PHY_LED1("gpio1", GPIO_LAN3_LED1_MODE_MASK, + AIROHA_PINCTRL_PHY_LED1("gpio11", GPIO_LAN3_LED1_MODE_MASK, LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(0)), }; @@ -1706,13 +1780,13 @@ static const struct airoha_pinctrl_func_group an7583_phy3_led1_func_group[] = { static const struct airoha_pinctrl_func_group an7583_phy4_led1_func_group[] = { AIROHA_PINCTRL_PHY_LED1("gpio8", GPIO_LAN0_LED1_MODE_MASK, - LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)), + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)), AIROHA_PINCTRL_PHY_LED1("gpio9", GPIO_LAN1_LED1_MODE_MASK, - LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)), + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)), AIROHA_PINCTRL_PHY_LED1("gpio10", GPIO_LAN2_LED1_MODE_MASK, - LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)), + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)), AIROHA_PINCTRL_PHY_LED1("gpio11", GPIO_LAN3_LED1_MODE_MASK, - LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)), + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)), }; static const struct airoha_pinctrl_func en7581_pinctrl_funcs[] = { @@ -1755,7 +1829,7 @@ static const struct airoha_pinctrl_func an7583_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("emmc", emmc), PINCTRL_FUNC_DESC("pnand", pnand), PINCTRL_FUNC_DESC("pcie_reset", an7583_pcie_reset), - PINCTRL_FUNC_DESC("pwm", pwm), + PINCTRL_FUNC_DESC("pwm", an7583_pwm), PINCTRL_FUNC_DESC("phy1_led0", an7583_phy1_led0), PINCTRL_FUNC_DESC("phy2_led0", an7583_phy2_led0), PINCTRL_FUNC_DESC("phy3_led0", an7583_phy3_led0), @@ -1794,7 +1868,7 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(29, REG_GPIO_L_PU, BIT(16)), PINCTRL_CONF_DESC(30, REG_GPIO_L_PU, BIT(17)), PINCTRL_CONF_DESC(31, REG_GPIO_L_PU, BIT(18)), - PINCTRL_CONF_DESC(32, REG_GPIO_L_PU, BIT(18)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_PU, BIT(19)), PINCTRL_CONF_DESC(33, REG_GPIO_L_PU, BIT(20)), PINCTRL_CONF_DESC(34, REG_GPIO_L_PU, BIT(21)), PINCTRL_CONF_DESC(35, REG_GPIO_L_PU, BIT(22)), @@ -1847,7 +1921,7 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(18, REG_GPIO_L_PU, BIT(16)), PINCTRL_CONF_DESC(19, REG_GPIO_L_PU, BIT(17)), PINCTRL_CONF_DESC(20, REG_GPIO_L_PU, BIT(18)), - PINCTRL_CONF_DESC(21, REG_GPIO_L_PU, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_PU, BIT(19)), PINCTRL_CONF_DESC(22, REG_GPIO_L_PU, BIT(20)), PINCTRL_CONF_DESC(23, REG_GPIO_L_PU, BIT(21)), PINCTRL_CONF_DESC(24, REG_GPIO_L_PU, BIT(22)), @@ -1911,7 +1985,7 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(29, REG_GPIO_L_PD, BIT(16)), PINCTRL_CONF_DESC(30, REG_GPIO_L_PD, BIT(17)), PINCTRL_CONF_DESC(31, REG_GPIO_L_PD, BIT(18)), - PINCTRL_CONF_DESC(32, REG_GPIO_L_PD, BIT(18)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_PD, BIT(19)), PINCTRL_CONF_DESC(33, REG_GPIO_L_PD, BIT(20)), PINCTRL_CONF_DESC(34, REG_GPIO_L_PD, BIT(21)), PINCTRL_CONF_DESC(35, REG_GPIO_L_PD, BIT(22)), @@ -1964,7 +2038,7 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(18, REG_GPIO_L_PD, BIT(16)), PINCTRL_CONF_DESC(19, REG_GPIO_L_PD, BIT(17)), PINCTRL_CONF_DESC(20, REG_GPIO_L_PD, BIT(18)), - PINCTRL_CONF_DESC(21, REG_GPIO_L_PD, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_PD, BIT(19)), PINCTRL_CONF_DESC(22, REG_GPIO_L_PD, BIT(20)), PINCTRL_CONF_DESC(23, REG_GPIO_L_PD, BIT(21)), PINCTRL_CONF_DESC(24, REG_GPIO_L_PD, BIT(22)), @@ -2028,7 +2102,7 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(29, REG_GPIO_L_E2, BIT(16)), PINCTRL_CONF_DESC(30, REG_GPIO_L_E2, BIT(17)), PINCTRL_CONF_DESC(31, REG_GPIO_L_E2, BIT(18)), - PINCTRL_CONF_DESC(32, REG_GPIO_L_E2, BIT(18)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_E2, BIT(19)), PINCTRL_CONF_DESC(33, REG_GPIO_L_E2, BIT(20)), PINCTRL_CONF_DESC(34, REG_GPIO_L_E2, BIT(21)), PINCTRL_CONF_DESC(35, REG_GPIO_L_E2, BIT(22)), @@ -2081,7 +2155,7 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(18, REG_GPIO_L_E2, BIT(16)), PINCTRL_CONF_DESC(19, REG_GPIO_L_E2, BIT(17)), PINCTRL_CONF_DESC(20, REG_GPIO_L_E2, BIT(18)), - PINCTRL_CONF_DESC(21, REG_GPIO_L_E2, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_E2, BIT(19)), PINCTRL_CONF_DESC(22, REG_GPIO_L_E2, BIT(20)), PINCTRL_CONF_DESC(23, REG_GPIO_L_E2, BIT(21)), PINCTRL_CONF_DESC(24, REG_GPIO_L_E2, BIT(22)), @@ -2145,7 +2219,7 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(29, REG_GPIO_L_E4, BIT(16)), PINCTRL_CONF_DESC(30, REG_GPIO_L_E4, BIT(17)), PINCTRL_CONF_DESC(31, REG_GPIO_L_E4, BIT(18)), - PINCTRL_CONF_DESC(32, REG_GPIO_L_E4, BIT(18)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_E4, BIT(19)), PINCTRL_CONF_DESC(33, REG_GPIO_L_E4, BIT(20)), PINCTRL_CONF_DESC(34, REG_GPIO_L_E4, BIT(21)), PINCTRL_CONF_DESC(35, REG_GPIO_L_E4, BIT(22)), @@ -2198,7 +2272,7 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(18, REG_GPIO_L_E4, BIT(16)), PINCTRL_CONF_DESC(19, REG_GPIO_L_E4, BIT(17)), PINCTRL_CONF_DESC(20, REG_GPIO_L_E4, BIT(18)), - PINCTRL_CONF_DESC(21, REG_GPIO_L_E4, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_E4, BIT(19)), PINCTRL_CONF_DESC(22, REG_GPIO_L_E4, BIT(20)), PINCTRL_CONF_DESC(23, REG_GPIO_L_E4, BIT(21)), PINCTRL_CONF_DESC(24, REG_GPIO_L_E4, BIT(22)), diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8167.c b/drivers/pinctrl/mediatek/pinctrl-mt8167.c index 143c266222725..c812d614e9d45 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt8167.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt8167.c @@ -244,7 +244,7 @@ static const struct mtk_pin_ies_smt_set mt8167_smt_set[] = { MTK_PIN_IES_SMT_SPEC(24, 25, 0xA00, 12), MTK_PIN_IES_SMT_SPEC(26, 30, 0xA00, 0), MTK_PIN_IES_SMT_SPEC(31, 33, 0xA00, 1), - MTK_PIN_IES_SMT_SPEC(34, 39, 0xA900, 2), + MTK_PIN_IES_SMT_SPEC(34, 39, 0xA00, 2), MTK_PIN_IES_SMT_SPEC(40, 40, 0xA10, 11), MTK_PIN_IES_SMT_SPEC(41, 43, 0xA00, 10), MTK_PIN_IES_SMT_SPEC(44, 47, 0xA00, 11), diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8516.c b/drivers/pinctrl/mediatek/pinctrl-mt8516.c index abda75d4354e2..68d6638e7f4b1 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt8516.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt8516.c @@ -244,7 +244,7 @@ static const struct mtk_pin_ies_smt_set mt8516_smt_set[] = { MTK_PIN_IES_SMT_SPEC(24, 25, 0xA00, 12), MTK_PIN_IES_SMT_SPEC(26, 30, 0xA00, 0), MTK_PIN_IES_SMT_SPEC(31, 33, 0xA00, 1), - MTK_PIN_IES_SMT_SPEC(34, 39, 0xA900, 2), + MTK_PIN_IES_SMT_SPEC(34, 39, 0xA00, 2), MTK_PIN_IES_SMT_SPEC(40, 40, 0xA10, 11), MTK_PIN_IES_SMT_SPEC(41, 43, 0xA00, 10), MTK_PIN_IES_SMT_SPEC(44, 47, 0xA00, 11), diff --git a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c index 35d27626a336b..1bd58fbbd26ac 100644 --- a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c +++ b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c @@ -548,11 +548,11 @@ static int aml_pinconf_set_output_drive(struct aml_pinctrl *info, { int ret; - ret = aml_pinconf_set_output(info, pin, true); + ret = aml_pinconf_set_drive(info, pin, high); if (ret) return ret; - return aml_pinconf_set_drive(info, pin, high); + return aml_pinconf_set_output(info, pin, true); } static int aml_pinconf_set(struct pinctrl_dev *pcdev, unsigned int pin, @@ -921,15 +921,14 @@ static int aml_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio, unsigned int bit, reg; int ret; - aml_gpio_calc_reg_and_bit(bank, AML_REG_DIR, gpio, ®, &bit); - ret = regmap_update_bits(bank->reg_gpio, reg, BIT(bit), 0); + aml_gpio_calc_reg_and_bit(bank, AML_REG_OUT, gpio, ®, &bit); + ret = regmap_update_bits(bank->reg_gpio, reg, BIT(bit), + value ? BIT(bit) : 0); if (ret < 0) return ret; - aml_gpio_calc_reg_and_bit(bank, AML_REG_OUT, gpio, ®, &bit); - - return regmap_update_bits(bank->reg_gpio, reg, BIT(bit), - value ? BIT(bit) : 0); + aml_gpio_calc_reg_and_bit(bank, AML_REG_DIR, gpio, ®, &bit); + return regmap_update_bits(bank->reg_gpio, reg, BIT(bit), 0); } static int aml_gpio_set(struct gpio_chip *chip, unsigned int gpio, int value) diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c index 4507dc8b5563c..18295b15ecd9d 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.c +++ b/drivers/pinctrl/meson/pinctrl-meson.c @@ -619,7 +619,7 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc) pc->chip.set = meson_gpio_set; pc->chip.base = -1; pc->chip.ngpio = pc->data->num_pins; - pc->chip.can_sleep = true; + pc->chip.can_sleep = false; ret = gpiochip_add_data(&pc->chip, pc); if (ret) { diff --git a/drivers/pinctrl/nuvoton/pinctrl-ma35.c b/drivers/pinctrl/nuvoton/pinctrl-ma35.c index f01344201628f..dafa85c105a13 100644 --- a/drivers/pinctrl/nuvoton/pinctrl-ma35.c +++ b/drivers/pinctrl/nuvoton/pinctrl-ma35.c @@ -1014,7 +1014,8 @@ static int ma35_pinctrl_parse_groups(struct fwnode_handle *fwnode, struct group_ grp->data = pin; for (i = 0, j = 0; i < count; i += 3, j++) { - pin->offset = elems[i] * MA35_MFP_REG_SZ_PER_BANK + MA35_MFP_REG_BASE; + pin->offset = elems[i] * MA35_MFP_REG_SZ_PER_BANK + MA35_MFP_REG_BASE + + (elems[i + 1] >= 8 ? 4 : 0); pin->shift = (elems[i + 1] * MA35_MFP_BITS_PER_PORT) % 32; pin->muxval = elems[i + 2]; pin->configs = configs; diff --git a/drivers/pinctrl/nuvoton/pinctrl-ma35d1.c b/drivers/pinctrl/nuvoton/pinctrl-ma35d1.c index eafa06ca08791..9d4627c80a52c 100644 --- a/drivers/pinctrl/nuvoton/pinctrl-ma35d1.c +++ b/drivers/pinctrl/nuvoton/pinctrl-ma35d1.c @@ -113,6 +113,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x0, "GPA14"), MA35_MUX(0x2, "UART7_RXD"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL3_DM"), MA35_MUX(0x6, "NAND_nWP"), MA35_MUX(0x7, "EBI_AD14"), MA35_MUX(0x9, "EBI_ADR14")), @@ -123,6 +124,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x3, "UART6_RXD"), MA35_MUX(0x4, "I2C4_SDA"), MA35_MUX(0x5, "CAN2_RXD"), + MA35_MUX(0x6, "USBHL0_DM"), MA35_MUX(0x7, "EBI_ALE"), MA35_MUX(0x9, "QEI0_A"), MA35_MUX(0xb, "TM1"), @@ -187,6 +189,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x1, "EPWM2_CH5"), MA35_MUX(0x2, "UART2_RXD"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL2_DM"), MA35_MUX(0x5, "SPI0_MOSI"), MA35_MUX(0x6, "EBI_MCLK"), MA35_MUX(0x7, "CCAP1_VSYNC"), @@ -202,6 +205,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x1, "EPWM2_BRAKE1"), MA35_MUX(0x2, "UART2_TXD"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL2_DP"), MA35_MUX(0x5, "SPI0_MISO"), MA35_MUX(0x6, "I2S1_MCLK"), MA35_MUX(0x7, "CCAP1_SFIELD"), @@ -220,6 +224,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x4, "I2C3_SDA"), MA35_MUX(0x5, "CAN2_RXD"), MA35_MUX(0x6, "I2S1_LRCK"), + MA35_MUX(0x7, "USBHL1_DM"), MA35_MUX(0x8, "ADC0_CH4"), MA35_MUX(0x9, "EBI_ADR16"), MA35_MUX(0xe, "ECAP2_IC0")), @@ -231,6 +236,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x4, "I2C3_SCL"), MA35_MUX(0x5, "CAN2_TXD"), MA35_MUX(0x6, "I2S1_BCLK"), + MA35_MUX(0x7, "USBHL1_DP"), MA35_MUX(0x8, "ADC0_CH5"), MA35_MUX(0x9, "EBI_ADR17"), MA35_MUX(0xe, "ECAP2_IC1")), @@ -239,6 +245,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x1, "EPWM2_CH2"), MA35_MUX(0x2, "UART4_RXD"), MA35_MUX(0x3, "CAN1_RXD"), + MA35_MUX(0x4, "USBHL3_DM"), MA35_MUX(0x5, "I2C4_SDA"), MA35_MUX(0x6, "I2S1_DI"), MA35_MUX(0x8, "ADC0_CH6"), @@ -249,6 +256,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x1, "EPWM2_CH3"), MA35_MUX(0x2, "UART4_TXD"), MA35_MUX(0x3, "CAN1_TXD"), + MA35_MUX(0x4, "USBHL3_DP"), MA35_MUX(0x5, "I2C4_SCL"), MA35_MUX(0x6, "I2S1_DO"), MA35_MUX(0x8, "ADC0_CH7"), @@ -264,10 +272,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_PIN(34, PC2, 0x90, 0x8, MA35_MUX(0x0, "GPC2"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL4_DM"), MA35_MUX(0x6, "SD0_DAT0/eMMC0_DAT0")), MA35_PIN(35, PC3, 0x90, 0xc, MA35_MUX(0x0, "GPC3"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL4_DP"), MA35_MUX(0x6, "SD0_DAT1/eMMC0_DAT1")), MA35_PIN(36, PC4, 0x90, 0x10, MA35_MUX(0x0, "GPC4"), @@ -280,65 +290,100 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_PIN(38, PC6, 0x90, 0x18, MA35_MUX(0x0, "GPC6"), MA35_MUX(0x3, "CAN1_RXD"), + MA35_MUX(0x4, "USBHL5_DM"), MA35_MUX(0x6, "SD0_nCD")), MA35_PIN(39, PC7, 0x90, 0x1c, MA35_MUX(0x0, "GPC7"), MA35_MUX(0x3, "CAN1_TXD"), + MA35_MUX(0x4, "USBHL5_DP"), MA35_MUX(0x6, "SD0_WP")), - MA35_PIN(40, PC12, 0x94, 0x10, + MA35_PIN(40, PC8, 0x94, 0x0, + MA35_MUX(0x0, "GPC8"), + MA35_MUX(0x1, "EPWM2_CH0"), + MA35_MUX(0x2, "UART10_nCTS"), + MA35_MUX(0x3, "UART9_RXD"), + MA35_MUX(0x4, "I2C0_SDA"), + MA35_MUX(0x5, "SPI1_SS0"), + MA35_MUX(0x6, "SD0_DAT4/eMMC0_DAT4")), + MA35_PIN(41, PC9, 0x94, 0x4, + MA35_MUX(0x0, "GPC9"), + MA35_MUX(0x1, "EPWM2_CH1"), + MA35_MUX(0x2, "UART10_nRTS"), + MA35_MUX(0x3, "UART9_TXD"), + MA35_MUX(0x4, "I2C0_SCL"), + MA35_MUX(0x5, "SPI1_CLK"), + MA35_MUX(0x6, "SD0_DAT5/eMMC0_DAT5")), + MA35_PIN(42, PC10, 0x94, 0x8, + MA35_MUX(0x0, "GPC10"), + MA35_MUX(0x1, "EPWM2_CH2"), + MA35_MUX(0x2, "UART10_RXD"), + MA35_MUX(0x3, "CAN2_RXD"), + MA35_MUX(0x4, "USBHL0_DM"), + MA35_MUX(0x5, "SPI1_MOSI"), + MA35_MUX(0x6, "SD0_DAT6/eMMC0_DAT6")), + MA35_PIN(43, PC11, 0x94, 0xc, + MA35_MUX(0x0, "GPC11"), + MA35_MUX(0x1, "EPWM2_CH3"), + MA35_MUX(0x2, "UART10_TXD"), + MA35_MUX(0x3, "CAN2_TXD"), + MA35_MUX(0x4, "USBHL0_DP"), + MA35_MUX(0x5, "SPI1_MISO"), + MA35_MUX(0x6, "SD0_DAT7/eMMC0_DAT7")), + MA35_PIN(44, PC12, 0x94, 0x10, MA35_MUX(0x0, "GPC12"), MA35_MUX(0x2, "UART12_nCTS"), MA35_MUX(0x3, "UART11_RXD"), MA35_MUX(0x6, "LCM_DATA16")), - MA35_PIN(41, PC13, 0x94, 0x14, + MA35_PIN(45, PC13, 0x94, 0x14, MA35_MUX(0x0, "GPC13"), MA35_MUX(0x2, "UART12_nRTS"), MA35_MUX(0x3, "UART11_TXD"), MA35_MUX(0x6, "LCM_DATA17")), - MA35_PIN(42, PC14, 0x94, 0x18, + MA35_PIN(46, PC14, 0x94, 0x18, MA35_MUX(0x0, "GPC14"), MA35_MUX(0x2, "UART12_RXD"), MA35_MUX(0x6, "LCM_DATA18")), - MA35_PIN(43, PC15, 0x94, 0x1c, + MA35_PIN(47, PC15, 0x94, 0x1c, MA35_MUX(0x0, "GPC15"), MA35_MUX(0x2, "UART12_TXD"), MA35_MUX(0x6, "LCM_DATA19"), MA35_MUX(0x7, "LCM_MPU_TE"), MA35_MUX(0x8, "LCM_MPU_VSYNC")), - MA35_PIN(44, PD0, 0x98, 0x0, + MA35_PIN(48, PD0, 0x98, 0x0, MA35_MUX(0x0, "GPD0"), MA35_MUX(0x2, "UART3_nCTS"), MA35_MUX(0x3, "UART4_RXD"), MA35_MUX(0x5, "QSPI0_SS0")), - MA35_PIN(45, PD1, 0x98, 0x4, + MA35_PIN(49, PD1, 0x98, 0x4, MA35_MUX(0x0, "GPD1"), MA35_MUX(0x2, "UART3_nRTS"), MA35_MUX(0x3, "UART4_TXD"), MA35_MUX(0x5, "QSPI0_CLK")), - MA35_PIN(46, PD2, 0x98, 0x8, + MA35_PIN(50, PD2, 0x98, 0x8, MA35_MUX(0x0, "GPD2"), MA35_MUX(0x2, "UART3_RXD"), MA35_MUX(0x5, "QSPI0_MOSI0")), - MA35_PIN(47, PD3, 0x98, 0xc, + MA35_PIN(51, PD3, 0x98, 0xc, MA35_MUX(0x0, "GPD3"), MA35_MUX(0x2, "UART3_TXD"), MA35_MUX(0x5, "QSPI0_MISO0")), - MA35_PIN(48, PD4, 0x98, 0x10, + MA35_PIN(52, PD4, 0x98, 0x10, MA35_MUX(0x0, "GPD4"), MA35_MUX(0x2, "UART1_nCTS"), MA35_MUX(0x3, "UART2_RXD"), MA35_MUX(0x4, "I2C2_SDA"), MA35_MUX(0x5, "QSPI0_MOSI1")), - MA35_PIN(49, PD5, 0x98, 0x14, + MA35_PIN(53, PD5, 0x98, 0x14, MA35_MUX(0x0, "GPD5"), MA35_MUX(0x2, "UART1_nRTS"), MA35_MUX(0x3, "UART2_TXD"), MA35_MUX(0x4, "I2C2_SCL"), MA35_MUX(0x5, "QSPI0_MISO1")), - MA35_PIN(50, PD6, 0x98, 0x18, + MA35_PIN(54, PD6, 0x98, 0x18, MA35_MUX(0x0, "GPD6"), MA35_MUX(0x1, "EPWM0_SYNC_IN"), MA35_MUX(0x2, "UART1_RXD"), + MA35_MUX(0x4, "USBHL3_DM"), MA35_MUX(0x5, "QSPI1_MOSI1"), MA35_MUX(0x6, "I2C0_SDA"), MA35_MUX(0x7, "I2S0_MCLK"), @@ -346,10 +391,11 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "EBI_AD5"), MA35_MUX(0xa, "SPI3_SS1"), MA35_MUX(0xb, "TRACE_CLK")), - MA35_PIN(51, PD7, 0x98, 0x1c, + MA35_PIN(55, PD7, 0x98, 0x1c, MA35_MUX(0x0, "GPD7"), MA35_MUX(0x1, "EPWM0_SYNC_OUT"), MA35_MUX(0x2, "UART1_TXD"), + MA35_MUX(0x4, "USBHL3_DP"), MA35_MUX(0x5, "QSPI1_MISO1"), MA35_MUX(0x6, "I2C0_SCL"), MA35_MUX(0x7, "I2S1_MCLK"), @@ -357,7 +403,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "EBI_AD6"), MA35_MUX(0xa, "SC1_nCD"), MA35_MUX(0xb, "EADC0_ST")), - MA35_PIN(52, PD8, 0x9c, 0x0, + MA35_PIN(56, PD8, 0x9c, 0x0, MA35_MUX(0x0, "GPD8"), MA35_MUX(0x1, "EPWM0_BRAKE0"), MA35_MUX(0x2, "UART16_nCTS"), @@ -368,7 +414,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "EBI_AD7"), MA35_MUX(0xa, "SC1_CLK"), MA35_MUX(0xb, "TM0")), - MA35_PIN(53, PD9, 0x9c, 0x4, + MA35_PIN(57, PD9, 0x9c, 0x4, MA35_MUX(0x0, "GPD9"), MA35_MUX(0x1, "EPWM0_BRAKE1"), MA35_MUX(0x2, "UART16_nRTS"), @@ -379,7 +425,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "EBI_AD8"), MA35_MUX(0xa, "SC1_DAT"), MA35_MUX(0xb, "TM0_EXT")), - MA35_PIN(54, PD10, 0x9c, 0x8, + MA35_PIN(58, PD10, 0x9c, 0x8, MA35_MUX(0x0, "GPD10"), MA35_MUX(0x1, "EPWM1_BRAKE0"), MA35_MUX(0x2, "UART16_RXD"), @@ -389,7 +435,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "EBI_AD9"), MA35_MUX(0xa, "SC1_RST"), MA35_MUX(0xb, "TM2")), - MA35_PIN(55, PD11, 0x9c, 0xc, + MA35_PIN(59, PD11, 0x9c, 0xc, MA35_MUX(0x0, "GPD11"), MA35_MUX(0x1, "EPWM1_BRAKE1"), MA35_MUX(0x2, "UART16_TXD"), @@ -399,7 +445,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "EBI_AD10"), MA35_MUX(0xa, "SC1_PWR"), MA35_MUX(0xb, "TM2_EXT")), - MA35_PIN(56, PD12, 0x9c, 0x10, + MA35_PIN(60, PD12, 0x9c, 0x10, MA35_MUX(0x0, "GPD12"), MA35_MUX(0x1, "EPWM0_BRAKE0"), MA35_MUX(0x2, "UART11_TXD"), @@ -412,7 +458,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xb, "TM5"), MA35_MUX(0xc, "I2S1_LRCK"), MA35_MUX(0xd, "INT1")), - MA35_PIN(57, PD13, 0x9c, 0x14, + MA35_PIN(61, PD13, 0x9c, 0x14, MA35_MUX(0x0, "GPD13"), MA35_MUX(0x1, "EPWM0_BRAKE1"), MA35_MUX(0x2, "UART11_RXD"), @@ -424,11 +470,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "ECAP0_IC0"), MA35_MUX(0xb, "TM5_EXT"), MA35_MUX(0xc, "I2S1_BCLK")), - MA35_PIN(58, PD14, 0x9c, 0x18, + MA35_PIN(62, PD14, 0x9c, 0x18, MA35_MUX(0x0, "GPD14"), MA35_MUX(0x1, "EPWM0_SYNC_IN"), MA35_MUX(0x2, "UART11_nCTS"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL5_DM"), MA35_MUX(0x6, "TRACE_DATA2"), MA35_MUX(0x7, "EBI_MCLK"), MA35_MUX(0x8, "EBI_AD6"), @@ -436,116 +483,117 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xb, "TM6"), MA35_MUX(0xc, "I2S1_DI"), MA35_MUX(0xd, "INT3")), - MA35_PIN(59, PD15, 0x9c, 0x1c, + MA35_PIN(63, PD15, 0x9c, 0x1c, MA35_MUX(0x0, "GPD15"), MA35_MUX(0x1, "EPWM0_SYNC_OUT"), MA35_MUX(0x2, "UART11_nRTS"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL5_DP"), MA35_MUX(0x6, "TRACE_DATA3"), MA35_MUX(0x7, "EBI_ALE"), MA35_MUX(0x8, "EBI_AD7"), MA35_MUX(0x9, "ECAP0_IC2"), MA35_MUX(0xb, "TM6_EXT"), MA35_MUX(0xc, "I2S1_DO")), - MA35_PIN(60, PE0, 0xa0, 0x0, + MA35_PIN(64, PE0, 0xa0, 0x0, MA35_MUX(0x0, "GPE0"), MA35_MUX(0x2, "UART9_nCTS"), MA35_MUX(0x3, "UART8_RXD"), MA35_MUX(0x7, "CCAP1_DATA0"), MA35_MUX(0x8, "RGMII0_MDC"), MA35_MUX(0x9, "RMII0_MDC")), - MA35_PIN(61, PE1, 0xa0, 0x4, + MA35_PIN(65, PE1, 0xa0, 0x4, MA35_MUX(0x0, "GPE1"), MA35_MUX(0x2, "UART9_nRTS"), MA35_MUX(0x3, "UART8_TXD"), MA35_MUX(0x7, "CCAP1_DATA1"), MA35_MUX(0x8, "RGMII0_MDIO"), MA35_MUX(0x9, "RMII0_MDIO")), - MA35_PIN(62, PE2, 0xa0, 0x8, + MA35_PIN(66, PE2, 0xa0, 0x8, MA35_MUX(0x0, "GPE2"), MA35_MUX(0x2, "UART9_RXD"), MA35_MUX(0x7, "CCAP1_DATA2"), MA35_MUX(0x8, "RGMII0_TXCTL"), MA35_MUX(0x9, "RMII0_TXEN")), - MA35_PIN(63, PE3, 0xa0, 0xc, + MA35_PIN(67, PE3, 0xa0, 0xc, MA35_MUX(0x0, "GPE3"), MA35_MUX(0x2, "UART9_TXD"), MA35_MUX(0x7, "CCAP1_DATA3"), MA35_MUX(0x8, "RGMII0_TXD0"), MA35_MUX(0x9, "RMII0_TXD0")), - MA35_PIN(64, PE4, 0xa0, 0x10, + MA35_PIN(68, PE4, 0xa0, 0x10, MA35_MUX(0x0, "GPE4"), MA35_MUX(0x2, "UART4_nCTS"), MA35_MUX(0x3, "UART3_RXD"), MA35_MUX(0x7, "CCAP1_DATA4"), MA35_MUX(0x8, "RGMII0_TXD1"), MA35_MUX(0x9, "RMII0_TXD1")), - MA35_PIN(65, PE5, 0xa0, 0x14, + MA35_PIN(69, PE5, 0xa0, 0x14, MA35_MUX(0x0, "GPE5"), MA35_MUX(0x2, "UART4_nRTS"), MA35_MUX(0x3, "UART3_TXD"), MA35_MUX(0x7, "CCAP1_DATA5"), MA35_MUX(0x8, "RGMII0_RXCLK"), MA35_MUX(0x9, "RMII0_REFCLK")), - MA35_PIN(66, PE6, 0xa0, 0x18, + MA35_PIN(70, PE6, 0xa0, 0x18, MA35_MUX(0x0, "GPE6"), MA35_MUX(0x2, "UART4_RXD"), MA35_MUX(0x7, "CCAP1_DATA6"), MA35_MUX(0x8, "RGMII0_RXCTL"), MA35_MUX(0x9, "RMII0_CRSDV")), - MA35_PIN(67, PE7, 0xa0, 0x1c, + MA35_PIN(71, PE7, 0xa0, 0x1c, MA35_MUX(0x0, "GPE7"), MA35_MUX(0x2, "UART4_TXD"), MA35_MUX(0x7, "CCAP1_DATA7"), MA35_MUX(0x8, "RGMII0_RXD0"), MA35_MUX(0x9, "RMII0_RXD0")), - MA35_PIN(68, PE8, 0xa4, 0x0, + MA35_PIN(72, PE8, 0xa4, 0x0, MA35_MUX(0x0, "GPE8"), MA35_MUX(0x2, "UART13_nCTS"), MA35_MUX(0x3, "UART12_RXD"), MA35_MUX(0x7, "CCAP1_SCLK"), MA35_MUX(0x8, "RGMII0_RXD1"), MA35_MUX(0x9, "RMII0_RXD1")), - MA35_PIN(69, PE9, 0xa4, 0x4, + MA35_PIN(73, PE9, 0xa4, 0x4, MA35_MUX(0x0, "GPE9"), MA35_MUX(0x2, "UART13_nRTS"), MA35_MUX(0x3, "UART12_TXD"), MA35_MUX(0x7, "CCAP1_PIXCLK"), MA35_MUX(0x8, "RGMII0_RXD2"), MA35_MUX(0x9, "RMII0_RXERR")), - MA35_PIN(70, PE10, 0xa4, 0x8, + MA35_PIN(74, PE10, 0xa4, 0x8, MA35_MUX(0x0, "GPE10"), MA35_MUX(0x2, "UART15_nCTS"), MA35_MUX(0x3, "UART14_RXD"), MA35_MUX(0x5, "SPI1_SS0"), MA35_MUX(0x7, "CCAP1_HSYNC"), MA35_MUX(0x8, "RGMII0_RXD3")), - MA35_PIN(71, PE11, 0xa4, 0xc, + MA35_PIN(75, PE11, 0xa4, 0xc, MA35_MUX(0x0, "GPE11"), MA35_MUX(0x2, "UART15_nRTS"), MA35_MUX(0x3, "UART14_TXD"), MA35_MUX(0x5, "SPI1_CLK"), MA35_MUX(0x7, "CCAP1_VSYNC"), MA35_MUX(0x8, "RGMII0_TXCLK")), - MA35_PIN(72, PE12, 0xa4, 0x10, + MA35_PIN(76, PE12, 0xa4, 0x10, MA35_MUX(0x0, "GPE12"), MA35_MUX(0x2, "UART15_RXD"), MA35_MUX(0x5, "SPI1_MOSI"), MA35_MUX(0x7, "CCAP1_DATA8"), MA35_MUX(0x8, "RGMII0_TXD2")), - MA35_PIN(73, PE13, 0xa4, 0x14, + MA35_PIN(77, PE13, 0xa4, 0x14, MA35_MUX(0x0, "GPE13"), MA35_MUX(0x2, "UART15_TXD"), MA35_MUX(0x5, "SPI1_MISO"), MA35_MUX(0x7, "CCAP1_DATA9"), MA35_MUX(0x8, "RGMII0_TXD3")), - MA35_PIN(74, PE14, 0xa4, 0x18, + MA35_PIN(78, PE14, 0xa4, 0x18, MA35_MUX(0x0, "GPE14"), MA35_MUX(0x1, "UART0_TXD")), - MA35_PIN(75, PE15, 0xa4, 0x1c, + MA35_PIN(79, PE15, 0xa4, 0x1c, MA35_MUX(0x0, "GPE15"), MA35_MUX(0x1, "UART0_RXD")), - MA35_PIN(76, PF0, 0xa8, 0x0, + MA35_PIN(80, PF0, 0xa8, 0x0, MA35_MUX(0x0, "GPF0"), MA35_MUX(0x2, "UART2_nCTS"), MA35_MUX(0x3, "UART1_RXD"), @@ -553,7 +601,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "RGMII1_MDC"), MA35_MUX(0x9, "RMII1_MDC"), MA35_MUX(0xe, "KPI_COL0")), - MA35_PIN(77, PF1, 0xa8, 0x4, + MA35_PIN(81, PF1, 0xa8, 0x4, MA35_MUX(0x0, "GPF1"), MA35_MUX(0x2, "UART2_nRTS"), MA35_MUX(0x3, "UART1_TXD"), @@ -561,21 +609,21 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "RGMII1_MDIO"), MA35_MUX(0x9, "RMII1_MDIO"), MA35_MUX(0xe, "KPI_COL1")), - MA35_PIN(78, PF2, 0xa8, 0x8, + MA35_PIN(82, PF2, 0xa8, 0x8, MA35_MUX(0x0, "GPF2"), MA35_MUX(0x2, "UART2_RXD"), MA35_MUX(0x6, "RGMII0_TXD2"), MA35_MUX(0x8, "RGMII1_TXCTL"), MA35_MUX(0x9, "RMII1_TXEN"), MA35_MUX(0xe, "KPI_COL2")), - MA35_PIN(79, PF3, 0xa8, 0xc, + MA35_PIN(83, PF3, 0xa8, 0xc, MA35_MUX(0x0, "GPF3"), MA35_MUX(0x2, "UART2_TXD"), MA35_MUX(0x6, "RGMII0_TXD3"), MA35_MUX(0x8, "RGMII1_TXD0"), MA35_MUX(0x9, "RMII1_TXD0"), MA35_MUX(0xe, "KPI_COL3")), - MA35_PIN(80, PF4, 0xa8, 0x10, + MA35_PIN(84, PF4, 0xa8, 0x10, MA35_MUX(0x0, "GPF4"), MA35_MUX(0x2, "UART11_nCTS"), MA35_MUX(0x3, "UART10_RXD"), @@ -583,9 +631,10 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x5, "SPI1_SS0"), MA35_MUX(0x8, "RGMII1_TXD1"), MA35_MUX(0x9, "RMII1_TXD1"), + MA35_MUX(0xc, "USBHL0_DM"), MA35_MUX(0xd, "CAN2_RXD"), MA35_MUX(0xe, "KPI_ROW0")), - MA35_PIN(81, PF5, 0xa8, 0x14, + MA35_PIN(85, PF5, 0xa8, 0x14, MA35_MUX(0x0, "GPF5"), MA35_MUX(0x2, "UART11_nRTS"), MA35_MUX(0x3, "UART10_TXD"), @@ -593,9 +642,10 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x5, "SPI1_CLK"), MA35_MUX(0x8, "RGMII1_RXCLK"), MA35_MUX(0x9, "RMII1_REFCLK"), + MA35_MUX(0xc, "USBHL0_DP"), MA35_MUX(0xd, "CAN2_TXD"), MA35_MUX(0xe, "KPI_ROW1")), - MA35_PIN(82, PF6, 0xa8, 0x18, + MA35_PIN(86, PF6, 0xa8, 0x18, MA35_MUX(0x0, "GPF6"), MA35_MUX(0x2, "UART11_RXD"), MA35_MUX(0x4, "I2S0_DI"), @@ -605,7 +655,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "I2C4_SDA"), MA35_MUX(0xd, "SC0_CLK"), MA35_MUX(0xe, "KPI_ROW2")), - MA35_PIN(83, PF7, 0xa8, 0x1c, + MA35_PIN(87, PF7, 0xa8, 0x1c, MA35_MUX(0x0, "GPF7"), MA35_MUX(0x2, "UART11_TXD"), MA35_MUX(0x4, "I2S0_DO"), @@ -615,7 +665,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "I2C4_SCL"), MA35_MUX(0xd, "SC0_DAT"), MA35_MUX(0xe, "KPI_ROW3")), - MA35_PIN(84, PF8, 0xac, 0x0, + MA35_PIN(88, PF8, 0xac, 0x0, MA35_MUX(0x0, "GPF8"), MA35_MUX(0x2, "UART13_RXD"), MA35_MUX(0x4, "I2C5_SDA"), @@ -624,7 +674,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "RMII1_RXD1"), MA35_MUX(0xd, "SC0_RST"), MA35_MUX(0xe, "KPI_COL4")), - MA35_PIN(85, PF9, 0xac, 0x4, + MA35_PIN(89, PF9, 0xac, 0x4, MA35_MUX(0x0, "GPF9"), MA35_MUX(0x2, "UART13_TXD"), MA35_MUX(0x4, "I2C5_SCL"), @@ -633,7 +683,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "RMII1_RXERR"), MA35_MUX(0xd, "SC0_PWR"), MA35_MUX(0xe, "KPI_COL5")), - MA35_PIN(86, PF10, 0xac, 0x8, + MA35_PIN(90, PF10, 0xac, 0x8, MA35_MUX(0x0, "GPF10"), MA35_MUX(0x2, "UART13_nCTS"), MA35_MUX(0x5, "I2S0_LRCK"), @@ -641,7 +691,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "RGMII1_RXD3"), MA35_MUX(0x9, "SC0_CLK"), MA35_MUX(0xe, "KPI_COL6")), - MA35_PIN(87, PF11, 0xac, 0xc, + MA35_PIN(91, PF11, 0xac, 0xc, MA35_MUX(0x0, "GPF11"), MA35_MUX(0x2, "UART13_nRTS"), MA35_MUX(0x5, "I2S0_BCLK"), @@ -649,21 +699,21 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "RGMII1_TXCLK"), MA35_MUX(0x9, "SC0_DAT"), MA35_MUX(0xe, "KPI_COL7")), - MA35_PIN(88, PF12, 0xac, 0x10, + MA35_PIN(92, PF12, 0xac, 0x10, MA35_MUX(0x0, "GPF12"), MA35_MUX(0x5, "I2S0_DI"), MA35_MUX(0x6, "SPI1_MOSI"), MA35_MUX(0x8, "RGMII1_TXD2"), MA35_MUX(0x9, "SC0_RST"), MA35_MUX(0xe, "KPI_ROW4")), - MA35_PIN(89, PF13, 0xac, 0x14, + MA35_PIN(93, PF13, 0xac, 0x14, MA35_MUX(0x0, "GPF13"), MA35_MUX(0x5, "I2S0_DO"), MA35_MUX(0x6, "SPI1_MISO"), MA35_MUX(0x8, "RGMII1_TXD3"), MA35_MUX(0x9, "SC0_PWR"), MA35_MUX(0xe, "KPI_ROW5")), - MA35_PIN(90, PF14, 0xac, 0x18, + MA35_PIN(94, PF14, 0xac, 0x18, MA35_MUX(0x0, "GPF14"), MA35_MUX(0x1, "EPWM2_BRAKE0"), MA35_MUX(0x2, "EADC0_ST"), @@ -679,10 +729,10 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xd, "SPI1_SS1"), MA35_MUX(0xe, "QEI2_INDEX"), MA35_MUX(0xf, "I2S0_MCLK")), - MA35_PIN(91, PF15, 0xac, 0x1c, + MA35_PIN(95, PF15, 0xac, 0x1c, MA35_MUX(0x0, "GPF15"), MA35_MUX(0x1, "HSUSB0_VBUSVLD")), - MA35_PIN(92, PG0, 0xb0, 0x0, + MA35_PIN(96, PG0, 0xb0, 0x0, MA35_MUX(0x0, "GPG0"), MA35_MUX(0x1, "EPWM0_CH0"), MA35_MUX(0x2, "UART7_TXD"), @@ -696,19 +746,20 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xc, "CLKO"), MA35_MUX(0xd, "INT0"), MA35_MUX(0xf, "EBI_ADR15")), - MA35_PIN(93, PG1, 0xb0, 0x4, + MA35_PIN(97, PG1, 0xb0, 0x4, MA35_MUX(0x0, "GPG1"), MA35_MUX(0x1, "EPWM0_CH3"), MA35_MUX(0x2, "UART9_nRTS"), MA35_MUX(0x3, "UART6_TXD"), MA35_MUX(0x4, "I2C4_SCL"), MA35_MUX(0x5, "CAN2_TXD"), + MA35_MUX(0x6, "USBHL0_DP"), MA35_MUX(0x7, "EBI_nCS0"), MA35_MUX(0x9, "QEI0_B"), MA35_MUX(0xb, "TM1_EXT"), MA35_MUX(0xe, "RGMII1_PPS"), MA35_MUX(0xf, "RMII1_PPS")), - MA35_PIN(94, PG2, 0xb0, 0x8, + MA35_PIN(98, PG2, 0xb0, 0x8, MA35_MUX(0x0, "GPG2"), MA35_MUX(0x1, "EPWM0_CH4"), MA35_MUX(0x2, "UART9_RXD"), @@ -719,7 +770,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "QEI0_A"), MA35_MUX(0xb, "TM3"), MA35_MUX(0xd, "INT1")), - MA35_PIN(95, PG3, 0xb0, 0xc, + MA35_PIN(99, PG3, 0xb0, 0xc, MA35_MUX(0x0, "GPG3"), MA35_MUX(0x1, "EPWM0_CH5"), MA35_MUX(0x2, "UART9_TXD"), @@ -731,7 +782,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "QEI0_B"), MA35_MUX(0xb, "TM3_EXT"), MA35_MUX(0xc, "I2S1_MCLK")), - MA35_PIN(96, PG4, 0xb0, 0x10, + MA35_PIN(100, PG4, 0xb0, 0x10, MA35_MUX(0x0, "GPG4"), MA35_MUX(0x1, "EPWM1_CH0"), MA35_MUX(0x2, "UART5_nCTS"), @@ -745,7 +796,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xb, "TM4"), MA35_MUX(0xd, "INT2"), MA35_MUX(0xe, "ECAP1_IC2")), - MA35_PIN(97, PG5, 0xb0, 0x14, + MA35_PIN(101, PG5, 0xb0, 0x14, MA35_MUX(0x0, "GPG5"), MA35_MUX(0x1, "EPWM1_CH1"), MA35_MUX(0x2, "UART5_nRTS"), @@ -757,7 +808,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "I2S1_DI"), MA35_MUX(0xa, "SC1_DAT"), MA35_MUX(0xb, "TM4_EXT")), - MA35_PIN(98, PG6, 0xb0, 0x18, + MA35_PIN(102, PG6, 0xb0, 0x18, MA35_MUX(0x0, "GPG6"), MA35_MUX(0x1, "EPWM1_CH2"), MA35_MUX(0x2, "UART5_RXD"), @@ -769,7 +820,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "SC1_RST"), MA35_MUX(0xb, "TM7"), MA35_MUX(0xd, "INT3")), - MA35_PIN(99, PG7, 0xb0, 0x1c, + MA35_PIN(103, PG7, 0xb0, 0x1c, MA35_MUX(0x0, "GPG7"), MA35_MUX(0x1, "EPWM1_CH3"), MA35_MUX(0x2, "UART5_TXD"), @@ -780,27 +831,29 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "I2S1_LRCK"), MA35_MUX(0xa, "SC1_PWR"), MA35_MUX(0xb, "TM7_EXT")), - MA35_PIN(100, PG8, 0xb4, 0x0, + MA35_PIN(104, PG8, 0xb4, 0x0, MA35_MUX(0x0, "GPG8"), MA35_MUX(0x1, "EPWM1_CH4"), MA35_MUX(0x2, "UART12_RXD"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL4_DM"), MA35_MUX(0x5, "SPI2_SS0"), MA35_MUX(0x6, "LCM_VSYNC"), MA35_MUX(0x7, "I2C3_SDA"), MA35_MUX(0xc, "EBI_AD7"), MA35_MUX(0xd, "EBI_nCS0")), - MA35_PIN(101, PG9, 0xb4, 0x4, + MA35_PIN(105, PG9, 0xb4, 0x4, MA35_MUX(0x0, "GPG9"), MA35_MUX(0x1, "EPWM1_CH5"), MA35_MUX(0x2, "UART12_TXD"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL4_DP"), MA35_MUX(0x5, "SPI2_CLK"), MA35_MUX(0x6, "LCM_HSYNC"), MA35_MUX(0x7, "I2C3_SCL"), MA35_MUX(0xc, "EBI_AD8"), MA35_MUX(0xd, "EBI_nCS1")), - MA35_PIN(102, PG10, 0xb4, 0x8, + MA35_PIN(106, PG10, 0xb4, 0x8, MA35_MUX(0x0, "GPG10"), MA35_MUX(0x2, "UART12_nRTS"), MA35_MUX(0x3, "UART13_TXD"), @@ -808,7 +861,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x6, "LCM_CLK"), MA35_MUX(0xc, "EBI_AD9"), MA35_MUX(0xd, "EBI_nWRH")), - MA35_PIN(103, PG11, 0xb4, 0xc, + MA35_PIN(107, PG11, 0xb4, 0xc, MA35_MUX(0x0, "GPG11"), MA35_MUX(0x3, "JTAG_TDO"), MA35_MUX(0x5, "I2S0_MCLK"), @@ -816,93 +869,93 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x7, "EBI_nWRH"), MA35_MUX(0x8, "EBI_nCS1"), MA35_MUX(0xa, "EBI_AD0")), - MA35_PIN(104, PG12, 0xb4, 0x10, + MA35_PIN(108, PG12, 0xb4, 0x10, MA35_MUX(0x0, "GPG12"), MA35_MUX(0x3, "JTAG_TCK/SW_CLK"), MA35_MUX(0x5, "I2S0_LRCK"), MA35_MUX(0x7, "EBI_nWRL"), MA35_MUX(0xa, "EBI_AD1")), - MA35_PIN(105, PG13, 0xb4, 0x14, + MA35_PIN(109, PG13, 0xb4, 0x14, MA35_MUX(0x0, "GPG13"), MA35_MUX(0x3, "JTAG_TMS/SW_DIO"), MA35_MUX(0x5, "I2S0_BCLK"), MA35_MUX(0x7, "EBI_MCLK"), MA35_MUX(0xa, "EBI_AD2")), - MA35_PIN(106, PG14, 0xb4, 0x18, + MA35_PIN(110, PG14, 0xb4, 0x18, MA35_MUX(0x0, "GPG14"), MA35_MUX(0x3, "JTAG_TDI"), MA35_MUX(0x5, "I2S0_DI"), MA35_MUX(0x6, "NAND_nCS1"), MA35_MUX(0x7, "EBI_ALE"), MA35_MUX(0xa, "EBI_AD3")), - MA35_PIN(107, PG15, 0xb4, 0x1c, + MA35_PIN(111, PG15, 0xb4, 0x1c, MA35_MUX(0x0, "GPG15"), MA35_MUX(0x3, "JTAG_nTRST"), MA35_MUX(0x5, "I2S0_DO"), MA35_MUX(0x7, "EBI_nCS0"), MA35_MUX(0xa, "EBI_AD4")), - MA35_PIN(108, PH0, 0xb8, 0x0, + MA35_PIN(112, PH0, 0xb8, 0x0, MA35_MUX(0x0, "GPH0"), MA35_MUX(0x2, "UART8_nCTS"), MA35_MUX(0x3, "UART7_RXD"), MA35_MUX(0x6, "LCM_DATA8")), - MA35_PIN(109, PH1, 0xb8, 0x4, + MA35_PIN(113, PH1, 0xb8, 0x4, MA35_MUX(0x0, "GPH1"), MA35_MUX(0x2, "UART8_nRTS"), MA35_MUX(0x3, "UART7_TXD"), MA35_MUX(0x6, "LCM_DATA9")), - MA35_PIN(110, PH2, 0xb8, 0x8, + MA35_PIN(114, PH2, 0xb8, 0x8, MA35_MUX(0x0, "GPH2"), MA35_MUX(0x2, "UART8_RXD"), MA35_MUX(0x6, "LCM_DATA10")), - MA35_PIN(111, PH3, 0xb8, 0xc, + MA35_PIN(115, PH3, 0xb8, 0xc, MA35_MUX(0x0, "GPH3"), MA35_MUX(0x2, "UART8_TXD"), MA35_MUX(0x6, "LCM_DATA11")), - MA35_PIN(112, PH4, 0xb8, 0x10, + MA35_PIN(116, PH4, 0xb8, 0x10, MA35_MUX(0x0, "GPH4"), MA35_MUX(0x2, "UART10_nCTS"), MA35_MUX(0x3, "UART9_RXD"), MA35_MUX(0x6, "LCM_DATA12")), - MA35_PIN(113, PH5, 0xb8, 0x14, + MA35_PIN(117, PH5, 0xb8, 0x14, MA35_MUX(0x0, "GPH5"), MA35_MUX(0x2, "UART10_nRTS"), MA35_MUX(0x3, "UART9_TXD"), MA35_MUX(0x6, "LCM_DATA13")), - MA35_PIN(114, PH6, 0xb8, 0x18, + MA35_PIN(118, PH6, 0xb8, 0x18, MA35_MUX(0x0, "GPH6"), MA35_MUX(0x2, "UART10_RXD"), MA35_MUX(0x6, "LCM_DATA14")), - MA35_PIN(115, PH7, 0xb8, 0x1c, + MA35_PIN(119, PH7, 0xb8, 0x1c, MA35_MUX(0x0, "GPH7"), MA35_MUX(0x2, "UART10_TXD"), MA35_MUX(0x6, "LCM_DATA15")), - MA35_PIN(116, PH8, 0xbc, 0x0, + MA35_PIN(120, PH8, 0xbc, 0x0, MA35_MUX(0x0, "GPH8"), MA35_MUX(0x6, "TAMPER0")), - MA35_PIN(117, PH9, 0xbc, 0x4, + MA35_PIN(121, PH9, 0xbc, 0x4, MA35_MUX(0x0, "GPH9"), MA35_MUX(0x4, "CLK_32KOUT"), MA35_MUX(0x6, "TAMPER1")), - MA35_PIN(118, PH12, 0xbc, 0x10, + MA35_PIN(124, PH12, 0xbc, 0x10, MA35_MUX(0x0, "GPH12"), MA35_MUX(0x2, "UART14_nCTS"), MA35_MUX(0x3, "UART13_RXD"), MA35_MUX(0x6, "LCM_DATA20")), - MA35_PIN(119, PH13, 0xbc, 0x14, + MA35_PIN(125, PH13, 0xbc, 0x14, MA35_MUX(0x0, "GPH13"), MA35_MUX(0x2, "UART14_nRTS"), MA35_MUX(0x3, "UART13_TXD"), MA35_MUX(0x6, "LCM_DATA21")), - MA35_PIN(120, PH14, 0xbc, 0x18, + MA35_PIN(126, PH14, 0xbc, 0x18, MA35_MUX(0x0, "GPH14"), MA35_MUX(0x2, "UART14_RXD"), MA35_MUX(0x6, "LCM_DATA22")), - MA35_PIN(121, PH15, 0xbc, 0x1c, + MA35_PIN(127, PH15, 0xbc, 0x1c, MA35_MUX(0x0, "GPH15"), MA35_MUX(0x2, "UART14_TXD"), MA35_MUX(0x6, "LCM_DATA23")), - MA35_PIN(122, PI0, 0xc0, 0x0, + MA35_PIN(128, PI0, 0xc0, 0x0, MA35_MUX(0x0, "GPI0"), MA35_MUX(0x1, "EPWM0_CH0"), MA35_MUX(0x2, "UART12_nCTS"), @@ -913,7 +966,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "EBI_ADR0"), MA35_MUX(0xb, "TM0"), MA35_MUX(0xc, "ECAP1_IC0")), - MA35_PIN(123, PI1, 0xc0, 0x4, + MA35_PIN(129, PI1, 0xc0, 0x4, MA35_MUX(0x0, "GPI1"), MA35_MUX(0x1, "EPWM0_CH1"), MA35_MUX(0x2, "UART12_nRTS"), @@ -924,26 +977,28 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "EBI_ADR1"), MA35_MUX(0xb, "TM0_EXT"), MA35_MUX(0xc, "ECAP1_IC1")), - MA35_PIN(124, PI2, 0xc0, 0x8, + MA35_PIN(130, PI2, 0xc0, 0x8, MA35_MUX(0x0, "GPI2"), MA35_MUX(0x1, "EPWM0_CH2"), MA35_MUX(0x2, "UART12_RXD"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL2_DM"), MA35_MUX(0x5, "SPI3_MOSI"), MA35_MUX(0x7, "SC0_DAT"), MA35_MUX(0x8, "EBI_ADR2"), MA35_MUX(0xb, "TM1"), MA35_MUX(0xc, "ECAP1_IC2")), - MA35_PIN(125, PI3, 0xc0, 0xc, + MA35_PIN(131, PI3, 0xc0, 0xc, MA35_MUX(0x0, "GPI3"), MA35_MUX(0x1, "EPWM0_CH3"), MA35_MUX(0x2, "UART12_TXD"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL2_DP"), MA35_MUX(0x5, "SPI3_MISO"), MA35_MUX(0x7, "SC0_RST"), MA35_MUX(0x8, "EBI_ADR3"), MA35_MUX(0xb, "TM1_EXT")), - MA35_PIN(126, PI4, 0xc0, 0x10, + MA35_PIN(132, PI4, 0xc0, 0x10, MA35_MUX(0x0, "GPI4"), MA35_MUX(0x1, "EPWM0_CH4"), MA35_MUX(0x2, "UART14_nCTS"), @@ -953,7 +1008,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x6, "I2S1_LRCK"), MA35_MUX(0x8, "EBI_ADR4"), MA35_MUX(0xd, "INT0")), - MA35_PIN(127, PI5, 0xc0, 0x14, + MA35_PIN(133, PI5, 0xc0, 0x14, MA35_MUX(0x0, "GPI5"), MA35_MUX(0x1, "EPWM0_CH5"), MA35_MUX(0x2, "UART14_nRTS"), @@ -962,65 +1017,67 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x6, "I2S1_BCLK"), MA35_MUX(0x8, "EBI_ADR5"), MA35_MUX(0xd, "INT1")), - MA35_PIN(128, PI6, 0xc0, 0x18, + MA35_PIN(134, PI6, 0xc0, 0x18, MA35_MUX(0x0, "GPI6"), MA35_MUX(0x1, "EPWM0_BRAKE0"), MA35_MUX(0x2, "UART14_RXD"), MA35_MUX(0x3, "CAN1_RXD"), + MA35_MUX(0x4, "USBHL3_DM"), MA35_MUX(0x6, "I2S1_DI"), MA35_MUX(0x8, "EBI_ADR6"), MA35_MUX(0xc, "QEI1_INDEX"), MA35_MUX(0xd, "INT2")), - MA35_PIN(129, PI7, 0xc0, 0x1c, + MA35_PIN(135, PI7, 0xc0, 0x1c, MA35_MUX(0x0, "GPI7"), MA35_MUX(0x1, "EPWM0_BRAKE1"), MA35_MUX(0x2, "UART14_TXD"), MA35_MUX(0x3, "CAN1_TXD"), + MA35_MUX(0x4, "USBHL3_DP"), MA35_MUX(0x6, "I2S1_DO"), MA35_MUX(0x8, "EBI_ADR7"), MA35_MUX(0xc, "ECAP0_IC0"), MA35_MUX(0xd, "INT3")), - MA35_PIN(130, PI8, 0xc4, 0x0, + MA35_PIN(136, PI8, 0xc4, 0x0, MA35_MUX(0x0, "GPI8"), MA35_MUX(0x2, "UART4_nCTS"), MA35_MUX(0x3, "UART3_RXD"), MA35_MUX(0x6, "LCM_DATA0"), MA35_MUX(0xc, "EBI_AD11")), - MA35_PIN(131, PI9, 0xc4, 0x4, + MA35_PIN(137, PI9, 0xc4, 0x4, MA35_MUX(0x0, "GPI9"), MA35_MUX(0x2, "UART4_nRTS"), MA35_MUX(0x3, "UART3_TXD"), MA35_MUX(0x6, "LCM_DATA1"), MA35_MUX(0xc, "EBI_AD12")), - MA35_PIN(132, PI10, 0xc4, 0x8, + MA35_PIN(138, PI10, 0xc4, 0x8, MA35_MUX(0x0, "GPI10"), MA35_MUX(0x2, "UART4_RXD"), MA35_MUX(0x6, "LCM_DATA2"), MA35_MUX(0xc, "EBI_AD13")), - MA35_PIN(133, PI11, 0xC4, 0xc, + MA35_PIN(139, PI11, 0xC4, 0xc, MA35_MUX(0x0, "GPI11"), MA35_MUX(0x2, "UART4_TXD"), MA35_MUX(0x6, "LCM_DATA3"), MA35_MUX(0xc, "EBI_AD14")), - MA35_PIN(134, PI12, 0xc4, 0x10, + MA35_PIN(140, PI12, 0xc4, 0x10, MA35_MUX(0x0, "GPI12"), MA35_MUX(0x2, "UART6_nCTS"), MA35_MUX(0x3, "UART5_RXD"), MA35_MUX(0x6, "LCM_DATA4")), - MA35_PIN(135, PI13, 0xc4, 0x14, + MA35_PIN(141, PI13, 0xc4, 0x14, MA35_MUX(0x0, "GPI13"), MA35_MUX(0x2, "UART6_nRTS"), MA35_MUX(0x3, "UART5_TXD"), MA35_MUX(0x6, "LCM_DATA5")), - MA35_PIN(136, PI14, 0xc4, 0x18, + MA35_PIN(142, PI14, 0xc4, 0x18, MA35_MUX(0x0, "GPI14"), MA35_MUX(0x2, "UART6_RXD"), MA35_MUX(0x6, "LCM_DATA6")), - MA35_PIN(137, PI15, 0xc4, 0x1c, + MA35_PIN(143, PI15, 0xc4, 0x1c, MA35_MUX(0x0, "GPI15"), MA35_MUX(0x2, "UART6_TXD"), MA35_MUX(0x6, "LCM_DATA7")), - MA35_PIN(138, PJ0, 0xc8, 0x0, + MA35_PIN(144, PJ0, 0xc8, 0x0, MA35_MUX(0x0, "GPJ0"), MA35_MUX(0x1, "EPWM1_BRAKE0"), MA35_MUX(0x2, "UART8_nCTS"), @@ -1034,7 +1091,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "EBI_ADR16"), MA35_MUX(0xb, "EBI_nCS0"), MA35_MUX(0xc, "EBI_AD7")), - MA35_PIN(139, PJ1, 0xc8, 0x4, + MA35_PIN(145, PJ1, 0xc8, 0x4, MA35_MUX(0x0, "GPJ1"), MA35_MUX(0x1, "EPWM1_BRAKE1"), MA35_MUX(0x2, "UART8_nRTS"), @@ -1048,11 +1105,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "EBI_ADR17"), MA35_MUX(0xb, "EBI_nCS1"), MA35_MUX(0xc, "EBI_AD8")), - MA35_PIN(140, PJ2, 0xc8, 0x8, + MA35_PIN(146, PJ2, 0xc8, 0x8, MA35_MUX(0x0, "GPJ2"), MA35_MUX(0x1, "EPWM1_CH4"), MA35_MUX(0x2, "UART8_RXD"), MA35_MUX(0x3, "CAN1_RXD"), + MA35_MUX(0x4, "USBHL5_DM"), MA35_MUX(0x5, "SPI2_MOSI"), MA35_MUX(0x6, "eMMC1_DAT6"), MA35_MUX(0x7, "I2S0_DI"), @@ -1061,11 +1119,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "EBI_ADR18"), MA35_MUX(0xb, "EBI_nWRH"), MA35_MUX(0xc, "EBI_AD9")), - MA35_PIN(141, PJ3, 0xc8, 0xc, + MA35_PIN(147, PJ3, 0xc8, 0xc, MA35_MUX(0x0, "GPJ3"), MA35_MUX(0x1, "EPWM1_CH5"), MA35_MUX(0x2, "UART8_TXD"), MA35_MUX(0x3, "CAN1_TXD"), + MA35_MUX(0x4, "USBHL5_DP"), MA35_MUX(0x5, "SPI2_MISO"), MA35_MUX(0x6, "eMMC1_DAT7"), MA35_MUX(0x7, "I2S0_DO"), @@ -1074,39 +1133,43 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "EBI_ADR19"), MA35_MUX(0xb, "EBI_nWRL"), MA35_MUX(0xc, "EBI_AD10")), - MA35_PIN(142, PJ4, 0xc8, 0x10, + MA35_PIN(148, PJ4, 0xc8, 0x10, MA35_MUX(0x0, "GPJ4"), MA35_MUX(0x4, "I2C3_SDA"), MA35_MUX(0x6, "SD1_WP")), - MA35_PIN(143, PJ5, 0xc8, 0x14, + MA35_PIN(149, PJ5, 0xc8, 0x14, MA35_MUX(0x0, "GPJ5"), MA35_MUX(0x4, "I2C3_SCL"), MA35_MUX(0x6, "SD1_nCD")), - MA35_PIN(144, PJ6, 0xc8, 0x18, + MA35_PIN(150, PJ6, 0xc8, 0x18, MA35_MUX(0x0, "GPJ6"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL0_DM"), MA35_MUX(0x6, "SD1_CMD/eMMC1_CMD")), - MA35_PIN(145, PJ7, 0xc8, 0x1c, + MA35_PIN(151, PJ7, 0xc8, 0x1c, MA35_MUX(0x0, "GPJ7"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL0_DP"), MA35_MUX(0x6, "SD1_CLK/eMMC1_CLK")), - MA35_PIN(146, PJ8, 0xcc, 0x0, + MA35_PIN(152, PJ8, 0xcc, 0x0, MA35_MUX(0x0, "GPJ8"), MA35_MUX(0x4, "I2C4_SDA"), MA35_MUX(0x6, "SD1_DAT0/eMMC1_DAT0")), - MA35_PIN(147, PJ9, 0xcc, 0x4, + MA35_PIN(153, PJ9, 0xcc, 0x4, MA35_MUX(0x0, "GPJ9"), MA35_MUX(0x4, "I2C4_SCL"), MA35_MUX(0x6, "SD1_DAT1/eMMC1_DAT1")), - MA35_PIN(148, PJ10, 0xcc, 0x8, + MA35_PIN(154, PJ10, 0xcc, 0x8, MA35_MUX(0x0, "GPJ10"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL1_DM"), MA35_MUX(0x6, "SD1_DAT2/eMMC1_DAT2")), - MA35_PIN(149, PJ11, 0xcc, 0xc, + MA35_PIN(155, PJ11, 0xcc, 0xc, MA35_MUX(0x0, "GPJ11"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL1_DP"), MA35_MUX(0x6, "SD1_DAT3/eMMC1_DAT3")), - MA35_PIN(150, PJ12, 0xcc, 0x10, + MA35_PIN(156, PJ12, 0xcc, 0x10, MA35_MUX(0x0, "GPJ12"), MA35_MUX(0x1, "EPWM1_CH2"), MA35_MUX(0x2, "UART2_nCTS"), @@ -1117,7 +1180,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "EBI_ADR12"), MA35_MUX(0xb, "TM2"), MA35_MUX(0xc, "QEI0_INDEX")), - MA35_PIN(151, PJ13, 0xcc, 0x14, + MA35_PIN(157, PJ13, 0xcc, 0x14, MA35_MUX(0x0, "GPJ13"), MA35_MUX(0x1, "EPWM1_CH3"), MA35_MUX(0x2, "UART2_nRTS"), @@ -1127,27 +1190,29 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x7, "SC1_DAT"), MA35_MUX(0x8, "EBI_ADR13"), MA35_MUX(0xb, "TM2_EXT")), - MA35_PIN(152, PJ14, 0xcc, 0x18, + MA35_PIN(158, PJ14, 0xcc, 0x18, MA35_MUX(0x0, "GPJ14"), MA35_MUX(0x1, "EPWM1_CH4"), MA35_MUX(0x2, "UART2_RXD"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL5_DM"), MA35_MUX(0x5, "SPI3_MISO"), MA35_MUX(0x7, "SC1_RST"), MA35_MUX(0x8, "EBI_ADR14"), MA35_MUX(0xb, "TM3")), - MA35_PIN(153, PJ15, 0xcc, 0x1c, + MA35_PIN(159, PJ15, 0xcc, 0x1c, MA35_MUX(0x0, "GPJ15"), MA35_MUX(0x1, "EPWM1_CH5"), MA35_MUX(0x2, "UART2_TXD"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL5_DP"), MA35_MUX(0x5, "SPI3_CLK"), MA35_MUX(0x6, "EADC0_ST"), MA35_MUX(0x7, "SC1_PWR"), MA35_MUX(0x8, "EBI_ADR15"), MA35_MUX(0xb, "TM3_EXT"), MA35_MUX(0xd, "INT1")), - MA35_PIN(154, PK0, 0xd0, 0x0, + MA35_PIN(160, PK0, 0xd0, 0x0, MA35_MUX(0x0, "GPK0"), MA35_MUX(0x1, "EPWM0_SYNC_IN"), MA35_MUX(0x2, "UART16_nCTS"), @@ -1157,7 +1222,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "EBI_ADR8"), MA35_MUX(0xb, "TM7"), MA35_MUX(0xc, "ECAP0_IC1")), - MA35_PIN(155, PK1, 0xd0, 0x4, + MA35_PIN(161, PK1, 0xd0, 0x4, MA35_MUX(0x0, "GPK1"), MA35_MUX(0x1, "EPWM0_SYNC_OUT"), MA35_MUX(0x2, "UART16_nRTS"), @@ -1167,25 +1232,27 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "EBI_ADR9"), MA35_MUX(0xb, "TM7_EXT"), MA35_MUX(0xc, "ECAP0_IC2")), - MA35_PIN(156, PK2, 0xd0, 0x8, + MA35_PIN(162, PK2, 0xd0, 0x8, MA35_MUX(0x0, "GPK2"), MA35_MUX(0x1, "EPWM1_CH0"), MA35_MUX(0x2, "UART16_RXD"), MA35_MUX(0x3, "CAN2_RXD"), + MA35_MUX(0x4, "USBHL4_DM"), MA35_MUX(0x5, "SPI3_I2SMCLK"), MA35_MUX(0x7, "SC0_PWR"), MA35_MUX(0x8, "EBI_ADR10"), MA35_MUX(0xc, "QEI0_A")), - MA35_PIN(157, PK3, 0xd0, 0xc, + MA35_PIN(163, PK3, 0xd0, 0xc, MA35_MUX(0x0, "GPK3"), MA35_MUX(0x1, "EPWM1_CH1"), MA35_MUX(0x2, "UART16_TXD"), MA35_MUX(0x3, "CAN2_TXD"), + MA35_MUX(0x4, "USBHL4_DP"), MA35_MUX(0x5, "SPI3_SS1"), MA35_MUX(0x7, "SC1_nCD"), MA35_MUX(0x8, "EBI_ADR11"), MA35_MUX(0xc, "QEI0_B")), - MA35_PIN(158, PK4, 0xd0, 0x10, + MA35_PIN(164, PK4, 0xd0, 0x10, MA35_MUX(0x0, "GPK4"), MA35_MUX(0x2, "UART12_nCTS"), MA35_MUX(0x3, "UART13_RXD"), @@ -1193,7 +1260,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x6, "LCM_DEN"), MA35_MUX(0xc, "EBI_AD10"), MA35_MUX(0xd, "EBI_nWRL")), - MA35_PIN(159, PK5, 0xd0, 0x14, + MA35_PIN(165, PK5, 0xd0, 0x14, MA35_MUX(0x0, "GPK5"), MA35_MUX(0x1, "EPWM1_CH1"), MA35_MUX(0x2, "UART12_nRTS"), @@ -1205,28 +1272,30 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "EADC0_ST"), MA35_MUX(0xb, "TM8_EXT"), MA35_MUX(0xd, "INT1")), - MA35_PIN(160, PK6, 0xd0, 0x18, + MA35_PIN(166, PK6, 0xd0, 0x18, MA35_MUX(0x0, "GPK6"), MA35_MUX(0x1, "EPWM1_CH2"), MA35_MUX(0x2, "UART12_RXD"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL4_DM"), MA35_MUX(0x5, "SPI2_MOSI"), MA35_MUX(0x7, "I2S1_BCLK"), MA35_MUX(0x8, "SC0_RST"), MA35_MUX(0xb, "TM6"), MA35_MUX(0xd, "INT2")), - MA35_PIN(161, PK7, 0xd0, 0x1c, + MA35_PIN(167, PK7, 0xd0, 0x1c, MA35_MUX(0x0, "GPK7"), MA35_MUX(0x1, "EPWM1_CH3"), MA35_MUX(0x2, "UART12_TXD"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL4_DP"), MA35_MUX(0x5, "SPI2_MISO"), MA35_MUX(0x7, "I2S1_LRCK"), MA35_MUX(0x8, "SC0_PWR"), MA35_MUX(0x9, "CLKO"), MA35_MUX(0xb, "TM6_EXT"), MA35_MUX(0xd, "INT3")), - MA35_PIN(162, PK8, 0xd4, 0x0, + MA35_PIN(168, PK8, 0xd4, 0x0, MA35_MUX(0x0, "GPK8"), MA35_MUX(0x1, "EPWM1_CH0"), MA35_MUX(0x4, "I2C3_SDA"), @@ -1237,25 +1306,27 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xa, "EBI_ADR15"), MA35_MUX(0xb, "TM8"), MA35_MUX(0xc, "QEI1_INDEX")), - MA35_PIN(163, PK9, 0xd4, 0x4, + MA35_PIN(169, PK9, 0xd4, 0x4, MA35_MUX(0x0, "GPK9"), MA35_MUX(0x4, "I2C3_SCL"), MA35_MUX(0x6, "CCAP0_SCLK"), MA35_MUX(0x8, "EBI_AD0"), MA35_MUX(0xa, "EBI_ADR0")), - MA35_PIN(164, PK10, 0xd4, 0x8, + MA35_PIN(170, PK10, 0xd4, 0x8, MA35_MUX(0x0, "GPK10"), MA35_MUX(0x3, "CAN1_RXD"), + MA35_MUX(0x4, "USBHL3_DM"), MA35_MUX(0x6, "CCAP0_PIXCLK"), MA35_MUX(0x8, "EBI_AD1"), MA35_MUX(0xa, "EBI_ADR1")), - MA35_PIN(165, PK11, 0xd4, 0xc, + MA35_PIN(171, PK11, 0xd4, 0xc, MA35_MUX(0x0, "GPK11"), MA35_MUX(0x3, "CAN1_TXD"), + MA35_MUX(0x4, "USBHL3_DP"), MA35_MUX(0x6, "CCAP0_HSYNC"), MA35_MUX(0x8, "EBI_AD2"), MA35_MUX(0xa, "EBI_ADR2")), - MA35_PIN(166, PK12, 0xd4, 0x10, + MA35_PIN(172, PK12, 0xd4, 0x10, MA35_MUX(0x0, "GPK12"), MA35_MUX(0x1, "EPWM2_CH0"), MA35_MUX(0x2, "UART1_nCTS"), @@ -1266,7 +1337,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "SC0_CLK"), MA35_MUX(0xb, "TM10"), MA35_MUX(0xd, "INT2")), - MA35_PIN(167, PK13, 0xd4, 0x14, + MA35_PIN(173, PK13, 0xd4, 0x14, MA35_MUX(0x0, "GPK13"), MA35_MUX(0x1, "EPWM2_CH1"), MA35_MUX(0x2, "UART1_nRTS"), @@ -1276,28 +1347,30 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x6, "SPI1_CLK"), MA35_MUX(0x8, "SC0_DAT"), MA35_MUX(0xb, "TM10_EXT")), - MA35_PIN(168, PK14, 0xd4, 0x18, + MA35_PIN(174, PK14, 0xd4, 0x18, MA35_MUX(0x0, "GPK14"), MA35_MUX(0x1, "EPWM2_CH2"), MA35_MUX(0x2, "UART1_RXD"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL4_DM"), MA35_MUX(0x5, "I2S0_DI"), MA35_MUX(0x6, "SPI1_MOSI"), MA35_MUX(0x8, "SC0_RST"), MA35_MUX(0xa, "I2C5_SDA"), MA35_MUX(0xb, "TM11"), MA35_MUX(0xd, "INT3")), - MA35_PIN(169, PK15, 0xd4, 0x1c, + MA35_PIN(175, PK15, 0xd4, 0x1c, MA35_MUX(0x0, "GPK15"), MA35_MUX(0x1, "EPWM2_CH3"), MA35_MUX(0x2, "UART1_TXD"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL4_DP"), MA35_MUX(0x5, "I2S0_DO"), MA35_MUX(0x6, "SPI1_MISO"), MA35_MUX(0x8, "SC0_PWR"), MA35_MUX(0xa, "I2C5_SCL"), MA35_MUX(0xb, "TM11_EXT")), - MA35_PIN(170, PL0, 0xd8, 0x0, + MA35_PIN(176, PL0, 0xd8, 0x0, MA35_MUX(0x0, "GPL0"), MA35_MUX(0x1, "EPWM1_CH0"), MA35_MUX(0x2, "UART11_nCTS"), @@ -1310,7 +1383,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC1_CLK"), MA35_MUX(0xb, "TM5"), MA35_MUX(0xc, "QEI1_A")), - MA35_PIN(171, PL1, 0xd8, 0x4, + MA35_PIN(177, PL1, 0xd8, 0x4, MA35_MUX(0x0, "GPL1"), MA35_MUX(0x1, "EPWM1_CH1"), MA35_MUX(0x2, "UART11_nRTS"), @@ -1323,11 +1396,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC1_DAT"), MA35_MUX(0xb, "TM5_EXT"), MA35_MUX(0xc, "QEI1_B")), - MA35_PIN(172, PL2, 0xd8, 0x8, + MA35_PIN(178, PL2, 0xd8, 0x8, MA35_MUX(0x0, "GPL2"), MA35_MUX(0x1, "EPWM1_CH2"), MA35_MUX(0x2, "UART11_RXD"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL4_DM"), MA35_MUX(0x5, "SPI2_SS0"), MA35_MUX(0x6, "QSPI1_SS1"), MA35_MUX(0x7, "I2S0_DI"), @@ -1335,11 +1409,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC1_RST"), MA35_MUX(0xb, "TM7"), MA35_MUX(0xc, "QEI1_INDEX")), - MA35_PIN(173, PL3, 0xd8, 0xc, + MA35_PIN(179, PL3, 0xd8, 0xc, MA35_MUX(0x0, "GPL3"), MA35_MUX(0x1, "EPWM1_CH3"), MA35_MUX(0x2, "UART11_TXD"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL4_DP"), MA35_MUX(0x5, "SPI2_CLK"), MA35_MUX(0x6, "QSPI1_CLK"), MA35_MUX(0x7, "I2S0_DO"), @@ -1347,7 +1422,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC1_PWR"), MA35_MUX(0xb, "TM7_EXT"), MA35_MUX(0xc, "ECAP0_IC0")), - MA35_PIN(174, PL4, 0xd8, 0x10, + MA35_PIN(180, PL4, 0xd8, 0x10, MA35_MUX(0x0, "GPL4"), MA35_MUX(0x1, "EPWM1_CH4"), MA35_MUX(0x2, "UART2_nCTS"), @@ -1360,7 +1435,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC1_nCD"), MA35_MUX(0xb, "TM9"), MA35_MUX(0xc, "ECAP0_IC1")), - MA35_PIN(175, PL5, 0xd8, 0x14, + MA35_PIN(181, PL5, 0xd8, 0x14, MA35_MUX(0x0, "GPL5"), MA35_MUX(0x1, "EPWM1_CH5"), MA35_MUX(0x2, "UART2_nRTS"), @@ -1373,28 +1448,30 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC0_nCD"), MA35_MUX(0xb, "TM9_EXT"), MA35_MUX(0xc, "ECAP0_IC2")), - MA35_PIN(176, PL6, 0xd8, 0x18, + MA35_PIN(182, PL6, 0xd8, 0x18, MA35_MUX(0x0, "GPL6"), MA35_MUX(0x1, "EPWM0_CH0"), MA35_MUX(0x2, "UART2_RXD"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL5_DM"), MA35_MUX(0x6, "QSPI1_MOSI1"), MA35_MUX(0x7, "TRACE_CLK"), MA35_MUX(0x8, "EBI_AD5"), MA35_MUX(0xb, "TM3"), MA35_MUX(0xc, "ECAP1_IC0"), MA35_MUX(0xd, "INT0")), - MA35_PIN(177, PL7, 0xd8, 0x1c, + MA35_PIN(183, PL7, 0xd8, 0x1c, MA35_MUX(0x0, "GPL7"), MA35_MUX(0x1, "EPWM0_CH1"), MA35_MUX(0x2, "UART2_TXD"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL5_DP"), MA35_MUX(0x6, "QSPI1_MISO1"), MA35_MUX(0x8, "EBI_AD6"), MA35_MUX(0xb, "TM3_EXT"), MA35_MUX(0xc, "ECAP1_IC1"), MA35_MUX(0xd, "INT1")), - MA35_PIN(178, PL8, 0xdc, 0x0, + MA35_PIN(184, PL8, 0xdc, 0x0, MA35_MUX(0x0, "GPL8"), MA35_MUX(0x1, "EPWM0_CH2"), MA35_MUX(0x2, "UART14_nCTS"), @@ -1408,7 +1485,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xb, "TM4"), MA35_MUX(0xc, "ECAP1_IC2"), MA35_MUX(0xd, "INT2")), - MA35_PIN(179, PL9, 0xdc, 0x4, + MA35_PIN(185, PL9, 0xdc, 0x4, MA35_MUX(0x0, "GPL9"), MA35_MUX(0x1, "EPWM0_CH3"), MA35_MUX(0x2, "UART14_nRTS"), @@ -1422,11 +1499,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xb, "TM4_EXT"), MA35_MUX(0xc, "QEI0_A"), MA35_MUX(0xd, "INT3")), - MA35_PIN(180, PL10, 0xdc, 0x8, + MA35_PIN(186, PL10, 0xdc, 0x8, MA35_MUX(0x0, "GPL10"), MA35_MUX(0x1, "EPWM0_CH4"), MA35_MUX(0x2, "UART14_RXD"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL2_DM"), MA35_MUX(0x5, "SPI3_MOSI"), MA35_MUX(0x6, "EPWM0_CH5"), MA35_MUX(0x7, "I2S1_DI"), @@ -1434,11 +1512,12 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC0_RST"), MA35_MUX(0xb, "EBI_nWRH"), MA35_MUX(0xc, "QEI0_B")), - MA35_PIN(181, PL11, 0xdc, 0xc, + MA35_PIN(187, PL11, 0xdc, 0xc, MA35_MUX(0x0, "GPL11"), MA35_MUX(0x1, "EPWM0_CH5"), MA35_MUX(0x2, "UART14_TXD"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL2_DP"), MA35_MUX(0x5, "SPI3_MISO"), MA35_MUX(0x6, "EPWM1_CH5"), MA35_MUX(0x7, "I2S1_DO"), @@ -1446,7 +1525,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x9, "SC0_PWR"), MA35_MUX(0xb, "EBI_nWRL"), MA35_MUX(0xc, "QEI0_INDEX")), - MA35_PIN(182, PL12, 0xdc, 0x10, + MA35_PIN(188, PL12, 0xdc, 0x10, MA35_MUX(0x0, "GPL12"), MA35_MUX(0x1, "EPWM0_SYNC_IN"), MA35_MUX(0x2, "UART7_nCTS"), @@ -1463,7 +1542,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xd, "EBI_AD11"), MA35_MUX(0xe, "RGMII0_PPS"), MA35_MUX(0xf, "RMII0_PPS")), - MA35_PIN(183, PL13, 0xdc, 0x14, + MA35_PIN(189, PL13, 0xdc, 0x14, MA35_MUX(0x0, "GPL13"), MA35_MUX(0x1, "EPWM0_SYNC_OUT"), MA35_MUX(0x2, "UART7_nRTS"), @@ -1480,7 +1559,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xd, "EBI_AD12"), MA35_MUX(0xe, "RGMII1_PPS"), MA35_MUX(0xf, "RMII1_PPS")), - MA35_PIN(184, PL14, 0xdc, 0x18, + MA35_PIN(190, PL14, 0xdc, 0x18, MA35_MUX(0x0, "GPL14"), MA35_MUX(0x1, "EPWM0_CH2"), MA35_MUX(0x2, "UART7_RXD"), @@ -1492,7 +1571,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xb, "TM2"), MA35_MUX(0xc, "INT0"), MA35_MUX(0xd, "EBI_AD13")), - MA35_PIN(185, PL15, 0xdc, 0x1c, + MA35_PIN(191, PL15, 0xdc, 0x1c, MA35_MUX(0x0, "GPL15"), MA35_MUX(0x1, "EPWM0_CH1"), MA35_MUX(0x2, "UART7_TXD"), @@ -1505,86 +1584,92 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0xb, "TM2_EXT"), MA35_MUX(0xc, "INT2"), MA35_MUX(0xd, "EBI_AD14")), - MA35_PIN(186, PM0, 0xe0, 0x0, + MA35_PIN(192, PM0, 0xe0, 0x0, MA35_MUX(0x0, "GPM0"), MA35_MUX(0x4, "I2C4_SDA"), MA35_MUX(0x6, "CCAP0_VSYNC"), MA35_MUX(0x8, "EBI_AD3"), MA35_MUX(0xa, "EBI_ADR3")), - MA35_PIN(187, PM1, 0xe0, 0x4, + MA35_PIN(193, PM1, 0xe0, 0x4, MA35_MUX(0x0, "GPM1"), MA35_MUX(0x4, "I2C4_SCL"), MA35_MUX(0x5, "SPI3_I2SMCLK"), MA35_MUX(0x6, "CCAP0_SFIELD"), MA35_MUX(0x8, "EBI_AD4"), MA35_MUX(0xa, "EBI_ADR4")), - MA35_PIN(188, PM2, 0xe0, 0x8, + MA35_PIN(194, PM2, 0xe0, 0x8, MA35_MUX(0x0, "GPM2"), MA35_MUX(0x3, "CAN3_RXD"), + MA35_MUX(0x4, "USBHL0_DM"), MA35_MUX(0x6, "CCAP0_DATA0"), MA35_MUX(0x8, "EBI_AD5"), MA35_MUX(0xa, "EBI_ADR5")), - MA35_PIN(189, PM3, 0xe0, 0xc, + MA35_PIN(195, PM3, 0xe0, 0xc, MA35_MUX(0x0, "GPM3"), MA35_MUX(0x3, "CAN3_TXD"), + MA35_MUX(0x4, "USBHL0_DP"), MA35_MUX(0x6, "CCAP0_DATA1"), MA35_MUX(0x8, "EBI_AD6"), MA35_MUX(0xa, "EBI_ADR6")), - MA35_PIN(190, PM4, 0xe0, 0x10, + MA35_PIN(196, PM4, 0xe0, 0x10, MA35_MUX(0x0, "GPM4"), MA35_MUX(0x4, "I2C5_SDA"), MA35_MUX(0x6, "CCAP0_DATA2"), MA35_MUX(0x8, "EBI_AD7"), MA35_MUX(0xa, "EBI_ADR7")), - MA35_PIN(191, PM5, 0xe0, 0x14, + MA35_PIN(197, PM5, 0xe0, 0x14, MA35_MUX(0x0, "GPM5"), MA35_MUX(0x4, "I2C5_SCL"), MA35_MUX(0x6, "CCAP0_DATA3"), MA35_MUX(0x8, "EBI_AD8"), MA35_MUX(0xa, "EBI_ADR8")), - MA35_PIN(192, PM6, 0xe0, 0x18, + MA35_PIN(198, PM6, 0xe0, 0x18, MA35_MUX(0x0, "GPM6"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL1_DM"), MA35_MUX(0x6, "CCAP0_DATA4"), MA35_MUX(0x8, "EBI_AD9"), MA35_MUX(0xa, "EBI_ADR9")), - MA35_PIN(193, PM7, 0xe0, 0x1c, + MA35_PIN(199, PM7, 0xe0, 0x1c, MA35_MUX(0x0, "GPM7"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL1_DP"), MA35_MUX(0x6, "CCAP0_DATA5"), MA35_MUX(0x8, "EBI_AD10"), MA35_MUX(0xa, "EBI_ADR10")), - MA35_PIN(194, PM8, 0xe4, 0x0, + MA35_PIN(200, PM8, 0xe4, 0x0, MA35_MUX(0x0, "GPM8"), MA35_MUX(0x4, "I2C0_SDA"), MA35_MUX(0x6, "CCAP0_DATA6"), MA35_MUX(0x8, "EBI_AD11"), MA35_MUX(0xa, "EBI_ADR11")), - MA35_PIN(195, PM9, 0xe4, 0x4, + MA35_PIN(201, PM9, 0xe4, 0x4, MA35_MUX(0x0, "GPM9"), MA35_MUX(0x4, "I2C0_SCL"), MA35_MUX(0x6, "CCAP0_DATA7"), MA35_MUX(0x8, "EBI_AD12"), MA35_MUX(0xa, "EBI_ADR12")), - MA35_PIN(196, PM10, 0xe4, 0x8, + MA35_PIN(202, PM10, 0xe4, 0x8, MA35_MUX(0x0, "GPM10"), MA35_MUX(0x1, "EPWM1_CH2"), MA35_MUX(0x3, "CAN2_RXD"), + MA35_MUX(0x4, "USBHL4_DM"), MA35_MUX(0x5, "SPI3_SS0"), MA35_MUX(0x6, "CCAP0_DATA8"), MA35_MUX(0x7, "SPI2_I2SMCLK"), MA35_MUX(0x8, "EBI_AD13"), MA35_MUX(0xa, "EBI_ADR13")), - MA35_PIN(197, PM11, 0xe4, 0xc, + MA35_PIN(203, PM11, 0xe4, 0xc, MA35_MUX(0x0, "GPM11"), MA35_MUX(0x1, "EPWM1_CH3"), MA35_MUX(0x3, "CAN2_TXD"), + MA35_MUX(0x4, "USBHL4_DP"), MA35_MUX(0x5, "SPI3_SS1"), MA35_MUX(0x6, "CCAP0_DATA9"), MA35_MUX(0x7, "SPI2_SS1"), MA35_MUX(0x8, "EBI_AD14"), MA35_MUX(0xa, "EBI_ADR14")), - MA35_PIN(198, PM12, 0xe4, 0x10, + MA35_PIN(204, PM12, 0xe4, 0x10, MA35_MUX(0x0, "GPM12"), MA35_MUX(0x1, "EPWM1_CH4"), MA35_MUX(0x2, "UART10_nCTS"), @@ -1595,7 +1680,7 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "EBI_AD8"), MA35_MUX(0x9, "I2S1_MCLK"), MA35_MUX(0xb, "TM8")), - MA35_PIN(199, PM13, 0xe4, 0x14, + MA35_PIN(205, PM13, 0xe4, 0x14, MA35_MUX(0x0, "GPM13"), MA35_MUX(0x1, "EPWM1_CH5"), MA35_MUX(0x2, "UART10_nRTS"), @@ -1605,99 +1690,66 @@ static const struct pinctrl_pin_desc ma35d1_pins[] = { MA35_MUX(0x8, "EBI_AD9"), MA35_MUX(0x9, "ECAP1_IC0"), MA35_MUX(0xb, "TM8_EXT")), - MA35_PIN(200, PM14, 0xe4, 0x18, + MA35_PIN(206, PM14, 0xe4, 0x18, MA35_MUX(0x0, "GPM14"), MA35_MUX(0x1, "EPWM1_BRAKE0"), MA35_MUX(0x2, "UART10_RXD"), MA35_MUX(0x3, "TRACE_DATA2"), MA35_MUX(0x4, "CAN2_RXD"), + MA35_MUX(0x5, "USBHL3_DM"), MA35_MUX(0x6, "I2C3_SDA"), MA35_MUX(0x8, "EBI_AD10"), MA35_MUX(0x9, "ECAP1_IC1"), MA35_MUX(0xb, "TM10"), MA35_MUX(0xd, "INT1")), - MA35_PIN(201, PM15, 0xe4, 0x1c, + MA35_PIN(207, PM15, 0xe4, 0x1c, MA35_MUX(0x0, "GPM15"), MA35_MUX(0x1, "EPWM1_BRAKE1"), MA35_MUX(0x2, "UART10_TXD"), MA35_MUX(0x3, "TRACE_DATA3"), MA35_MUX(0x4, "CAN2_TXD"), + MA35_MUX(0x5, "USBHL3_DP"), MA35_MUX(0x6, "I2C3_SCL"), MA35_MUX(0x8, "EBI_AD11"), MA35_MUX(0x9, "ECAP1_IC2"), MA35_MUX(0xb, "TM10_EXT"), MA35_MUX(0xd, "INT2")), - MA35_PIN(202, PN0, 0xe8, 0x0, + MA35_PIN(208, PN0, 0xe8, 0x0, MA35_MUX(0x0, "GPN0"), MA35_MUX(0x4, "I2C2_SDA"), MA35_MUX(0x6, "CCAP1_DATA0")), - MA35_PIN(203, PN1, 0xe8, 0x4, + MA35_PIN(209, PN1, 0xe8, 0x4, MA35_MUX(0x0, "GPN1"), MA35_MUX(0x4, "I2C2_SCL"), MA35_MUX(0x6, "CCAP1_DATA1")), - MA35_PIN(204, PN2, 0xe8, 0x8, + MA35_PIN(210, PN2, 0xe8, 0x8, MA35_MUX(0x0, "GPN2"), MA35_MUX(0x3, "CAN0_RXD"), + MA35_MUX(0x4, "USBHL0_DM"), MA35_MUX(0x6, "CCAP1_DATA2")), - MA35_PIN(205, PN3, 0xe8, 0xc, + MA35_PIN(211, PN3, 0xe8, 0xc, MA35_MUX(0x0, "GPN3"), MA35_MUX(0x3, "CAN0_TXD"), + MA35_MUX(0x4, "USBHL0_DP"), MA35_MUX(0x6, "CCAP1_DATA3")), - MA35_PIN(206, PN4, 0xe8, 0x10, + MA35_PIN(212, PN4, 0xe8, 0x10, MA35_MUX(0x0, "GPN4"), MA35_MUX(0x4, "I2C1_SDA"), MA35_MUX(0x6, "CCAP1_DATA4")), - MA35_PIN(207, PN5, 0xe8, 0x14, + MA35_PIN(213, PN5, 0xe8, 0x14, MA35_MUX(0x0, "GPN5"), MA35_MUX(0x4, "I2C1_SCL"), MA35_MUX(0x6, "CCAP1_DATA5")), - MA35_PIN(208, PN6, 0xe8, 0x18, + MA35_PIN(214, PN6, 0xe8, 0x18, MA35_MUX(0x0, "GPN6"), MA35_MUX(0x3, "CAN1_RXD"), + MA35_MUX(0x4, "USBHL1_DM"), MA35_MUX(0x6, "CCAP1_DATA6")), - MA35_PIN(209, PN7, 0xe8, 0x1c, + MA35_PIN(215, PN7, 0xe8, 0x1c, MA35_MUX(0x0, "GPN7"), MA35_MUX(0x3, "CAN1_TXD"), + MA35_MUX(0x4, "USBHL1_DP"), MA35_MUX(0x6, "CCAP1_DATA7")), - MA35_PIN(210, PN10, 0xec, 0x8, - MA35_MUX(0x0, "GPN10"), - MA35_MUX(0x3, "CAN2_RXD"), - MA35_MUX(0x6, "CCAP1_SCLK")), - MA35_PIN(211, PN11, 0xec, 0xc, - MA35_MUX(0x0, "GPN11"), - MA35_MUX(0x3, "CAN2_TXD"), - MA35_MUX(0x6, "CCAP1_PIXCLK")), - MA35_PIN(212, PN12, 0xec, 0x10, - MA35_MUX(0x0, "GPN12"), - MA35_MUX(0x2, "UART6_nCTS"), - MA35_MUX(0x3, "UART12_RXD"), - MA35_MUX(0x4, "I2C5_SDA"), - MA35_MUX(0x6, "CCAP1_HSYNC")), - MA35_PIN(213, PN13, 0xec, 0x14, - MA35_MUX(0x0, "GPN13"), - MA35_MUX(0x2, "UART6_nRTS"), - MA35_MUX(0x3, "UART12_TXD"), - MA35_MUX(0x4, "I2C5_SCL"), - MA35_MUX(0x6, "CCAP1_VSYNC")), - MA35_PIN(214, PN14, 0xec, 0x18, - MA35_MUX(0x0, "GPN14"), - MA35_MUX(0x2, "UART6_RXD"), - MA35_MUX(0x3, "CAN3_RXD"), - MA35_MUX(0x5, "SPI1_SS1"), - MA35_MUX(0x6, "CCAP1_SFIELD"), - MA35_MUX(0x7, "SPI1_I2SMCLK")), - MA35_PIN(215, PN15, 0xec, 0x1c, - MA35_MUX(0x0, "GPN15"), - MA35_MUX(0x1, "EPWM2_CH4"), - MA35_MUX(0x2, "UART6_TXD"), - MA35_MUX(0x3, "CAN3_TXD"), - MA35_MUX(0x5, "I2S0_MCLK"), - MA35_MUX(0x6, "SPI1_SS1"), - MA35_MUX(0x7, "SPI1_I2SMCLK"), - MA35_MUX(0x8, "SC0_nCD"), - MA35_MUX(0x9, "EADC0_ST"), - MA35_MUX(0xa, "CLKO"), - MA35_MUX(0xb, "TM6")), MA35_PIN(216, PN8, 0xec, 0x0, MA35_MUX(0x0, "GPN8"), MA35_MUX(0x1, "EPWM2_CH4"), diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index 64ed283097887..0771491216280 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -225,10 +225,9 @@ static int parse_fw_cfg(struct fwnode_handle *fwnode, unsigned int count, unsigned long *cfg, unsigned int *ncfg) { - unsigned long *properties; int i, test; - properties = bitmap_zalloc(count, GFP_KERNEL); + unsigned long *properties __free(bitmap) = bitmap_zalloc(count, GFP_KERNEL); for (i = 0; i < count; i++) { u32 val; @@ -263,7 +262,6 @@ static int parse_fw_cfg(struct fwnode_handle *fwnode, if (ret) { pr_err("%pfw: conflicting setting detected for %s\n", fwnode, par->property); - bitmap_free(properties); return -EINVAL; } } @@ -295,7 +293,6 @@ static int parse_fw_cfg(struct fwnode_handle *fwnode, pr_err("%pfw: cannot have multiple drive configurations\n", fwnode); - bitmap_free(properties); return 0; } diff --git a/drivers/pinctrl/qcom/tlmm-test.c b/drivers/pinctrl/qcom/tlmm-test.c index 7d7fff538755a..4ac96538a417f 100644 --- a/drivers/pinctrl/qcom/tlmm-test.c +++ b/drivers/pinctrl/qcom/tlmm-test.c @@ -581,6 +581,9 @@ static int tlmm_reg_base(struct device_node *tlmm, struct resource *res) int ret; int i; + if (!strcmp(tlmm_reg_name, "default_region")) + return of_address_to_resource(tlmm, 0, res); + count = of_property_count_strings(tlmm, "reg-names"); if (count <= 0) { pr_err("failed to find tlmm reg name\n"); @@ -597,18 +600,14 @@ static int tlmm_reg_base(struct device_node *tlmm, struct resource *res) return -EINVAL; } - if (!strcmp(tlmm_reg_name, "default_region")) { - ret = of_address_to_resource(tlmm, 0, res); - } else { - for (i = 0; i < count; i++) { - if (!strcmp(reg_names[i], tlmm_reg_name)) { - ret = of_address_to_resource(tlmm, i, res); - break; - } + for (i = 0; i < count; i++) { + if (!strcmp(reg_names[i], tlmm_reg_name)) { + ret = of_address_to_resource(tlmm, i, res); + break; } - if (i == count) - ret = -EINVAL; } + if (i == count) + ret = -EINVAL; kfree(reg_names); diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c index 1c6b115e65d85..8e7c503f9bb32 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c +++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c @@ -1103,7 +1103,7 @@ static int rzg2l_read_oen(struct rzg2l_pinctrl *pctrl, unsigned int _pin) int bit; if (!pctrl->data->pin_to_oen_bit) - return -EOPNOTSUPP; + return -ENOTSUPP; bit = pctrl->data->pin_to_oen_bit(pctrl, _pin); if (bit < 0) @@ -1145,7 +1145,7 @@ static int rzg2l_write_oen(struct rzg2l_pinctrl *pctrl, unsigned int _pin, u8 oe u8 val; if (!pctrl->data->pin_to_oen_bit) - return -EOPNOTSUPP; + return -ENOTSUPP; bit = pctrl->data->pin_to_oen_bit(pctrl, _pin); if (bit < 0) @@ -1576,7 +1576,7 @@ static int rzg2l_pinctrl_pinconf_set(struct pinctrl_dev *pctldev, break; default: - return -EOPNOTSUPP; + return -ENOTSUPP; } } @@ -1658,7 +1658,7 @@ static int rzg2l_pinctrl_pinconf_group_get(struct pinctrl_dev *pctldev, /* Check config matching between to pin */ if (i && prev_config != *config) - return -EOPNOTSUPP; + return -ENOTSUPP; prev_config = *config; } diff --git a/drivers/pinctrl/spacemit/pinctrl-k1.c b/drivers/pinctrl/spacemit/pinctrl-k1.c index b0be62b1c8161..95024e2bb5a58 100644 --- a/drivers/pinctrl/spacemit/pinctrl-k1.c +++ b/drivers/pinctrl/spacemit/pinctrl-k1.c @@ -795,7 +795,7 @@ static int spacemit_pin_set_config(struct spacemit_pinctrl *pctrl, void __iomem *reg; unsigned int mux; - if (!pin) + if (!spin) return -EINVAL; reg = spacemit_pin_to_reg(pctrl, spin->pin); diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c index 69cd2b4ebd7d7..462aa1c4a5fa6 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c @@ -26,7 +26,6 @@ static const u8 a523_r_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] = static struct sunxi_pinctrl_desc a523_r_pinctrl_data = { .irq_banks = ARRAY_SIZE(a523_r_irq_bank_map), .irq_bank_map = a523_r_irq_bank_map, - .irq_read_needs_mux = true, .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL, .pin_base = PL_BASE, }; diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c index 7d2308c37d29e..b6f78f1f30ac4 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c @@ -26,7 +26,6 @@ static const u8 a523_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] = static struct sunxi_pinctrl_desc a523_pinctrl_data = { .irq_banks = ARRAY_SIZE(a523_irq_bank_map), .irq_bank_map = a523_irq_bank_map, - .irq_read_needs_mux = true, .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL, }; diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c index d3042e0c9712e..25489beeb3125 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c +++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c @@ -925,7 +925,7 @@ static int sunxi_pmx_request(struct pinctrl_dev *pctldev, unsigned offset) return 0; out: - regulator_put(s_reg->regulator); + regulator_put(reg); return ret; } diff --git a/drivers/platform/chrome/chromeos_privacy_screen.c b/drivers/platform/chrome/chromeos_privacy_screen.c index abc5d189a3894..407b04207de2f 100644 --- a/drivers/platform/chrome/chromeos_privacy_screen.c +++ b/drivers/platform/chrome/chromeos_privacy_screen.c @@ -104,6 +104,9 @@ static const struct drm_privacy_screen_ops chromeos_privacy_screen_ops = { static int chromeos_privacy_screen_probe(struct platform_device *pdev) { + if (!ACPI_COMPANION(&pdev->dev)) + return -ENODEV; + struct drm_privacy_screen *drm_privacy_screen = drm_privacy_screen_register(&pdev->dev, &chromeos_privacy_screen_ops, diff --git a/drivers/platform/chrome/chromeos_tbmc.c b/drivers/platform/chrome/chromeos_tbmc.c index 5133806b2d958..fd756761a481f 100644 --- a/drivers/platform/chrome/chromeos_tbmc.c +++ b/drivers/platform/chrome/chromeos_tbmc.c @@ -69,9 +69,13 @@ static int chromeos_tbmc_probe(struct platform_device *pdev) { struct input_dev *idev; struct device *dev = &pdev->dev; - struct acpi_device *adev = ACPI_COMPANION(dev); + struct acpi_device *adev; int ret; + adev = ACPI_COMPANION(dev); + if (!adev) + return -ENODEV; + idev = devm_input_allocate_device(dev); if (!idev) return -ENOMEM; diff --git a/drivers/platform/chrome/wilco_ec/event.c b/drivers/platform/chrome/wilco_ec/event.c index b6e935badc0e9..1b5cb89839e08 100644 --- a/drivers/platform/chrome/wilco_ec/event.c +++ b/drivers/platform/chrome/wilco_ec/event.c @@ -452,8 +452,13 @@ static void hangup_device(struct event_device_data *dev_data) static int event_device_probe(struct platform_device *pdev) { struct event_device_data *dev_data; + struct acpi_device *adev; int error, minor; + adev = ACPI_COMPANION(&pdev->dev); + if (!adev) + return -ENODEV; + minor = ida_alloc_max(&event_ida, EVENT_MAX_DEV-1, GFP_KERNEL); if (minor < 0) { error = minor; @@ -494,8 +499,7 @@ static int event_device_probe(struct platform_device *pdev) goto free_dev_data; /* Install an ACPI notify handler. */ - error = acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev), - ACPI_DEVICE_NOTIFY, + error = acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY, event_device_notify, &pdev->dev); if (error) goto free_cdev; diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 7a4956088300c..cb97aac80248d 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -330,6 +330,21 @@ config ASUS_TF103C_DOCK If you have an Asus TF103C tablet say Y or M here, for a generic x86 distro config say M here. +config AYN_EC + tristate "AYN x86 devices EC platform control" + depends on ACPI + depends on HWMON + depends on NEW_LEDS + select LEDS_CLASS + select LEDS_CLASS_MULTICOLOR + help + This is a driver for AYN and Tectoy x86 handheld devices. It provides + temperature monitoring, manual fan speed control, fan curve control, + and chassis RGB settings. + + If you have an x86 AYN or Tectoy handheld device say M here. The module + will be called ayn-platform. + config AYANEO_EC tristate "Ayaneo EC platform control" depends on DMI diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index 872ac3842391f..31b4fd9083948 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -40,6 +40,9 @@ obj-$(CONFIG_ASUS_TF103C_DOCK) += asus-tf103c-dock.o obj-$(CONFIG_EEEPC_LAPTOP) += eeepc-laptop.o obj-$(CONFIG_EEEPC_WMI) += eeepc-wmi.o +# Ayn +obj-$(CONFIG_AYN_EC) += ayn-ec.o + # Ayaneo obj-$(CONFIG_AYANEO_EC) += ayaneo-ec.o diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c index 24506e3429430..74ddf1d8289a9 100644 --- a/drivers/platform/x86/amd/pmc/pmc-quirks.c +++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c @@ -18,6 +18,7 @@ struct quirk_entry { u32 s2idle_bug_mmio; bool spurious_8042; + bool need_suspend_delay; }; static struct quirk_entry quirk_s2idle_bug = { @@ -33,6 +34,10 @@ static struct quirk_entry quirk_s2idle_spurious_8042 = { .spurious_8042 = true, }; +static struct quirk_entry quirk_s2idle_need_suspend_delay = { + .need_suspend_delay = true, +}; + static const struct dmi_system_id fwbug_list[] = { { .ident = "L14 Gen2 AMD", @@ -203,6 +208,35 @@ static const struct dmi_system_id fwbug_list[] = { DMI_MATCH(DMI_PRODUCT_NAME, "82XQ"), } }, + /* https://bugzilla.kernel.org/show_bug.cgi?id=221383 */ + { + .ident = "Zen3-based IdeaPad Slim and similar", + .driver_data = &quirk_s2idle_need_suspend_delay, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + /* + * Note: there are also some Zen2-based 82X* devices that + * need different quirks, they're already handled above + */ + DMI_MATCH(DMI_PRODUCT_NAME, "82X"), + } + }, + { + .ident = "Zen3+-based IdeaPad Slim and similar", + .driver_data = &quirk_s2idle_need_suspend_delay, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83K"), + } + }, + { + .ident = "IdeaPad Slim 3 15ARP10 (83MM)", + .driver_data = &quirk_s2idle_need_suspend_delay, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83MM"), + } + }, /* https://bugzilla.kernel.org/show_bug.cgi?id=221273 */ { .ident = "Thinkpad L14 Gen3", @@ -356,6 +390,11 @@ void amd_pmc_process_restore_quirks(struct amd_pmc_dev *dev) amd_pmc_skip_nvme_smi_handler(dev->quirks->s2idle_bug_mmio); } +bool amd_pmc_quirk_need_suspend_delay(struct amd_pmc_dev *dev) +{ + return dev->quirks && dev->quirks->need_suspend_delay; +} + void amd_pmc_quirks_init(struct amd_pmc_dev *dev) { const struct dmi_system_id *dmi_id; diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c index cae3fcafd4d7b..2cd77a44c7e7c 100644 --- a/drivers/platform/x86/amd/pmc/pmc.c +++ b/drivers/platform/x86/amd/pmc/pmc.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,11 @@ static bool disable_workarounds; module_param(disable_workarounds, bool, 0644); MODULE_PARM_DESC(disable_workarounds, "Disable workarounds for platform bugs"); +static int delay_suspend = -1; +module_param(delay_suspend, int, 0644); +MODULE_PARM_DESC(delay_suspend, + "Delays s2idle by 2.5 seconds to work around buggy ECs, often causing keyboard issues after suspend. 0: don't delay, 1: do delay, -1 (default): let amd_pmc decide. If you need this please report this to: platform-driver-x86@vger.kernel.org"); + static struct amd_pmc_dev pmc; static inline u32 amd_pmc_reg_read(struct amd_pmc_dev *dev, int reg_offset) @@ -598,6 +604,72 @@ static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg) return rc; } +static bool amd_pmc_intermediate_wakeup_need_delay(struct amd_pmc_dev *pdev) +{ + /* + * Starting a new HW sleep cycle right after waking from one + * can cause electrical problems triggering the over voltage protection. + * That is avoided by delaying the next suspend a bit, see also + * https://lore.kernel.org/all/20250414162446.3853194-1-superm1@kernel.org/ + */ + struct smu_metrics table; + + return get_metrics_table(pdev, &table) == 0 && table.s0i3_last_entry_status; +} + +static bool amd_pmc_want_suspend_delay(struct amd_pmc_dev *pdev) +{ + /* + * intermediate_wakeup implies that the machine didn't get to deepest sleep + * state before - otherwise this function isn't called in amd_pmc_s2idle_check() + * because amd_pmc_intermediate_wakeup_need_delay() returns true first. + * On some IdeaPads that happens when charging, because the EC seems + * to send lots of messages then that wake the machine. + * + * But even in that case, the sleep here is necessary (on those IdeaPads), + * otherwise they wake up completely (resume) after a few seconds. + * So this variable is only used to avoid spamming dmesg on each + * intermediate wakeup. + */ + bool intermediate_wakeup = !pdev->is_first_check_after_suspend; + + /* + * Some Lenovo Laptops (like different IdeaPad 3 Slims) need some + * me-time before sleeping or they get uncooperative after waking + * up and don't send events for keyboard and lid switch anymore. + * + * Unfortunately this doesn't entirely fix the problem: It can still + * happen when resuming with a timer (wakealarm), but at least the + * more common usecases (wakeup by opening lid or pressing a key) + * work fine with this workaround. + * + * See https://bugzilla.kernel.org/show_bug.cgi?id=221383 + */ + if (amd_pmc_quirk_need_suspend_delay(pdev)) { + /* + * delay_suspend=1 force-enables this, otherwise it can be + * disabled with disable_workarounds or delay_suspend=0 + */ + if (delay_suspend == 1 || (delay_suspend == -1 && !disable_workarounds)) { + if (!intermediate_wakeup) + dev_info(pdev->dev, "Delaying suspend by 2.5s to avoid platform bug\n"); + return true; + } + if (!intermediate_wakeup) + dev_info(pdev->dev, "Not delaying suspend because of module parameter, even though your device is assumed to need it!\n"); + } else if (delay_suspend == 1) { + if (!intermediate_wakeup) + dev_info(pdev->dev, "Delaying suspend by 2.5s because delay_suspend=1. If this solves problems on your machine, please report this whole line to: platform-driver-x86@vger.kernel.org so it can be automatically detected as affected in the future. System Vendor: \"%s\" Product Name: \"%s\" Product Family: \"%s\" Board Vendor: \"%s\" Board Name: \"%s\"\n", + dmi_get_system_info(DMI_SYS_VENDOR) ?: "(Unknown)", + dmi_get_system_info(DMI_PRODUCT_NAME) ?: "(Unknown)", + dmi_get_system_info(DMI_PRODUCT_FAMILY) ?: "(Unknown)", + dmi_get_system_info(DMI_BOARD_VENDOR) ?: "(Unknown)", + dmi_get_system_info(DMI_BOARD_NAME) ?: "(Unknown)"); + return true; + } + return false; +} + static void amd_pmc_s2idle_prepare(void) { struct amd_pmc_dev *pdev = &pmc; @@ -605,6 +677,9 @@ static void amd_pmc_s2idle_prepare(void) u8 msg; u32 arg = 1; + /* Reset this variable because this is a fresh suspend */ + pdev->is_first_check_after_suspend = true; + /* Reset and Start SMU logging - to monitor the s0i3 stats */ amd_pmc_setup_smu_logging(pdev); @@ -632,11 +707,10 @@ static void amd_pmc_s2idle_prepare(void) static void amd_pmc_s2idle_check(void) { struct amd_pmc_dev *pdev = &pmc; - struct smu_metrics table; int rc; - /* Avoid triggering OVP */ - if (!get_metrics_table(pdev, &table) && table.s0i3_last_entry_status) + if (amd_pmc_intermediate_wakeup_need_delay(pdev) || + amd_pmc_want_suspend_delay(pdev)) msleep(2500); /* Dump the IdleMask before we add to the STB */ @@ -645,6 +719,9 @@ static void amd_pmc_s2idle_check(void) rc = amd_stb_write(pdev, AMD_PMC_STB_S2IDLE_CHECK); if (rc) dev_err(pdev->dev, "error writing to STB: %d\n", rc); + + /* remember that first check after suspend is done (until next prepare) */ + pdev->is_first_check_after_suspend = false; } static int amd_pmc_dump_data(struct amd_pmc_dev *pdev) diff --git a/drivers/platform/x86/amd/pmc/pmc.h b/drivers/platform/x86/amd/pmc/pmc.h index fe3f53eb59558..8aa7073ed09f2 100644 --- a/drivers/platform/x86/amd/pmc/pmc.h +++ b/drivers/platform/x86/amd/pmc/pmc.h @@ -114,6 +114,7 @@ struct amd_pmc_dev { struct dentry *dbgfs_dir; struct quirk_entry *quirks; bool disable_8042_wakeup; + bool is_first_check_after_suspend; struct amd_mp2_dev *mp2; struct stb_arg stb_arg; }; @@ -147,6 +148,7 @@ enum amd_pmc_def { }; void amd_pmc_process_restore_quirks(struct amd_pmc_dev *dev); +bool amd_pmc_quirk_need_suspend_delay(struct amd_pmc_dev *dev); void amd_pmc_quirks_init(struct amd_pmc_dev *dev); void amd_mp2_stb_init(struct amd_pmc_dev *dev); void amd_mp2_stb_deinit(struct amd_pmc_dev *dev); diff --git a/drivers/platform/x86/ayn-ec.c b/drivers/platform/x86/ayn-ec.c new file mode 100644 index 0000000000000..12c8605d3a8ec --- /dev/null +++ b/drivers/platform/x86/ayn-ec.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Platform driver for AYN x86 Handhelds. + * + * Implements multiple attributes provided by the EC. Fan reading and control, + * as well as temperature sensor readings are exposed via hwmon sysfs. EC RGB + * control is exposed via an led-class-multicolor interface. + * + * Fan control is provided via a pwm interface in the range [0-255]. AYN use + * [0-128] as the range in the EC, the written value is scaled to accommodate. + * The EC also provides a configurable fan curve with five set points that + * associate a temperature in Celcius [0-100] with a fan speed [0-128]. The + * auto_point fan speeds are also scaled from the range [0-255]. Temperature + * readings are scaled from degrees to millidegrees when read. + * + * RGB control is provided using 4 registers. One each for the colors red, + * green, and blue are [0-255]. There is also a effect register that takes + * switches between an EC controlled breathing that cycles through all colors + * and fades in/out, and manual, which enables setting a user defined color. + * + * Copyright (C) 2025 Derek J. Clark + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Fan speed and PWM registers */ +#define AYN_SENSOR_PWM_FAN_ENABLE_REG 0x10 /* PWM operating mode */ +#define AYN_SENSOR_PWM_FAN_SET_REG 0x11 /* PWM duty cycle */ +#define AYN_SENSOR_PWM_FAN_SPEED_REG 0x20 /* Fan speed */ + +/* EC controlled fan curve registers */ +#define AYN_SENSOR_PWM_FAN_SPEED_1_REG 0x12 +#define AYN_SENSOR_PWM_FAN_SPEED_2_REG 0x14 +#define AYN_SENSOR_PWM_FAN_SPEED_3_REG 0x16 +#define AYN_SENSOR_PWM_FAN_SPEED_4_REG 0x18 +#define AYN_SENSOR_PWM_FAN_SPEED_5_REG 0x1A +#define AYN_SENSOR_PWM_FAN_TEMP_1_REG 0x13 +#define AYN_SENSOR_PWM_FAN_TEMP_2_REG 0x15 +#define AYN_SENSOR_PWM_FAN_TEMP_3_REG 0x17 +#define AYN_SENSOR_PWM_FAN_TEMP_4_REG 0x19 +#define AYN_SENSOR_PWM_FAN_TEMP_5_REG 0x1B + +/* AYN EC PWM Fan modes */ +#define AYN_PWM_FAN_MODE_MANUAL 0x00 +#define AYN_PWM_FAN_MODE_AUTO 0x01 +#define AYN_PWM_FAN_MODE_EC_CURVE 0x02 + +/* hwmon fan modes */ +#define HWMON_PWM_FAN_MODE_FULL 0x00 +#define HWMON_PWM_FAN_MODE_MANUAL 0x01 +#define HWMON_PWM_FAN_MODE_AUTO 0x02 +#define HWMON_PWM_FAN_MODE_EC_CURVE 0x03 + +/* EC Temperature Sensors */ +#define AYN_SENSOR_BAT_TEMP_REG 0x04 /* Battery */ +#define AYN_SENSOR_CHARGE_TEMP_REG 0x07 /* Charger IC */ +#define AYN_SENSOR_MB_TEMP_REG 0x05 /* Motherboard */ +#define AYN_SENSOR_PROC_TEMP_REG 0x09 /* CPU Core */ +#define AYN_SENSOR_VCORE_TEMP_REG 0x08 /* vCore */ + +/* EC Controlled RGB registers */ +#define AYN_LED_MC_RED_REG 0xB0 /* Range 0x00-0xFF */ +#define AYN_LED_MC_GREEN_REG 0xB1 /* Range 0x00-0xFF */ +#define AYN_LED_MC_BLUE_REG 0xB2 /* Range 0x00-0xFF */ +#define AYN_RGB_EFFECT_REG 0xB3 + +/* RGB effect modes */ +#define AYN_RGB_EFFECT_BREATHE 0x00 +#define AYN_RGB_EFFECT_MONOCOLOR 0x55 +#define AYN_RGB_EFFECT_WRITE 0xAA + +/* Handle ACPI lock mechanism */ +#define ACPI_LOCK_DELAY_MS 500 + +int ayn_pwm_curve_registers[10] = { + AYN_SENSOR_PWM_FAN_SPEED_1_REG, + AYN_SENSOR_PWM_FAN_SPEED_2_REG, + AYN_SENSOR_PWM_FAN_SPEED_3_REG, + AYN_SENSOR_PWM_FAN_SPEED_4_REG, + AYN_SENSOR_PWM_FAN_SPEED_5_REG, + AYN_SENSOR_PWM_FAN_TEMP_1_REG, + AYN_SENSOR_PWM_FAN_TEMP_2_REG, + AYN_SENSOR_PWM_FAN_TEMP_3_REG, + AYN_SENSOR_PWM_FAN_TEMP_4_REG, + AYN_SENSOR_PWM_FAN_TEMP_5_REG, +}; + +struct ayn_device { + struct led_classdev *led_cdev; + u32 ayn_lock; /* ACPI EC Lock */ + u8 rgb_effect; +} drvdata; + +struct thermal_sensor { + char *name; + int reg; +}; + +static struct thermal_sensor thermal_sensors[] = { + { "Battery", AYN_SENSOR_BAT_TEMP_REG }, + { "Motherboard", AYN_SENSOR_MB_TEMP_REG }, + { "Charger IC", AYN_SENSOR_CHARGE_TEMP_REG }, + { "vCore", AYN_SENSOR_VCORE_TEMP_REG }, + { "CPU Core", AYN_SENSOR_PROC_TEMP_REG }, + {} +}; + +#define DEVICE_ATTR_RW_NAMED(_name, _attrname) \ + struct device_attribute dev_attr_##_name = { \ + .attr = { .name = _attrname, .mode = 0644 }, \ + .show = _name##_show, \ + .store = _name##_store, \ + } + +#define DEVICE_ATTR_RO_NAMED(_name, _attrname) \ + struct device_attribute dev_attr_##_name = { \ + .attr = { .name = _attrname, .mode = 0444 }, \ + .show = _name##_show, \ + } + +/* Handle ACPI lock mechanism */ +#define ACPI_LOCK_DELAY_MS 500 + +/* RGB effect values */ +enum RGB_EFFECT_OPTION { + BREATHE, + MONOCOLOR, +}; + +static const char *const RGB_EFFECT_TEXT[] = { + [BREATHE] = "breathe", + [MONOCOLOR] = "monocolor", +}; + +static bool lock_global_acpi_lock(void) +{ + return ACPI_SUCCESS(acpi_acquire_global_lock(ACPI_LOCK_DELAY_MS, + &drvdata.ayn_lock)); +} + +static bool unlock_global_acpi_lock(void) +{ + return ACPI_SUCCESS(acpi_release_global_lock(drvdata.ayn_lock)); +} + +/** + * read_from_ec() - Reads a value from the embedded controller. + * + * @reg: The register to start the read from. + * @size: The number of sequential registers the data is contained in. + * @val: Pointer to return the data with. + * + * Return: 0, or an error. + */ +static int read_from_ec(u8 reg, int size, long *val) +{ + int ret, i; + u8 buf; + + if (!lock_global_acpi_lock()) + return -EBUSY; + + *val = 0; + for (i = 0; i < size; i++) { + ret = ec_read(reg + i, &buf); + if (ret) + return ret; + *val <<= i * 8; + *val += buf; + } + + if (!unlock_global_acpi_lock()) + return -EBUSY; + + return 0; +} + +/** + * write_to_ec() - Writes a value to the embedded controller. + * + * @reg: The register to write to. + * @val: Value to write + * + * Return: 0, or an error. + */ +static int write_to_ec(u8 reg, u8 val) +{ + int ret; + + if (!lock_global_acpi_lock()) + return -EBUSY; + + pr_info("Writing EC value %d to register %u\n", val, reg); + ret = ec_write(reg, val); + + if (!unlock_global_acpi_lock()) + return -EBUSY; + + return ret; +} + +/** + * ayn_pwm_manual() - Enable manual control of the fan. + */ +static int ayn_pwm_manual(void) +{ + return write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x00); +} + +/** + * ayn_pwm_full() - Set fan to 100% speed. + */ +static int ayn_pwm_full(void) +{ + int ret; + + ret = write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x00); + if (ret) + return ret; + + return write_to_ec(AYN_SENSOR_PWM_FAN_SET_REG, 128); +} + +/** + * ayn_pwm_auto() - Enable automatic EC control of the fan. + */ +static int ayn_pwm_auto(void) +{ + return write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x01); +} + +/** + * ayn_pwm_ec_curve() - Enable manually setting the fan curve for automatic + * EC control of the fan. + */ +static int ayn_pwm_ec_curve(void) +{ + return write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x02); +} + +/** + * ayn_ec_hwmon_is_visible() - Determines RO or RW for hwmon attribute sysfs. + * + * @drvdata: Unused void pointer to context data. + * @type: The hwmon_sensor_types type. + * @attr: The attribute to set RO/RW on. + * @channel: HWMON subsystem usage flags for the attribute. + * + * Return: Permission level. + */ +static umode_t ayn_ec_hwmon_is_visible(const void *drvdata, + enum hwmon_sensor_types type, u32 attr, + int channel) +{ + switch (type) { + case hwmon_fan: + return 0444; + case hwmon_pwm: + return 0644; + default: + return 0; + } +} + +/** + * ayn_pwm_fan_read() - Read from a hwmon pwm or fan attribute. + * + * @dev: parent device of the given attribute. + * @type: The hwmon_sensor_types type. + * @attr: The attribute to read from. + * @channel: HWMON subsystem usage flags for the attribute. + * @val: Pointer to return the read value from. + * + * Return: 0, or an error. + */ +static int ayn_pwm_fan_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + int ret; + + switch (type) { + case hwmon_fan: + switch (attr) { + case hwmon_fan_input: + return read_from_ec(AYN_SENSOR_PWM_FAN_SPEED_REG, 2, + val); + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_enable: + ret = read_from_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 1, + val); + if (ret) + return ret; + + /* EC uses 0 for manual, 1 for automatic, 2 for user + * fan curve. Reflect hwmon usage instead. + */ + if (*val == 1) { + *val = 2; + return 0; + } + + if (*val == 2) { + *val = 3; + return 0; + } + + /* Return 0 when fan at max, otherwise 1 for manual. */ + ret = read_from_ec(AYN_SENSOR_PWM_FAN_SET_REG, 1, val); + if (ret) + return ret; + + if (*val == 128) + *val = 0; + else + *val = 1; + + return ret; + case hwmon_pwm_input: + ret = read_from_ec(AYN_SENSOR_PWM_FAN_SET_REG, 1, val); + if (ret) + return ret; + + *val = *val << 1; /* Max value is 128, scale to 255 */ + + return 0; + default: + break; + } + break; + default: + break; + } + return -EOPNOTSUPP; +} + +/** + * ayn_pwm_fan_write() - Write to a hwmon pwm attribute. + * + * @dev: parent device of the given attribute. + * @type: The hwmon_sensor_types type. + * @attr: The attribute to write to. + * @channel: HWMON subsystem usage flags for the attribute. + * @val: Value to write. + * + * Return: 0, or an error. + */ +static int ayn_pwm_fan_write(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long val) +{ + if (type != hwmon_pwm) + return -EOPNOTSUPP; + switch (attr) { + case hwmon_pwm_enable: + switch (val) { + case HWMON_PWM_FAN_MODE_FULL: + return ayn_pwm_full(); + case HWMON_PWM_FAN_MODE_MANUAL: + return ayn_pwm_manual(); + case HWMON_PWM_FAN_MODE_AUTO: + return ayn_pwm_auto(); + case HWMON_PWM_FAN_MODE_EC_CURVE: + return ayn_pwm_ec_curve(); + default: + return -EINVAL; + } + case hwmon_pwm_input: + if (val < 0 || val > 255) + return -EINVAL; + + val = val >> 1; /* Max value is 128, scale from 255 */ + + return write_to_ec(AYN_SENSOR_PWM_FAN_SET_REG, val); + default: + return -EOPNOTSUPP; + } +} + +static const struct hwmon_channel_info *ayn_ec_sensors[] = { + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT), + HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT | HWMON_PWM_ENABLE), + NULL, +}; + +static const struct hwmon_ops ayn_ec_hwmon_ops = { + .is_visible = ayn_ec_hwmon_is_visible, + .read = ayn_pwm_fan_read, + .write = ayn_pwm_fan_write, +}; + +static const struct hwmon_chip_info ayn_ec_chip_info = { + .ops = &ayn_ec_hwmon_ops, + .info = ayn_ec_sensors, +}; + +/** + * pwm_curve_store() - Write a fan curve speed or temperature value. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Input value string from sysfs write. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t pwm_curve_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + int i = to_sensor_dev_attr(attr)->index; + int ret, val; + u8 reg; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (i < 5) { + if (val < 0 || val > 255) + return -EINVAL; + val = val >> 1; /* Max EC value is 128, scale from 255 */ + } else + if (val < 0 || val > 100) + return -EINVAL; + + reg = ayn_pwm_curve_registers[i]; + + ret = write_to_ec(reg, val); + if (ret) + return ret; + + return count; +} + +/** + * pwm_curve_show() - Read a fan curve speed or temperature value. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Output buffer. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t pwm_curve_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int i = to_sensor_dev_attr(attr)->index; + long val; + int ret; + u8 reg; + + reg = ayn_pwm_curve_registers[i]; + + ret = read_from_ec(reg, 1, &val); + if (ret) + return ret; + + if (i < 5) + val = val << 1; /* Max EC value is 128, scale to 255 */ + + return sysfs_emit(buf, "%ld\n", val); +} + +/* Fan curve attributes */ +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point1_pwm, pwm_curve, 0); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point2_pwm, pwm_curve, 1); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point3_pwm, pwm_curve, 2); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point4_pwm, pwm_curve, 3); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point5_pwm, pwm_curve, 4); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point1_temp, pwm_curve, 5); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point2_temp, pwm_curve, 6); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point3_temp, pwm_curve, 7); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point4_temp, pwm_curve, 8); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point5_temp, pwm_curve, 9); + +/** + * thermal_sensor_show() - Read a thermal sensor attribute value. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Buffer to write the result into. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t thermal_sensor_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + long ret, val; + int i; + + i = to_sensor_dev_attr(attr)->index; + + ret = read_from_ec(thermal_sensors[i].reg, 1, &val); + if (ret) + return ret; + + val = val * 1000L; + + return sysfs_emit(buf, "%ld\n", val); +} + +/** + * thermal_sensor_label_show() - Read a thermal sensor attribute label. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Buffer to read to. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t thermal_sensor_label_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int i = to_sensor_dev_attr(attr)->index; + + return sysfs_emit(buf, "%s\n", thermal_sensors[i].name); +} + +static SENSOR_DEVICE_ATTR_RO(temp1_input, thermal_sensor, 0); +static SENSOR_DEVICE_ATTR_RO(temp2_input, thermal_sensor, 1); +static SENSOR_DEVICE_ATTR_RO(temp3_input, thermal_sensor, 2); +static SENSOR_DEVICE_ATTR_RO(temp4_input, thermal_sensor, 3); +static SENSOR_DEVICE_ATTR_RO(temp5_input, thermal_sensor, 4); +static SENSOR_DEVICE_ATTR_RO(temp1_label, thermal_sensor_label, 0); +static SENSOR_DEVICE_ATTR_RO(temp2_label, thermal_sensor_label, 1); +static SENSOR_DEVICE_ATTR_RO(temp3_label, thermal_sensor_label, 2); +static SENSOR_DEVICE_ATTR_RO(temp4_label, thermal_sensor_label, 3); +static SENSOR_DEVICE_ATTR_RO(temp5_label, thermal_sensor_label, 4); + +static struct attribute *ayn_sensors_attrs[] = { + &sensor_dev_attr_pwm1_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point1_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point2_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point3_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point4_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point5_temp.dev_attr.attr, + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_temp1_label.dev_attr.attr, + &sensor_dev_attr_temp2_input.dev_attr.attr, + &sensor_dev_attr_temp2_label.dev_attr.attr, + &sensor_dev_attr_temp3_input.dev_attr.attr, + &sensor_dev_attr_temp3_label.dev_attr.attr, + &sensor_dev_attr_temp4_input.dev_attr.attr, + &sensor_dev_attr_temp4_label.dev_attr.attr, + &sensor_dev_attr_temp5_input.dev_attr.attr, + &sensor_dev_attr_temp5_label.dev_attr.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(ayn_sensors); + +/** + * rgb_effect_write() - Set the RGB effect stored in drvdata.rgb_effect. + */ +static int rgb_effect_write(void) +{ + return write_to_ec(AYN_RGB_EFFECT_REG, drvdata.rgb_effect); +}; + +/** + * rgb_effect_read() - Read the RGB effect and store it in drvdata.rgb_effect. + */ +static int rgb_effect_read(void) +{ + int ret; + long effect; + + ret = read_from_ec(AYN_RGB_EFFECT_REG, 1, &effect); + if (ret) + return ret; + + switch (effect) { + case AYN_RGB_EFFECT_WRITE: + case AYN_RGB_EFFECT_MONOCOLOR: + drvdata.rgb_effect = AYN_RGB_EFFECT_WRITE; + break; + default: + drvdata.rgb_effect = AYN_RGB_EFFECT_BREATHE; + } + + return 0; +} + +/** + * rgb_effect_store() - Store the given RGB effect and set it. + * + * @dev: parent device of the given attribute. + * @attr: The attribute to write to. + * @buf: Input value string from sysfs write. + * @count: The number of bytes written. + * + * Return: The number of bytes written, or an error. + */ +static ssize_t rgb_effect_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + int ret; + + ret = sysfs_match_string(RGB_EFFECT_TEXT, buf); + if (ret < 0) + return ret; + + if (ret) + drvdata.rgb_effect = AYN_RGB_EFFECT_WRITE; + else + drvdata.rgb_effect = AYN_RGB_EFFECT_BREATHE; + + ret = rgb_effect_write(); + if (ret) + return ret; + + return count; +}; + +/** + * rgb_effect_show() - Read the current RGB effect. + * + * @dev: parent device of the given attribute. + * @attr: The attribute to read. + * @buf: Buffer to read to. + * + * Return: The number of bytes read, or an error. + */ +static ssize_t rgb_effect_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret, i; + + ret = rgb_effect_read(); + if (ret) + return ret; + + switch (drvdata.rgb_effect) { + case AYN_RGB_EFFECT_WRITE: + case AYN_RGB_EFFECT_MONOCOLOR: + i = MONOCOLOR; + break; + default: + i = BREATHE; + break; + } + + return sysfs_emit(buf, "%s\n", RGB_EFFECT_TEXT[i]); +}; + +static DEVICE_ATTR_RW_NAMED(rgb_effect, "effect"); + +/** + * rgb_effect_show() - Display the RGB effects available. + * + * @dev: parent device of the given attribute. + * @attr: The attribute to read. + * @buf: Buffer to read to. + * + * Return: The number of bytes read, or an error. + */ +static ssize_t rgb_effect_index_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + size_t count = 0; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(RGB_EFFECT_TEXT); i++) + count += sysfs_emit_at(buf, count, "%s ", RGB_EFFECT_TEXT[i]); + + buf[count - 1] = '\n'; + + return count; +} + +static DEVICE_ATTR_RO_NAMED(rgb_effect_index, "effect_index"); + +/** + * ayn_led_mc_brightness_set() - Write the brightness for the RGB LED. + * + * @led_cdev: Parent LED device for the led_classdev_mc. + * @brightness: Brightness value to write [0-255]. + */ +static void ayn_led_mc_brightness_set(struct led_classdev *led_cdev, + enum led_brightness brightness) +{ + struct led_classdev_mc *led_cdev_mc = lcdev_to_mccdev(led_cdev); + struct mc_subled s_led; + int i, ret, val; + + switch (drvdata.rgb_effect) { + case AYN_RGB_EFFECT_WRITE: + case AYN_RGB_EFFECT_MONOCOLOR: + break; + case AYN_RGB_EFFECT_BREATHE: + return; + } + + led_cdev->brightness = brightness; + for (i = 0; i < led_cdev_mc->num_colors; i++) { + s_led = led_cdev_mc->subled_info[i]; + val = brightness * s_led.intensity / led_cdev->max_brightness; + ret = write_to_ec(s_led.channel, val); + if (ret) { + dev_err(led_cdev->dev, + "Error setting brightness: %d\n", ret); + return; + } + } + + /* Must write mode again to change to set color */ + write_to_ec(AYN_RGB_EFFECT_REG, AYN_RGB_EFFECT_WRITE); +}; + +/** + * ayn_led_mc_brightness_get() - Get the brightness for the RGB LED. + * + * @led_cdev: Parent LED device for the led_classdev_mc. + * + * Return: Current brightness. + */ +static enum led_brightness ayn_led_mc_brightness_get(struct led_classdev *led_cdev) +{ + return led_cdev->brightness; +}; + +static struct attribute *ayn_led_mc_attrs[] = { + &dev_attr_rgb_effect.attr, + &dev_attr_rgb_effect_index.attr, + NULL, +}; + +static struct attribute_group ayn_led_mc_group = { + .attrs = ayn_led_mc_attrs, +}; + +struct mc_subled ayn_led_mc_subled_info[] = { + { + .color_index = LED_COLOR_ID_RED, + .brightness = 0, + .intensity = 0, + .channel = AYN_LED_MC_RED_REG, + }, + { + .color_index = LED_COLOR_ID_GREEN, + .brightness = 0, + .intensity = 0, + .channel = AYN_LED_MC_GREEN_REG, + }, + { + .color_index = LED_COLOR_ID_BLUE, + .brightness = 0, + .intensity = 0, + .channel = AYN_LED_MC_BLUE_REG, + }, +}; + +struct led_classdev_mc ayn_led_mc = { + .led_cdev = { + .name = "go:rgb:joystick_rings", + .brightness = 0, + .max_brightness = 255, + .brightness_set = ayn_led_mc_brightness_set, + .brightness_get = ayn_led_mc_brightness_get, + .color = LED_COLOR_ID_RGB, + }, + .num_colors = ARRAY_SIZE(ayn_led_mc_subled_info), + .subled_info = ayn_led_mc_subled_info, +}; + +static int ayn_ec_resume(struct platform_device *pdev) +{ + struct led_classdev *led_cdev = drvdata.led_cdev; + int ret; + + ret = rgb_effect_write(); + if (ret) + return ret; + + ayn_led_mc_brightness_set(led_cdev, led_cdev->brightness); + + return 0; +} + +static int ayn_ec_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *hwdev; + int ret; + + ret = devm_led_classdev_multicolor_register(dev, &ayn_led_mc); + if (ret) + return ret; + + ret = devm_device_add_group(ayn_led_mc.led_cdev.dev, &ayn_led_mc_group); + if (ret) + return ret; + + drvdata.led_cdev = &ayn_led_mc.led_cdev; + ret = rgb_effect_read(); + if (ret) + return ret; + + hwdev = devm_hwmon_device_register_with_info(dev, "aynec", NULL, + &ayn_ec_chip_info, + ayn_sensors_groups); + return PTR_ERR_OR_ZERO(hwdev); +} + +static struct platform_driver ayn_ec_driver = { + .driver = { + .name = "ayn-ec", + }, + .probe = ayn_ec_probe, + .resume = ayn_ec_resume, +}; + +static struct platform_device *ayn_ec_device; + +static int __init ayn_ec_init(void) +{ + ayn_ec_device = platform_create_bundle(&ayn_ec_driver, ayn_ec_probe, + NULL, 0, NULL, 0); + + return PTR_ERR_OR_ZERO(ayn_ec_device); +} + +static void __exit ayn_ec_exit(void) +{ + platform_device_unregister(ayn_ec_device); + platform_driver_unregister(&ayn_ec_driver); +} + +static const struct dmi_system_id ayn_dmi_table[] = { + { + .ident = "AYN Loki Max", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ayn"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Loki Max"), + }, + }, + { + .ident = "AYN Loki MiniPro", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ayn"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Loki MiniPro"), + }, + }, + { + .ident = "AYN Loki Zero", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ayn"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Loki Zero"), + }, + }, + { + .ident = "Tectoy Zeenix Lite", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Tectoy"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Zeenix Lite"), + }, + }, + {}, +}; + +MODULE_DEVICE_TABLE(dmi, ayn_dmi_table); + +module_init(ayn_ec_init); +module_exit(ayn_ec_exit); + +MODULE_AUTHOR("Derek J. Clark "); +MODULE_DESCRIPTION("Platform driver that handles EC sensors of AYN x86 devices"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/x86/bitland-mifs-wmi.c b/drivers/platform/x86/bitland-mifs-wmi.c index b0d06a80e89ef..3a373184519da 100644 --- a/drivers/platform/x86/bitland-mifs-wmi.c +++ b/drivers/platform/x86/bitland-mifs-wmi.c @@ -300,6 +300,10 @@ static int bitland_mifs_wmi_suspend(struct device *dev) enum platform_profile_option profile; int ret; + /* Skip event device */ + if (!data->pp_dev) + return 0; + ret = laptop_profile_get(data->pp_dev, &profile); if (ret == 0) data->saved_profile = profile; @@ -311,6 +315,10 @@ static int bitland_mifs_wmi_resume(struct device *dev) { struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev); + /* Skip event device */ + if (!data->pp_dev) + return 0; + dev_dbg(dev, "Resuming, restoring profile %d\n", data->saved_profile); return laptop_profile_set(dev, data->saved_profile); } diff --git a/drivers/platform/x86/classmate-laptop.c b/drivers/platform/x86/classmate-laptop.c index e6eed3d65580a..a5fe34211afb9 100644 --- a/drivers/platform/x86/classmate-laptop.c +++ b/drivers/platform/x86/classmate-laptop.c @@ -400,7 +400,7 @@ static int cmpc_accel_add_v4(struct acpi_device *acpi) struct input_dev *inputdev; struct cmpc_accel *accel; - accel = kmalloc_obj(*accel); + accel = devm_kzalloc(&acpi->dev, sizeof(*accel), GFP_KERNEL); if (!accel) return -ENOMEM; @@ -411,7 +411,7 @@ static int cmpc_accel_add_v4(struct acpi_device *acpi) error = device_create_file(&acpi->dev, &cmpc_accel_sensitivity_attr_v4); if (error) - goto failed_sensitivity; + return error; accel->g_select = CMPC_ACCEL_G_SELECT_DEFAULT; cmpc_accel_set_g_select_v4(acpi->handle, accel->g_select); @@ -434,8 +434,6 @@ static int cmpc_accel_add_v4(struct acpi_device *acpi) device_remove_file(&acpi->dev, &cmpc_accel_g_select_attr_v4); failed_g_select: device_remove_file(&acpi->dev, &cmpc_accel_sensitivity_attr_v4); -failed_sensitivity: - kfree(accel); return error; } @@ -650,7 +648,7 @@ static int cmpc_accel_add(struct acpi_device *acpi) struct input_dev *inputdev; struct cmpc_accel *accel; - accel = kmalloc_obj(*accel); + accel = devm_kzalloc(&acpi->dev, sizeof(*accel), GFP_KERNEL); if (!accel) return -ENOMEM; @@ -659,7 +657,7 @@ static int cmpc_accel_add(struct acpi_device *acpi) error = device_create_file(&acpi->dev, &cmpc_accel_sensitivity_attr); if (error) - goto failed_file; + return error; error = cmpc_add_acpi_notify_device(acpi, "cmpc_accel", cmpc_accel_idev_init); @@ -673,8 +671,6 @@ static int cmpc_accel_add(struct acpi_device *acpi) failed_input: device_remove_file(&acpi->dev, &cmpc_accel_sensitivity_attr); -failed_file: - kfree(accel); return error; } diff --git a/drivers/platform/x86/dell/dell-laptop.c b/drivers/platform/x86/dell/dell-laptop.c index 57748c3ea24fc..053f40572bf6e 100644 --- a/drivers/platform/x86/dell/dell-laptop.c +++ b/drivers/platform/x86/dell/dell-laptop.c @@ -2551,7 +2551,12 @@ static int __init dell_init(void) if (mute_led_registered) led_classdev_unregister(&mute_led_cdev); fail_led: + dell_laptop_unregister_notifier(&dell_laptop_notifier); + debugfs_remove_recursive(dell_laptop_dir); dell_battery_exit(); + kbd_led_exit(); + if (quirks && quirks->touchpad_led) + touchpad_led_exit(); dell_cleanup_rfkill(); fail_rfkill: platform_device_del(platform_device); diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index f63bc00d9a9b3..37937d8a059bf 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -249,6 +249,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C9C") }, .driver_data = (void *)&victus_s_thermal_params, }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D26") }, + .driver_data = (void *)&omen_v1_legacy_thermal_params, + }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") }, .driver_data = (void *)&omen_v1_no_ec_thermal_params, @@ -257,6 +261,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D87") }, .driver_data = (void *)&omen_v1_no_ec_thermal_params, }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8E35") }, + .driver_data = (void *)&omen_v1_legacy_thermal_params, + }, {}, }; diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c index 085093506dda9..6b4bac5f7051f 100644 --- a/drivers/platform/x86/intel/hid.c +++ b/drivers/platform/x86/intel/hid.c @@ -7,11 +7,13 @@ */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -230,6 +232,7 @@ static const struct dmi_system_id dmi_auto_add_switch[] = { }; struct intel_hid_priv { + struct mutex mutex; /* Avoid notify_handler() racing with itself */ struct input_dev *input_dev; struct input_dev *array; struct input_dev *switches; @@ -565,6 +568,8 @@ static void notify_handler(acpi_handle handle, u32 event, void *context) struct key_entry *ke; int err; + guard(mutex)(&priv->mutex); + /* * Some convertible have unreliable VGBS return which could cause incorrect * SW_TABLET_MODE report, in these cases we enable support when receiving @@ -720,6 +725,10 @@ static int intel_hid_probe(struct platform_device *device) return -ENOMEM; dev_set_drvdata(&device->dev, priv); + err = devm_mutex_init(&device->dev, &priv->mutex); + if (err) + return err; + /* See dual_accel_detect.h for more info on the dual_accel check. */ if (enable_sw_tablet_mode == TABLET_SW_AUTO) { if (dmi_check_system(dmi_vgbs_allow_list)) diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index b804cb753f942..24334ae70d825 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -1804,7 +1804,7 @@ void tpmi_sst_dev_resume(struct auxiliary_device *auxdev) if (!(pd_info->sst_header.cap_mask & SST_PP_CAP_PP_ENABLE)) continue; - writeq(pd_info->saved_pp_control, power_domain_info->sst_base + + writeq(pd_info->saved_pp_control, pd_info->sst_base + pd_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET); } } diff --git a/drivers/platform/x86/intel/vsec.c b/drivers/platform/x86/intel/vsec.c index 18e4a892bf0f0..439c0c8ac896c 100644 --- a/drivers/platform/x86/intel/vsec.c +++ b/drivers/platform/x86/intel/vsec.c @@ -488,10 +488,25 @@ static int intel_vsec_walk_header(struct device *dev, const struct intel_vsec_platform_info *info) { struct intel_vsec_header **header = info->headers; + u64 base_addr; int ret; for ( ; *header; header++) { - ret = intel_vsec_register_device(dev, *header, info, info->base_addr); + if (info->base_addr) { + base_addr = info->base_addr; + } else { + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) { + dev_err(dev, "non-PCI device without a base address\n"); + return -EINVAL; + } + + pdev = to_pci_dev(dev); + base_addr = pci_resource_start(pdev, (*header)->tbir); + } + + ret = intel_vsec_register_device(dev, *header, info, base_addr); if (ret) return ret; } diff --git a/drivers/platform/x86/intel/vsec_tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c index 16fd7aa41f20e..88f14d0ad4102 100644 --- a/drivers/platform/x86/intel/vsec_tpmi.c +++ b/drivers/platform/x86/intel/vsec_tpmi.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -473,7 +474,7 @@ static ssize_t mem_write(struct file *file, const char __user *userbuf, size_t l struct seq_file *m = file->private_data; struct intel_tpmi_pm_feature *pfs = m->private; u32 addr, value, punit, size; - u32 num_elems, *array; + u32 num_elems; void __iomem *mem; int ret; @@ -481,15 +482,14 @@ static ssize_t mem_write(struct file *file, const char __user *userbuf, size_t l if (!size) return -EIO; + u32 *array __free(kfree) = NULL; ret = parse_int_array_user(userbuf, len, (int **)&array); if (ret < 0) return ret; num_elems = *array; - if (num_elems != 3) { - ret = -EINVAL; - goto exit_write; - } + if (num_elems != 3) + return -EINVAL; punit = array[1]; addr = array[2]; @@ -498,15 +498,11 @@ static ssize_t mem_write(struct file *file, const char __user *userbuf, size_t l if (!IS_ALIGNED(addr, sizeof(u32))) return -EINVAL; - if (punit >= pfs->pfs_header.num_entries) { - ret = -EINVAL; - goto exit_write; - } + if (punit >= pfs->pfs_header.num_entries) + return -EINVAL; - if (addr >= size) { - ret = -EINVAL; - goto exit_write; - } + if (addr >= size) + return -EINVAL; mutex_lock(&tpmi_dev_lock); @@ -525,9 +521,6 @@ static ssize_t mem_write(struct file *file, const char __user *userbuf, size_t l unlock_mem_write: mutex_unlock(&tpmi_dev_lock); -exit_write: - kfree(array); - return ret; } diff --git a/drivers/platform/x86/xo15-ebook.c b/drivers/platform/x86/xo15-ebook.c index 4d1b1b310cc56..1568169b78727 100644 --- a/drivers/platform/x86/xo15-ebook.c +++ b/drivers/platform/x86/xo15-ebook.c @@ -38,6 +38,7 @@ MODULE_DEVICE_TABLE(acpi, ebook_device_ids); struct ebook_switch { struct input_dev *input; char phys[32]; /* for input device */ + bool gpe_enabled; }; static int ebook_send_state(struct acpi_device *device) @@ -128,7 +129,7 @@ static int ebook_switch_add(struct acpi_device *device) /* Button's GPE is run-wake GPE */ acpi_enable_gpe(device->wakeup.gpe_device, device->wakeup.gpe_number); - device_set_wakeup_enable(&device->dev, true); + button->gpe_enabled = true; } return 0; @@ -144,6 +145,10 @@ static void ebook_switch_remove(struct acpi_device *device) { struct ebook_switch *button = acpi_driver_data(device); + if (button->gpe_enabled) + acpi_disable_gpe(device->wakeup.gpe_device, + device->wakeup.gpe_number); + input_unregister_device(button->input); kfree(button); } diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c index 19e992d2ee3b8..99d100e1d923f 100644 --- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c @@ -54,6 +54,15 @@ struct imx8m_blk_ctrl_domain_data { * register. */ u32 mipi_phy_rst_mask; + + /* + * VC8000E reset de-assertion edge and AXI clock may have a timing issue. + * Workaround: Set bit2 (vc8000e_clk_en) of BLK_CLK_EN_CSR to 0 to gate off + * both AXI clock and VC8000E clock sent to VC8000E and AXI clock sent to + * VPU_NOC m_v_2 interface during VC8000E power up(VC8000E reset is + * de-asserted by HW) + */ + bool is_errata_err050531; }; #define DOMAIN_MAX_CLKS 4 @@ -108,7 +117,11 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd) dev_err(bc->dev, "failed to enable clocks\n"); goto bus_put; } - regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask); + + if (data->is_errata_err050531) + regmap_clear_bits(bc->regmap, BLK_CLK_EN, data->clk_mask); + else + regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask); /* power up upstream GPC domain */ ret = pm_runtime_get_sync(domain->power_dev); @@ -117,6 +130,9 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd) goto clk_disable; } + if (data->is_errata_err050531) + regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask); + /* wait for reset to propagate */ udelay(5); @@ -511,12 +527,38 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_vpu_blk_ctl_domain_data[] .clk_mask = BIT(2), .path_names = (const char *[]){"vc8000e"}, .num_paths = 1, + .is_errata_err050531 = true, }, }; +static int imx8mp_vpu_power_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct imx8m_blk_ctrl *bc = container_of(nb, struct imx8m_blk_ctrl, + power_nb); + + if (action == GENPD_NOTIFY_ON) { + /* + * On power up we have no software backchannel to the GPC to + * wait for the ADB handshake to happen, so we just delay for a + * bit. On power down the GPC driver waits for the handshake. + */ + + udelay(5); + + /* set "fuse" bits to enable the VPUs */ + regmap_set_bits(bc->regmap, 0x8, 0xffffffff); + regmap_set_bits(bc->regmap, 0xc, 0xffffffff); + regmap_set_bits(bc->regmap, 0x10, 0xffffffff); + regmap_set_bits(bc->regmap, 0x14, 0xffffffff); + } + + return NOTIFY_OK; +} + static const struct imx8m_blk_ctrl_data imx8mp_vpu_blk_ctl_dev_data = { .max_reg = 0x18, - .power_notifier_fn = imx8mm_vpu_power_notifier, + .power_notifier_fn = imx8mp_vpu_power_notifier, .domains = imx8mp_vpu_blk_ctl_domain_data, .num_domains = ARRAY_SIZE(imx8mp_vpu_blk_ctl_domain_data), }; diff --git a/drivers/pmdomain/imx/imx93-blk-ctrl.c b/drivers/pmdomain/imx/imx93-blk-ctrl.c index 1afc78b034fad..243ce939ba680 100644 --- a/drivers/pmdomain/imx/imx93-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx93-blk-ctrl.c @@ -48,6 +48,8 @@ #define PRIO(X) (X) +#define BLK_CTRL_NO_PARENT UINT_MAX + struct imx93_blk_ctrl_domain; struct imx93_blk_ctrl { @@ -68,12 +70,18 @@ struct imx93_blk_ctrl_qos { u32 cfg_prio; }; +struct imx93_blk_ctrl_subdomain_link { + struct generic_pm_domain *parent; + struct generic_pm_domain *subdomain; +}; + struct imx93_blk_ctrl_domain_data { const char *name; const char * const *clk_names; int num_clks; u32 rst_mask; u32 clk_mask; + u32 parent; int num_qos; struct imx93_blk_ctrl_qos qos[DOMAIN_MAX_QOS]; }; @@ -203,6 +211,13 @@ static void imx93_release_pm_genpd(void *data) pm_genpd_remove(genpd); } +static void imx93_release_subdomain(void *data) +{ + struct imx93_blk_ctrl_subdomain_link *link = data; + + pm_genpd_remove_subdomain(link->parent, link->subdomain); +} + static struct lock_class_key blk_ctrl_genpd_lock_class; static int imx93_blk_ctrl_probe(struct platform_device *pdev) @@ -302,6 +317,34 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev) bc->onecell_data.domains[i] = &domain->genpd; } + for (i = 0; i < bc_data->num_domains; i++) { + struct imx93_blk_ctrl_domain *domain = &bc->domains[i]; + const struct imx93_blk_ctrl_domain_data *data = domain->data; + struct imx93_blk_ctrl_subdomain_link *link; + + if (bc_data->skip_mask & BIT(i) || + data->parent == BLK_CTRL_NO_PARENT) + continue; + + link = devm_kzalloc(dev, sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; + + link->parent = &bc->domains[data->parent].genpd; + link->subdomain = &domain->genpd; + + ret = pm_genpd_add_subdomain(&bc->domains[data->parent].genpd, + &domain->genpd); + if (ret) + return dev_err_probe(dev, ret, "failed to add subdomain %s\n", + domain->genpd.name); + + ret = devm_add_action_or_reset(dev, imx93_release_subdomain, link); + if (ret) + return dev_err_probe(dev, ret, + "failed to add subdomain release callback\n"); + } + ret = devm_pm_runtime_enable(dev); if (ret) return dev_err_probe(dev, ret, "failed to enable pm-runtime\n"); @@ -326,8 +369,9 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .name = "mediablk-mipi-dsi", .clk_names = (const char *[]){ "dsi" }, .num_clks = 1, - .rst_mask = BIT(11) | BIT(12), - .clk_mask = BIT(11) | BIT(12), + .rst_mask = BIT(11), + .clk_mask = BIT(11), + .parent = IMX93_MEDIABLK_PD_MIPI_PHY, }, [IMX93_MEDIABLK_PD_MIPI_CSI] = { .name = "mediablk-mipi-csi", @@ -335,6 +379,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 2, .rst_mask = BIT(9) | BIT(10), .clk_mask = BIT(9) | BIT(10), + .parent = IMX93_MEDIABLK_PD_MIPI_PHY, }, [IMX93_MEDIABLK_PD_PXP] = { .name = "mediablk-pxp", @@ -342,6 +387,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 1, .rst_mask = BIT(7) | BIT(8), .clk_mask = BIT(7) | BIT(8), + .parent = BLK_CTRL_NO_PARENT, .num_qos = 2, .qos = { { @@ -363,6 +409,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 2, .rst_mask = BIT(4) | BIT(5) | BIT(6), .clk_mask = BIT(4) | BIT(5) | BIT(6), + .parent = BLK_CTRL_NO_PARENT, .num_qos = 1, .qos = { { @@ -379,6 +426,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 1, .rst_mask = BIT(2) | BIT(3), .clk_mask = BIT(2) | BIT(3), + .parent = BLK_CTRL_NO_PARENT, .num_qos = 4, .qos = { { @@ -404,6 +452,14 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] } } }, + [IMX93_MEDIABLK_PD_MIPI_PHY] = { + .name = "mediablk-mipi-phy", + .clk_names = NULL, + .num_clks = 0, + .rst_mask = BIT(12), + .clk_mask = BIT(12), + .parent = BLK_CTRL_NO_PARENT, + }, }; static const struct regmap_range imx93_media_blk_ctl_yes_ranges[] = { diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c index e1cfd42234734..f0a6339affd75 100644 --- a/drivers/pmdomain/mediatek/mtk-pm-domains.c +++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c @@ -393,9 +393,8 @@ static int scpsys_hwv_power_on(struct generic_pm_domain *genpd) return ret; }; -static int scpsys_hwv_power_off(struct generic_pm_domain *genpd) +static int scpsys_hwv_power_off_internal(struct scpsys_domain *pd) { - struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd); const struct scpsys_hwv_domain_data *hwv = pd->hwv_data; struct scpsys *scpsys = pd->scpsys; u32 val; @@ -464,6 +463,13 @@ static int scpsys_hwv_power_off(struct generic_pm_domain *genpd) return ret; }; +static int scpsys_hwv_power_off(struct generic_pm_domain *genpd) +{ + struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd); + + return scpsys_hwv_power_off_internal(pd); +} + static int scpsys_ctl_pwrseq_on(struct scpsys_domain *pd) { struct scpsys *scpsys = pd->scpsys; @@ -694,9 +700,8 @@ static int scpsys_power_on(struct generic_pm_domain *genpd) return ret; } -static int scpsys_power_off(struct generic_pm_domain *genpd) +static int scpsys_power_off_internal(struct scpsys_domain *pd) { - struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd); struct scpsys *scpsys = pd->scpsys; bool tmp; int ret; @@ -737,6 +742,13 @@ static int scpsys_power_off(struct generic_pm_domain *genpd) return 0; } +static int scpsys_power_off(struct generic_pm_domain *genpd) +{ + struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd); + + return scpsys_power_off_internal(pd); +} + static struct generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_node *node) { @@ -884,7 +896,14 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no * late_init time. */ if (MTK_SCPD_CAPS(pd, MTK_SCPD_KEEP_DEFAULT_OFF)) { - if (scpsys_domain_is_on(pd)) + bool domain_is_on; + + if (scpsys->soc_data->type == SCPSYS_MTCMOS_TYPE_HW_VOTER) + domain_is_on = scpsys_hwv_domain_is_enable_done(pd); + else + domain_is_on = scpsys_domain_is_on(pd); + + if (domain_is_on) dev_warn(scpsys->dev, "%pOF: A default off power domain has been ON\n", node); } else { @@ -973,6 +992,7 @@ static int scpsys_add_subdomain(struct scpsys *scpsys, struct device_node *paren static void scpsys_remove_one_domain(struct scpsys_domain *pd) { + struct scpsys *scpsys = pd->scpsys; int ret; /* @@ -984,8 +1004,14 @@ static void scpsys_remove_one_domain(struct scpsys_domain *pd) dev_err(pd->scpsys->dev, "failed to remove domain '%s' : %d - state may be inconsistent\n", pd->genpd.name, ret); - if (scpsys_domain_is_on(pd)) - scpsys_power_off(&pd->genpd); + + if (scpsys->soc_data->type == SCPSYS_MTCMOS_TYPE_HW_VOTER) { + if (scpsys_hwv_domain_is_enable_done(pd)) + scpsys_hwv_power_off_internal(pd); + } else { + if (scpsys_domain_is_on(pd)) + scpsys_power_off_internal(pd); + } clk_bulk_put(pd->num_clks, pd->clks); clk_bulk_put(pd->num_subsys_clks, pd->subsys_clks); diff --git a/drivers/power/reset/linkstation-poweroff.c b/drivers/power/reset/linkstation-poweroff.c index 02f5fdb8ffc4c..e56d75bfcc43f 100644 --- a/drivers/power/reset/linkstation-poweroff.c +++ b/drivers/power/reset/linkstation-poweroff.c @@ -163,10 +163,10 @@ static int __init linkstation_poweroff_init(void) dn = of_find_matching_node(NULL, ls_poweroff_of_match); if (!dn) return -ENODEV; - of_node_put(dn); match = of_match_node(ls_poweroff_of_match, dn); cfg = match->data; + of_node_put(dn); dn = of_find_node_by_name(NULL, cfg->mdio_node_name); if (!dn) diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c index 4dff71be11b60..c12aa494698a6 100644 --- a/drivers/power/sequencing/core.c +++ b/drivers/power/sequencing/core.c @@ -543,15 +543,18 @@ void pwrseq_device_unregister(struct pwrseq_device *pwrseq) struct device *dev = &pwrseq->dev; struct pwrseq_target *target; - scoped_guard(mutex, &pwrseq->state_lock) { + scoped_guard(rwsem_write, &pwrseq_sem) { guard(rwsem_write)(&pwrseq->rw_lock); + /* + * Holding rw_lock for write excludes all power on/off callers + * (they hold it for read), so it's safe to read enable_count + * here without taking the state_lock. + */ list_for_each_entry(target, &pwrseq->targets, list) WARN(target->unit->enable_count, "REMOVING POWER SEQUENCER WITH ACTIVE USERS\n"); - guard(rwsem_write)(&pwrseq_sem); - device_del(dev); } @@ -989,8 +992,9 @@ static void *pwrseq_debugfs_seq_start(struct seq_file *seq, loff_t *pos) ctx.index = *pos; /* - * We're holding the lock for the entire printout so no need to fiddle - * with device reference count. + * Hold the lock for the entire printout to prevent device removal. + * Reference counts are managed by start()/next()/stop() as required + * by the seq_file contract. */ down_read(&pwrseq_sem); @@ -998,7 +1002,7 @@ static void *pwrseq_debugfs_seq_start(struct seq_file *seq, loff_t *pos) if (!ctx.index) return NULL; - return ctx.dev; + return get_device(ctx.dev); } static void *pwrseq_debugfs_seq_next(struct seq_file *seq, void *data, @@ -1008,8 +1012,9 @@ static void *pwrseq_debugfs_seq_next(struct seq_file *seq, void *data, ++*pos; - struct device *next __free(put_device) = - bus_find_next_device(&pwrseq_bus, curr); + struct device *next = bus_find_next_device(&pwrseq_bus, curr); + + put_device(curr); return next; } @@ -1058,6 +1063,8 @@ static int pwrseq_debugfs_seq_show(struct seq_file *seq, void *data) static void pwrseq_debugfs_seq_stop(struct seq_file *seq, void *data) { + if (data) + put_device(data); up_read(&pwrseq_sem); } diff --git a/drivers/power/supply/bq257xx_charger.c b/drivers/power/supply/bq257xx_charger.c index 02c7d8b61e82b..7ca4ae6109027 100644 --- a/drivers/power/supply/bq257xx_charger.c +++ b/drivers/power/supply/bq257xx_charger.c @@ -128,9 +128,8 @@ static int bq25703_get_min_vsys(struct bq257xx_chg *pdata, int *intval) * @vsys: voltage value to set in uV. * * This function takes a requested minimum system voltage value, clamps - * it between the minimum supported value by the charger and a user - * defined minimum system value, and then writes the value to the - * appropriate register. + * it between the user defined minimum system value and the maximum supported + * value by the charger, and then writes the value to the appropriate register. * * Return: Returns 0 on success or error if an error occurs. */ @@ -139,7 +138,7 @@ static int bq25703_set_min_vsys(struct bq257xx_chg *pdata, int vsys) unsigned int reg; int vsys_min = pdata->vsys_min; - vsys = clamp(vsys, BQ25703_MINVSYS_MIN_UV, vsys_min); + vsys = clamp(vsys, vsys_min, BQ25703_MINVSYS_MAX_UV); reg = ((vsys - BQ25703_MINVSYS_MIN_UV) / BQ25703_MINVSYS_STEP_UV); reg = FIELD_PREP(BQ25703_MINVSYS_MASK, reg); diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c index c49e0e4d02f77..c3644018b6bb4 100644 --- a/drivers/power/supply/charger-manager.c +++ b/drivers/power/supply/charger-manager.c @@ -303,8 +303,10 @@ static bool is_full_charged(struct charger_manager *cm) if (cm->battery_status == POWER_SUPPLY_STATUS_FULL && desc->fullbatt_vchkdrop_uV) uV += desc->fullbatt_vchkdrop_uV; - if (uV >= desc->fullbatt_uV) - return true; + if (uV >= desc->fullbatt_uV) { + is_full = true; + goto out; + } } } diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index 7b7bdce3162fc..59c741993ef8d 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -439,10 +439,13 @@ static void cpcap_battery_detect_battery_type(struct cpcap_battery_ddata *ddata) if (IS_ERR_OR_NULL(nvmem)) { ddata->check_nvmem = true; dev_info_once(ddata->dev, "Can not find battery nvmem device. Assuming generic lipo battery\n"); - } else if (nvmem_device_read(nvmem, 2, 1, &battery_id) < 0) { - battery_id = 0; - ddata->check_nvmem = true; - dev_warn(ddata->dev, "Can not read battery nvmem device. Assuming generic lipo battery\n"); + } else { + if (nvmem_device_read(nvmem, 2, 1, &battery_id) < 0) { + battery_id = 0; + ddata->check_nvmem = true; + dev_warn(ddata->dev, "Can not read battery nvmem device. Assuming generic lipo battery\n"); + } + nvmem_device_put(nvmem); } switch (battery_id) { diff --git a/drivers/power/supply/max17042_battery.c b/drivers/power/supply/max17042_battery.c index 167fb3fb37320..75ed9374a51fb 100644 --- a/drivers/power/supply/max17042_battery.c +++ b/drivers/power/supply/max17042_battery.c @@ -1254,7 +1254,8 @@ static int max17042_platform_probe(struct platform_device *pdev) if (!i2c) return -EINVAL; - dev->of_node = dev->parent->of_node; + device_set_of_node_from_dev(dev, dev->parent); + id = platform_get_device_id(pdev); irq = platform_get_irq(pdev, 0); diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index a446d3d086fcd..2532e221b2e19 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -292,18 +292,13 @@ static int power_supply_check_supplies(struct power_supply *psy) if (cnt == 1) return 0; - /* All supplies found, allocate char ** array for filling */ - psy->supplied_from = devm_kzalloc(&psy->dev, sizeof(*psy->supplied_from), + /* All supplies found, allocate char * array for filling */ + psy->supplied_from = devm_kcalloc(&psy->dev, + cnt - 1, sizeof(*psy->supplied_from), GFP_KERNEL); if (!psy->supplied_from) return -ENOMEM; - *psy->supplied_from = devm_kcalloc(&psy->dev, - cnt - 1, sizeof(**psy->supplied_from), - GFP_KERNEL); - if (!*psy->supplied_from) - return -ENOMEM; - return power_supply_populate_supplied_from(psy); } #else diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c index 3d34cdc4a3a51..c8b801fcb5251 100644 --- a/drivers/pwm/pwm-imx27.c +++ b/drivers/pwm/pwm-imx27.c @@ -200,7 +200,7 @@ static void pwm_imx27_wait_fifo_slot(struct pwm_chip *chip, static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { - unsigned long period_cycles, duty_cycles, prescale, period_us, tmp; + unsigned long period_cycles, duty_cycles, prescale, period_us; struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); unsigned long long c; unsigned long long clkrate; @@ -208,6 +208,7 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, int val; int ret; u32 cr; + u64 tmp; clkrate = clk_get_rate(imx->clks[PWM_IMX27_PER].clk); c = clkrate * state->period; @@ -249,6 +250,11 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, val = readl(imx->mmio_base + MX3_PWMPR); val = val >= MX3_PWMPR_MAX ? MX3_PWMPR_MAX : val; cr = readl(imx->mmio_base + MX3_PWMCR); + + /* + * tmp stores period in nanoseconds. Result fits in u64 since + * val <= 0xfffe and prescaler in [1, 0x1000]. + */ tmp = NSEC_PER_SEC * (u64)(val + 2) * MX3_PWMCR_PRESCALER_GET(cr); tmp = DIV_ROUND_UP_ULL(tmp, clkrate); period_us = DIV_ROUND_UP_ULL(tmp, 1000); diff --git a/drivers/pwm/pwm-rzg2l-gpt.c b/drivers/pwm/pwm-rzg2l-gpt.c index 4856af080e8e9..dfa1d11a48a82 100644 --- a/drivers/pwm/pwm-rzg2l-gpt.c +++ b/drivers/pwm/pwm-rzg2l-gpt.c @@ -81,7 +81,7 @@ struct rzg2l_gpt_chip { void __iomem *mmio; struct mutex lock; /* lock to protect shared channel resources */ unsigned long rate_khz; - u32 period_ticks[RZG2L_MAX_HW_CHANNELS]; + u64 period_ticks[RZG2L_MAX_HW_CHANNELS]; u32 channel_request_count[RZG2L_MAX_HW_CHANNELS]; u32 channel_enable_count[RZG2L_MAX_HW_CHANNELS]; }; @@ -408,14 +408,14 @@ static int rzg2l_gpt_probe(struct platform_device *pdev) rate = clk_get_rate(clk); if (!rate) - return dev_err_probe(dev, -EINVAL, "The gpt clk rate is 0"); + return dev_err_probe(dev, -EINVAL, "The gpt clk rate is 0\n"); /* * Refuse clk rates > 1 GHz to prevent overflow later for computing * period and duty cycle. */ if (rate > NSEC_PER_SEC) - return dev_err_probe(dev, -EINVAL, "The gpt clk rate is > 1GHz"); + return dev_err_probe(dev, -EINVAL, "The gpt clk rate is > 1GHz\n"); /* * Rate is in MHz and is always integer for peripheral clk @@ -424,7 +424,7 @@ static int rzg2l_gpt_probe(struct platform_device *pdev) */ rzg2l_gpt->rate_khz = rate / KILO; if (rzg2l_gpt->rate_khz * KILO != rate) - return dev_err_probe(dev, -EINVAL, "Rate is not multiple of 1000"); + return dev_err_probe(dev, -EINVAL, "Rate is not multiple of 1000\n"); mutex_init(&rzg2l_gpt->lock); diff --git a/drivers/rapidio/devices/tsi721.c b/drivers/rapidio/devices/tsi721.c index 66331e67cf4ef..71b87bf8c31d8 100644 --- a/drivers/rapidio/devices/tsi721.c +++ b/drivers/rapidio/devices/tsi721.c @@ -394,7 +394,6 @@ static void tsi721_db_dpc(struct work_struct *work) idb_work); struct rio_mport *mport; struct rio_dbell *dbell; - int found = 0; u32 wr_ptr, rd_ptr; u64 *idb_entry; u32 regval; @@ -412,6 +411,8 @@ static void tsi721_db_dpc(struct work_struct *work) rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE; while (wr_ptr != rd_ptr) { + int found = 0; + idb_entry = (u64 *)(priv->idb_base + (TSI721_IDB_ENTRY_SIZE * rd_ptr)); rd_ptr++; diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index dc5d67767336e..1797929dfe566 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -248,7 +248,7 @@ static void regulator_lock_two(struct regulator_dev *rdev1, ret = regulator_lock_nested(rdev1, ww_ctx); WARN_ON(ret); ret = regulator_lock_nested(rdev2, ww_ctx); - if (ret != -EDEADLOCK) { + if (ret != -EDEADLK) { WARN_ON(ret); goto exit; } @@ -264,7 +264,7 @@ static void regulator_lock_two(struct regulator_dev *rdev1, swap(held, contended); ret = regulator_lock_nested(contended, ww_ctx); - if (ret != -EDEADLOCK) { + if (ret != -EDEADLK) { WARN_ON(ret); break; } diff --git a/drivers/regulator/ltc3676.c b/drivers/regulator/ltc3676.c index 73d511eb1c1dc..768cdce0d4ecf 100644 --- a/drivers/regulator/ltc3676.c +++ b/drivers/regulator/ltc3676.c @@ -45,11 +45,11 @@ #define LTC3676_DVBxA_REF_SELECT BIT(5) #define LTC3676_DVBxB_PGOOD_MASK BIT(5) -#define LTC3676_IRQSTAT_PGOOD_TIMEOUT BIT(3) -#define LTC3676_IRQSTAT_UNDERVOLT_WARN BIT(4) -#define LTC3676_IRQSTAT_UNDERVOLT_FAULT BIT(5) -#define LTC3676_IRQSTAT_THERMAL_WARN BIT(6) -#define LTC3676_IRQSTAT_THERMAL_FAULT BIT(7) +#define LTC3676_IRQSTAT_PGOOD_TIMEOUT BIT(2) +#define LTC3676_IRQSTAT_UNDERVOLT_WARN BIT(3) +#define LTC3676_IRQSTAT_UNDERVOLT_FAULT BIT(4) +#define LTC3676_IRQSTAT_THERMAL_WARN BIT(5) +#define LTC3676_IRQSTAT_THERMAL_FAULT BIT(6) enum ltc3676_reg { LTC3676_SW1, diff --git a/drivers/regulator/scmi-regulator.c b/drivers/regulator/scmi-regulator.c index 6d609c42e4793..c005e65ba0ecb 100644 --- a/drivers/regulator/scmi-regulator.c +++ b/drivers/regulator/scmi-regulator.c @@ -345,8 +345,10 @@ static int scmi_regulator_probe(struct scmi_device *sdev) for_each_child_of_node_scoped(np, child) { ret = process_scmi_regulator_of_node(sdev, ph, child, rinfo); /* abort on any mem issue */ - if (ret == -ENOMEM) + if (ret == -ENOMEM) { + of_node_put(np); return ret; + } } of_node_put(np); /* diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c index fd2b6824ad265..e1a955476c9b3 100644 --- a/drivers/remoteproc/qcom_common.c +++ b/drivers/remoteproc/qcom_common.c @@ -109,6 +109,7 @@ static int qcom_add_minidump_segments(struct rproc *rproc, struct minidump_subsy struct minidump_region __iomem *ptr; struct minidump_region region; int seg_cnt, i; + int ret = 0; dma_addr_t da; size_t size; char *name; @@ -129,17 +130,22 @@ static int qcom_add_minidump_segments(struct rproc *rproc, struct minidump_subsy if (le32_to_cpu(region.valid) == MINIDUMP_REGION_VALID) { name = kstrndup(region.name, MAX_REGION_NAME_LENGTH - 1, GFP_KERNEL); if (!name) { - iounmap(ptr); - return -ENOMEM; + ret = -ENOMEM; + break; } da = le64_to_cpu(region.address); size = le64_to_cpu(region.size); - rproc_coredump_add_custom_segment(rproc, da, size, rproc_dumpfn_t, name); + ret = rproc_coredump_add_custom_segment(rproc, da, size, rproc_dumpfn_t, + name); + if (ret) { + kfree(name); + break; + } } } iounmap(ptr); - return 0; + return ret; } void qcom_minidump(struct rproc *rproc, unsigned int minidump_id, diff --git a/drivers/remoteproc/qcom_q6v5_wcss.c b/drivers/remoteproc/qcom_q6v5_wcss.c index c27200159a88a..b391724cfd08e 100644 --- a/drivers/remoteproc/qcom_q6v5_wcss.c +++ b/drivers/remoteproc/qcom_q6v5_wcss.c @@ -96,7 +96,6 @@ struct wcss_data { unsigned int crash_reason_smem; u32 version; bool aon_reset_required; - bool wcss_q6_reset_required; const char *ssr_name; const char *sysmon_name; int ssctl_id; @@ -134,7 +133,6 @@ struct q6v5_wcss { struct reset_control *wcss_aon_reset; struct reset_control *wcss_reset; struct reset_control *wcss_q6_reset; - struct reset_control *wcss_q6_bcr_reset; struct qcom_q6v5 q6v5; @@ -309,7 +307,7 @@ static int q6v5_wcss_qcs404_power_on(struct q6v5_wcss *wcss) return ret; /* Remove reset to the WCNSS QDSP6SS */ - reset_control_deassert(wcss->wcss_q6_bcr_reset); + reset_control_deassert(wcss->wcss_q6_reset); /* Enable Q6SSTOP_AHBFABRIC_CBCR clock */ ret = clk_prepare_enable(wcss->ahbfabric_cbcr_clk); @@ -803,19 +801,10 @@ static int q6v5_wcss_init_reset(struct q6v5_wcss *wcss, return PTR_ERR(wcss->wcss_reset); } - if (desc->wcss_q6_reset_required) { - wcss->wcss_q6_reset = devm_reset_control_get_exclusive(dev, "wcss_q6_reset"); - if (IS_ERR(wcss->wcss_q6_reset)) { - dev_err(wcss->dev, "unable to acquire wcss_q6_reset\n"); - return PTR_ERR(wcss->wcss_q6_reset); - } - } - - wcss->wcss_q6_bcr_reset = devm_reset_control_get_optional_exclusive(dev, - "wcss_q6_bcr_reset"); - if (IS_ERR(wcss->wcss_q6_bcr_reset)) { - dev_err(wcss->dev, "unable to acquire wcss_q6_bcr_reset\n"); - return PTR_ERR(wcss->wcss_q6_bcr_reset); + wcss->wcss_q6_reset = devm_reset_control_get_exclusive(dev, "wcss_q6_reset"); + if (IS_ERR(wcss->wcss_q6_reset)) { + dev_err(wcss->dev, "unable to acquire wcss_q6_reset\n"); + return PTR_ERR(wcss->wcss_q6_reset); } return 0; @@ -1062,7 +1051,6 @@ static const struct wcss_data wcss_ipq8074_res_init = { .firmware_name = "IPQ8074/q6_fw.mdt", .crash_reason_smem = WCSS_CRASH_REASON, .aon_reset_required = true, - .wcss_q6_reset_required = true, .ops = &q6v5_wcss_ipq8074_ops, .requires_force_stop = true, }; @@ -1072,7 +1060,6 @@ static const struct wcss_data wcss_qcs404_res_init = { .firmware_name = "wcnss.mdt", .version = WCSS_QCS404, .aon_reset_required = false, - .wcss_q6_reset_required = false, .ssr_name = "mpss", .sysmon_name = "wcnss", .ssctl_id = 0x12, diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c index 50a9974f3202e..45a62cb980725 100644 --- a/drivers/remoteproc/xlnx_r5_remoteproc.c +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c @@ -948,16 +948,6 @@ static struct zynqmp_r5_core *zynqmp_r5_add_rproc_core(struct device *cdev) goto free_rproc; } - /* - * If firmware is already available in the memory then move rproc state - * to DETACHED. Firmware can be preloaded via debugger or by any other - * agent (processors) in the system. - * If firmware isn't available in the memory and resource table isn't - * found, then rproc state remains OFFLINE. - */ - if (!zynqmp_r5_get_rsc_table_va(r5_core)) - r5_rproc->state = RPROC_DETACHED; - r5_core->rproc = r5_rproc; return r5_core; @@ -1210,6 +1200,7 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, { struct device *dev = cluster->dev; struct zynqmp_r5_core *r5_core; + u32 req, usage, status; int ret = -EINVAL, i; r5_core = cluster->r5_cores[0]; @@ -1255,6 +1246,41 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, ret = zynqmp_r5_get_sram_banks(r5_core); if (ret) return ret; + + /* + * It is possible that firmware is loaded into the memory, but + * RPU (remote) is not running. In such case, RPU state will be + * moved to RPROC_DETACHED wrongfully. To avoid it first make + * sure RPU is power-on and out of reset before parsing for the + * resource table. + */ + ret = zynqmp_pm_get_rpu_node_status(r5_core->pm_domain_id, + &status, &req, &usage); + if (ret) { + dev_warn(r5_core->dev, + "failed to get rpu node status, err %d\n", ret); + continue; + } + + /* + * If RPU state is power on and out of reset i.e. running, then + * assign RPROC_DETACHED state. If the RPU is not out of reset + * then do not attempt to attach to the remote processor. + */ + if (status == PM_NODE_RUNNING) { + /* + * Not all the firmware that is running on the remote + * core is expected to have the resource table. The + * firmware might not use RPMsg at all, and in that case + * resource table becomes irrelevant. However, we still + * need to make sure that running core is not reported + * as offline. so do not decide remote core state based + * on the resource table availability + */ + if (zynqmp_r5_get_rsc_table_va(r5_core)) + dev_dbg(r5_core->dev, "rsc tbl not found\n"); + r5_core->rproc->state = RPROC_DETACHED; + } } return 0; diff --git a/drivers/reset/reset-imx7.c b/drivers/reset/reset-imx7.c index dd01fe11c5cb3..a3cb8244d76a6 100644 --- a/drivers/reset/reset-imx7.c +++ b/drivers/reset/reset-imx7.c @@ -236,6 +236,12 @@ static int imx8mq_reset_set(struct reset_controller_dev *rcdev, case IMX8MQ_RESET_PCIE_CTRL_APPS_EN: case IMX8MQ_RESET_PCIE2_CTRL_APPS_EN: + case IMX8MQ_RESET_MIPI_CSI1_CORE_RESET: + case IMX8MQ_RESET_MIPI_CSI1_PHY_REF_RESET: + case IMX8MQ_RESET_MIPI_CSI1_ESC_RESET: + case IMX8MQ_RESET_MIPI_CSI2_CORE_RESET: + case IMX8MQ_RESET_MIPI_CSI2_PHY_REF_RESET: + case IMX8MQ_RESET_MIPI_CSI2_ESC_RESET: case IMX8MQ_RESET_MIPI_DSI_PCLK_RESET_N: case IMX8MQ_RESET_MIPI_DSI_ESC_RESET_N: case IMX8MQ_RESET_MIPI_DSI_DPI_RESET_N: diff --git a/drivers/reset/reset-sunxi.c b/drivers/reset/reset-sunxi.c index 2544de6576e44..2f6df7707cad1 100644 --- a/drivers/reset/reset-sunxi.c +++ b/drivers/reset/reset-sunxi.c @@ -44,7 +44,7 @@ static int sunxi_reset_init(struct device_node *np) data->membase = ioremap(res.start, size); if (!data->membase) { ret = -ENOMEM; - goto err_alloc; + goto err_mem_region; } spin_lock_init(&data->lock); @@ -57,6 +57,8 @@ static int sunxi_reset_init(struct device_node *np) return reset_controller_register(&data->rcdev); +err_mem_region: + release_mem_region(res.start, size); err_alloc: kfree(data); return ret; diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index 401a4ece0c977..d9d4468e4cbdf 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -1626,7 +1626,6 @@ static void qcom_glink_rpdev_release(struct device *dev) { struct rpmsg_device *rpdev = to_rpmsg_device(dev); - kfree(rpdev->driver_override); kfree(rpdev); } @@ -1862,7 +1861,6 @@ static void qcom_glink_device_release(struct device *dev) /* Release qcom_glink_alloc_channel() reference */ kref_put(&channel->refcount, qcom_glink_channel_release); - kfree(rpdev->driver_override); kfree(rpdev); } diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c index ca9cf8858a5ef..bff5aefee2126 100644 --- a/drivers/rpmsg/rpmsg_char.c +++ b/drivers/rpmsg/rpmsg_char.c @@ -104,6 +104,9 @@ static int rpmsg_ept_cb(struct rpmsg_device *rpdev, void *buf, int len, struct rpmsg_eptdev *eptdev = priv; struct sk_buff *skb; + if (!eptdev) + return 0; + skb = alloc_skb(len, GFP_ATOMIC); if (!skb) return -ENOMEM; @@ -124,6 +127,9 @@ static int rpmsg_ept_flow_cb(struct rpmsg_device *rpdev, void *priv, bool enable { struct rpmsg_eptdev *eptdev = priv; + if (!eptdev) + return 0; + eptdev->remote_flow_restricted = enable; eptdev->remote_flow_updated = true; @@ -490,6 +496,7 @@ static int rpmsg_chrdev_probe(struct rpmsg_device *rpdev) struct rpmsg_channel_info chinfo; struct rpmsg_eptdev *eptdev; struct device *dev = &rpdev->dev; + int ret; memcpy(chinfo.name, rpdev->id.name, RPMSG_NAME_SIZE); chinfo.src = rpdev->src; @@ -502,13 +509,17 @@ static int rpmsg_chrdev_probe(struct rpmsg_device *rpdev) /* Set the default_ept to the rpmsg device endpoint */ eptdev->default_ept = rpdev->ept; + ret = rpmsg_chrdev_eptdev_add(eptdev, chinfo); + + if (ret) + return ret; /* * The rpmsg_ept_cb uses *priv parameter to get its rpmsg_eptdev context. - * Storedit in default_ept *priv field. + * Stored it in default_ept *priv field. */ eptdev->default_ept->priv = eptdev; - return rpmsg_chrdev_eptdev_add(eptdev, chinfo); + return 0; } static void rpmsg_chrdev_remove(struct rpmsg_device *rpdev) diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index e7f7831d37f89..c56f69c22e420 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -358,33 +358,6 @@ rpmsg_show_attr(src, src, "0x%x\n"); rpmsg_show_attr(dst, dst, "0x%x\n"); rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n"); -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); - int ret; - - ret = driver_set_override(dev, &rpdev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} - -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", rpdev->driver_override); - device_unlock(dev); - return len; -} -static DEVICE_ATTR_RW(driver_override); - static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -405,7 +378,6 @@ static struct attribute *rpmsg_dev_attrs[] = { &dev_attr_dst.attr, &dev_attr_src.attr, &dev_attr_announce.attr, - &dev_attr_driver_override.attr, NULL, }; ATTRIBUTE_GROUPS(rpmsg_dev); @@ -424,9 +396,11 @@ static int rpmsg_dev_match(struct device *dev, const struct device_driver *drv) const struct rpmsg_driver *rpdrv = to_rpmsg_driver(drv); const struct rpmsg_device_id *ids = rpdrv->id_table; unsigned int i; + int ret; - if (rpdev->driver_override) - return !strcmp(rpdev->driver_override, drv->name); + ret = device_match_driver_override(dev, drv); + if (ret >= 0) + return ret; if (ids) for (i = 0; ids[i].name[0]; i++) @@ -535,6 +509,7 @@ static const struct bus_type rpmsg_bus = { .name = "rpmsg", .match = rpmsg_dev_match, .dev_groups = rpmsg_dev_groups, + .driver_override = true, .uevent = rpmsg_uevent, .probe = rpmsg_dev_probe, .remove = rpmsg_dev_remove, @@ -560,11 +535,9 @@ int rpmsg_register_device_override(struct rpmsg_device *rpdev, device_initialize(dev); if (driver_override) { - ret = driver_set_override(dev, &rpdev->driver_override, - driver_override, - strlen(driver_override)); + ret = device_set_driver_override(dev, driver_override); if (ret) { - dev_err(dev, "device_set_override failed: %d\n", ret); + dev_err(dev, "device_set_driver_override() failed: %d\n", ret); put_device(dev); return ret; } @@ -573,8 +546,6 @@ int rpmsg_register_device_override(struct rpmsg_device *rpdev, ret = device_add(dev); if (ret) { dev_err(dev, "device_add failed: %d\n", ret); - kfree(rpdev->driver_override); - rpdev->driver_override = NULL; put_device(dev); } diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index 5ae15111fb4f1..1b8bb05924af7 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -374,7 +374,6 @@ static void virtio_rpmsg_release_device(struct device *dev) struct rpmsg_device *rpdev = to_rpmsg_device(dev); struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); - kfree(rpdev->driver_override); kfree(vch); } diff --git a/drivers/rtc/rtc-abx80x.c b/drivers/rtc/rtc-abx80x.c index 00d7de64ed3e7..008a70baa69fe 100644 --- a/drivers/rtc/rtc-abx80x.c +++ b/drivers/rtc/rtc-abx80x.c @@ -545,7 +545,8 @@ static int abx80x_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) status &= ~ABX8XX_STATUS_BLF; - tmp = i2c_smbus_write_byte_data(client, ABX8XX_REG_STATUS, 0); + tmp = i2c_smbus_write_byte_data(client, ABX8XX_REG_STATUS, + status); if (tmp < 0) return tmp; diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index f89ab58f5048a..fa04ece151b8b 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -934,6 +934,7 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq) unsigned char rtc_control; unsigned address_space; u32 flags = 0; + bool hpet_registered = false; struct nvmem_config nvmem_cfg = { .name = "cmos_nvram", .word_size = 1, @@ -1091,6 +1092,7 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq) " failed in rtc_init()."); goto cleanup1; } + hpet_registered = true; } else rtc_cmos_int_handler = cmos_interrupt; @@ -1140,6 +1142,10 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq) if (is_valid_irq(rtc_irq)) free_irq(rtc_irq, cmos_rtc.rtc); cleanup1: + if (hpet_registered) { + hpet_mask_rtc_irq_bit(RTC_IRQMASK); + hpet_unregister_irq_handler(cmos_interrupt); + } cmos_rtc.dev = NULL; cleanup0: if (RTC_IOMAPPED) diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c index 7205c59ff7294..9ba5958ca7a6e 100644 --- a/drivers/rtc/rtc-ds1307.c +++ b/drivers/rtc/rtc-ds1307.c @@ -269,6 +269,16 @@ static int ds1307_get_time(struct device *dev, struct rtc_time *t) if (tmp & DS1338_BIT_OSF) return -EINVAL; break; + case ds_1337: + case ds_1339: + case ds_1341: + case ds_3231: + ret = regmap_read(ds1307->regmap, DS1337_REG_STATUS, &tmp); + if (ret) + return ret; + if (tmp & DS1337_BIT_OSF) + return -EINVAL; + break; case ds_1340: if (tmp & DS1340_BIT_nEOSC) return -EINVAL; @@ -279,13 +289,6 @@ static int ds1307_get_time(struct device *dev, struct rtc_time *t) if (tmp & DS1340_BIT_OSF) return -EINVAL; break; - case ds_1341: - ret = regmap_read(ds1307->regmap, DS1337_REG_STATUS, &tmp); - if (ret) - return ret; - if (tmp & DS1337_BIT_OSF) - return -EINVAL; - break; case ds_1388: ret = regmap_read(ds1307->regmap, DS1388_REG_FLAG, &tmp); if (ret) @@ -308,7 +311,7 @@ static int ds1307_get_time(struct device *dev, struct rtc_time *t) t->tm_hour = bcd2bin(tmp); /* rx8130 is bit position, not BCD */ if (ds1307->type == rx_8130) - t->tm_wday = fls(regs[DS1307_REG_WDAY] & 0x7f); + t->tm_wday = fls(regs[DS1307_REG_WDAY] & 0x7f) - 1; else t->tm_wday = bcd2bin(regs[DS1307_REG_WDAY] & 0x07) - 1; t->tm_mday = bcd2bin(regs[DS1307_REG_MDAY] & 0x3f); @@ -380,14 +383,17 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t) regmap_update_bits(ds1307->regmap, DS1307_REG_CONTROL, DS1338_BIT_OSF, 0); break; - case ds_1340: - regmap_update_bits(ds1307->regmap, DS1340_REG_FLAG, - DS1340_BIT_OSF, 0); - break; + case ds_1337: + case ds_1339: case ds_1341: + case ds_3231: regmap_update_bits(ds1307->regmap, DS1337_REG_STATUS, DS1337_BIT_OSF, 0); break; + case ds_1340: + regmap_update_bits(ds1307->regmap, DS1340_REG_FLAG, + DS1340_BIT_OSF, 0); + break; case ds_1388: regmap_update_bits(ds1307->regmap, DS1388_REG_FLAG, DS1388_BIT_OSF, 0); diff --git a/drivers/rtc/rtc-isl1208.c b/drivers/rtc/rtc-isl1208.c index f71a6bb77b2a1..a0df53991fbdb 100644 --- a/drivers/rtc/rtc-isl1208.c +++ b/drivers/rtc/rtc-isl1208.c @@ -822,6 +822,11 @@ static const struct nvmem_config isl1208_nvmem_config = { .reg_write = isl1208_nvmem_write, }; +static void isl1208_disable_irq_wake_action(void *data) +{ + disable_irq_wake((unsigned long)data); +} + static int isl1208_setup_irq(struct i2c_client *client, int irq) { int rc = devm_request_threaded_irq(&client->dev, irq, NULL, @@ -831,7 +836,15 @@ static int isl1208_setup_irq(struct i2c_client *client, int irq) client); if (!rc) { device_init_wakeup(&client->dev, true); - enable_irq_wake(irq); + rc = enable_irq_wake(irq); + if (rc) + return rc; + + rc = devm_add_action_or_reset(&client->dev, + isl1208_disable_irq_wake_action, + (void *)(unsigned long)irq); + if (rc) + return rc; } else { dev_err(&client->dev, "Unable to request irq %d, no alarm support\n", diff --git a/drivers/rtc/rtc-mpfs.c b/drivers/rtc/rtc-mpfs.c index 6aa3eae575d2a..ece6de4a6adbd 100644 --- a/drivers/rtc/rtc-mpfs.c +++ b/drivers/rtc/rtc-mpfs.c @@ -112,7 +112,7 @@ static int mpfs_rtc_settime(struct device *dev, struct rtc_time *tm) ctrl |= CONTROL_UPLOAD_BIT; writel(ctrl, rtcdev->base + CONTROL_REG); - ret = read_poll_timeout(readl, prog, prog & CONTROL_UPLOAD_BIT, 0, UPLOAD_TIMEOUT_US, + ret = read_poll_timeout(readl, prog, !(prog & CONTROL_UPLOAD_BIT), 0, UPLOAD_TIMEOUT_US, false, rtcdev->base + CONTROL_REG); if (ret) { dev_err(dev, "timed out uploading time to rtc"); diff --git a/drivers/rtc/rtc-msc313.c b/drivers/rtc/rtc-msc313.c index 8d7737e0e2e02..6ef9c4efd7c92 100644 --- a/drivers/rtc/rtc-msc313.c +++ b/drivers/rtc/rtc-msc313.c @@ -160,7 +160,7 @@ static const struct rtc_class_ops msc313_rtc_ops = { static irqreturn_t msc313_rtc_interrupt(s32 irq, void *dev_id) { - struct msc313_rtc *priv = dev_get_drvdata(dev_id); + struct msc313_rtc *priv = dev_id; u16 reg; reg = readw(priv->rtc_base + REG_RTC_STATUS_INT); @@ -206,7 +206,7 @@ static int msc313_rtc_probe(struct platform_device *pdev) priv->rtc_dev->range_max = U32_MAX; ret = devm_request_irq(dev, irq, msc313_rtc_interrupt, IRQF_SHARED, - dev_name(&pdev->dev), &pdev->dev); + dev_name(&pdev->dev), priv); if (ret) { dev_err(dev, "Could not request IRQ\n"); return ret; diff --git a/drivers/rtc/rtc-renesas-rtca3.c b/drivers/rtc/rtc-renesas-rtca3.c index cbabaa4dc96a5..2dc080d0eb6cd 100644 --- a/drivers/rtc/rtc-renesas-rtca3.c +++ b/drivers/rtc/rtc-renesas-rtca3.c @@ -455,7 +455,7 @@ static int rtca3_set_alarm(struct device *dev, struct rtc_wkalrm *wkalrm) * specified timeout for setup. */ writeb(rcr1 & ~RTCA3_RCR1_PIE, priv->base + RTCA3_RCR1); - readb_poll_timeout_atomic(priv->base + RTCA3_RCR1, tmp, !(tmp & ~RTCA3_RCR1_PIE), + readb_poll_timeout_atomic(priv->base + RTCA3_RCR1, tmp, !(tmp & RTCA3_RCR1_PIE), 10, RTCA3_DEFAULT_TIMEOUT_US); atomic_set(&priv->alrm_sstep, RTCA3_ALRM_SSTEP_DONE); } diff --git a/drivers/s390/char/monwriter.c b/drivers/s390/char/monwriter.c index eaeb4a6384d1f..ecf121a87f88f 100644 --- a/drivers/s390/char/monwriter.c +++ b/drivers/s390/char/monwriter.c @@ -122,6 +122,9 @@ static int monwrite_new_hdr(struct mon_private *monpriv) kfree(monbuf->data); kfree(monbuf); monbuf = NULL; + } else if (monbuf->hdr.datalen != monhdr->datalen) { + /* Data with buffer reuse must not change its length */ + return -EINVAL; } } else if (monhdr->mon_function != MONWRITE_STOP_INTERVAL) { if (mon_buf_count >= mon_max_bufs) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index d6b595eb33709..5d8f63f390a88 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -327,16 +327,14 @@ static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp) { struct pkey_verifyprotk kvp; struct protaeskeytoken *t; - u32 keytype; u8 *tmpbuf; int rc; if (copy_from_user(&kvp, uvp, sizeof(kvp))) return -EFAULT; - keytype = pkey_aes_bitsize_to_keytype(8 * kvp.protkey.len); - if (!keytype) { - PKEY_DBF_ERR("%s unknown/unsupported protkey length %u\n", + if (kvp.protkey.len > sizeof(kvp.protkey.protkey)) { + PKEY_DBF_ERR("%s protkey length %u exceeds protkey buffer size\n", __func__, kvp.protkey.len); memzero_explicit(&kvp, sizeof(kvp)); return -EINVAL; @@ -351,7 +349,7 @@ static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp) t = (struct protaeskeytoken *)tmpbuf; t->type = TOKTYPE_NON_CCA; t->version = TOKVER_PROTECTED_KEY; - t->keytype = keytype; + t->keytype = kvp.protkey.type; t->len = kvp.protkey.len; memcpy(t->protkey, kvp.protkey.protkey, kvp.protkey.len); diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c index ea774ab891806..0cc015cc9f6bd 100644 --- a/drivers/s390/crypto/pkey_pckmo.c +++ b/drivers/s390/crypto/pkey_pckmo.c @@ -257,6 +257,10 @@ static int pckmo_key2protkey(const u8 *key, u32 keylen, goto out; break; } + if (t->len > *protkeylen) { + rc = -EINVAL; + goto out; + } memcpy(protkey, t->protkey, t->len); *protkeylen = t->len; *protkeytype = t->keytype; diff --git a/drivers/s390/crypto/zcrypt_cex2a.c b/drivers/s390/crypto/zcrypt_cex2a.c deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/drivers/s390/crypto/zcrypt_cex2a.h b/drivers/s390/crypto/zcrypt_cex2a.h deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/drivers/s390/crypto/zcrypt_cex2c.c b/drivers/s390/crypto/zcrypt_cex2c.c deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/drivers/s390/crypto/zcrypt_cex2c.h b/drivers/s390/crypto/zcrypt_cex2c.h deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/drivers/scsi/elx/efct/efct_hw.c b/drivers/scsi/elx/efct/efct_hw.c index 1838032f64862..b79c6a7ea7912 100644 --- a/drivers/scsi/elx/efct/efct_hw.c +++ b/drivers/scsi/elx/efct/efct_hw.c @@ -1997,6 +1997,8 @@ efct_hw_io_abort(struct efct_hw *hw, struct efct_hw_io *io_to_abort, wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_abort, io_to_abort); if (!wqcb) { efc_log_err(hw->os, "can't allocate request tag\n"); + io_to_abort->abort_in_progress = false; + kref_put(&io_to_abort->ref, io_to_abort->release); return -ENOSPC; } diff --git a/drivers/scsi/elx/efct/efct_unsol.c b/drivers/scsi/elx/efct/efct_unsol.c index e6addab66a603..6a871a59c9096 100644 --- a/drivers/scsi/elx/efct/efct_unsol.c +++ b/drivers/scsi/elx/efct/efct_unsol.c @@ -385,6 +385,7 @@ efct_dispatch_fcp_cmd(struct efct_node *node, struct efc_hw_sequence *seq) if (cmnd->fc_flags & FCP_CFL_LEN_MASK) { efc_log_err(efct, "Additional CDB not supported\n"); + efct_scsi_io_free(io); return -EIO; } /* diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index 14d563e82d208..213d5b5dea94f 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -2977,7 +2977,7 @@ static int sdev_configure_v3_hw(struct scsi_device *sdev, return 0; if (!device_link_add(&sdev->sdev_gendev, dev, - DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)) { + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)) { if (pm_runtime_enabled(dev)) { dev_info(dev, "add device link failed, disable runtime PM for the host\n"); pm_runtime_disable(dev); @@ -2987,6 +2987,15 @@ static int sdev_configure_v3_hw(struct scsi_device *sdev, return 0; } +static void hisi_sas_sdev_destroy(struct scsi_device *sdev) +{ + struct Scsi_Host *shost = dev_to_shost(&sdev->sdev_gendev); + struct hisi_hba *hisi_hba = shost_priv(shost); + struct device *dev = hisi_hba->dev; + + device_link_remove(&sdev->sdev_gendev, dev); +} + static struct attribute *host_v3_hw_attrs[] = { &dev_attr_phy_event_threshold.attr, &dev_attr_intr_conv_v3_hw.attr, @@ -3401,6 +3410,7 @@ static const struct scsi_host_template sht_v3_hw = { .sg_tablesize = HISI_SAS_SGE_PAGE_CNT, .sg_prot_tablesize = HISI_SAS_SGE_PAGE_CNT, .sdev_init = hisi_sas_sdev_init, + .sdev_destroy = hisi_sas_sdev_destroy, .shost_groups = host_v3_hw_groups, .sdev_groups = sdev_groups_v3_hw, .tag_alloc_policy_rr = true, diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index a1b116cd4723d..8edad1830abe4 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -5017,6 +5017,10 @@ static int hpsa_scsi_ioaccel2_queue_command(struct ctlr_info *h, if (phys_disk->in_reset) { cmd->result = DID_RESET << 16; + atomic_dec(&phys_disk->ioaccel_cmds_out); + scsi_dma_unmap(cmd); + if (use_sg > h->ioaccel_maxsg) + hpsa_unmap_ioaccel2_sg_chain_block(h, cp); return -1; } diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 968a25235a2d7..f90ac2aea3f50 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -8189,6 +8189,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) mempool_free(mboxq, phba->mbox_mem_pool); goto out_free_bsmbx; } + mempool_free(mboxq, phba->mbox_mem_pool); /* * 1 for cmd, 1 for rsp, NVME adds an extra one @@ -8311,8 +8312,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) goto out_free_sg_dma_buf; } - mempool_free(mboxq, phba->mbox_mem_pool); - /* Verify OAS is supported */ lpfc_sli4_oas_verify(phba); diff --git a/drivers/scsi/pm8001/pm8001_ctl.c b/drivers/scsi/pm8001/pm8001_ctl.c index cbfda8c04e956..c10854ec44c7b 100644 --- a/drivers/scsi/pm8001/pm8001_ctl.c +++ b/drivers/scsi/pm8001/pm8001_ctl.c @@ -588,10 +588,7 @@ static DEVICE_ATTR(fatal_log, S_IRUGO, pm8001_ctl_fatal_log_show, NULL); static ssize_t non_fatal_log_show(struct device *cdev, struct device_attribute *attr, char *buf) { - u32 count; - - count = pm80xx_get_non_fatal_dump(cdev, attr, buf); - return count; + return pm80xx_get_non_fatal_dump(cdev, attr, buf); } static DEVICE_ATTR_RO(non_fatal_log); diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index ef22a4228b855..b118ed0bf53f8 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -1910,7 +1910,7 @@ int scsi_scan_host_selected(struct Scsi_Host *shost, unsigned int channel, return 0; } -EXPORT_SYMBOL(scsi_scan_host_selected); + static void scsi_sysfs_add_devices(struct Scsi_Host *shost) { struct scsi_device *sdev; diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index 13412702188e4..d689b9ed08a6c 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -40,8 +41,6 @@ #include #include "scsi_sas_internal.h" -#include "scsi_priv.h" - struct sas_host_attrs { struct list_head rphy_list; struct mutex lock; @@ -222,12 +221,45 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy) * SAS host attributes */ +/* + * Set shost->opt_sectors from the DMA optimal mapping size, but only + * when dma_opt_mapping_size() is strictly less than dma_max_mapping_size(), + * indicating a genuine optimization hint from an IOMMU or DMA backend. + * When the two are equal (e.g. IOMMU disabled / passthrough), no real + * hint exists, so leave opt_sectors at 0 to avoid bogus optimal_io_size + * values that break filesystem geometry (e.g. mkfs.xfs stripe alignment). + */ +static void sas_dma_setup_opt_sectors(struct Scsi_Host *shost) +{ + struct device *dma_dev = shost->dma_dev; + size_t opt = dma_opt_mapping_size(dma_dev); + size_t max = dma_max_mapping_size(dma_dev); + unsigned int opt_sectors; + + /* opt >= max means no real hint was provided by the DMA layer */ + if (opt >= max) + return; + + /* Clamp to max_sectors to avoid overflow in sector arithmetic */ + opt_sectors = min_t(unsigned int, opt >> SECTOR_SHIFT, + shost->max_sectors); + + /* Guard against zero before rounddown_pow_of_two() */ + if (!opt_sectors) + return; + + /* + * Round down to power-of-two so filesystem geometry calculations + * (e.g. XFS stripe width/unit) always produce clean divisors. + */ + shost->opt_sectors = rounddown_pow_of_two(opt_sectors); +} + static int sas_host_setup(struct transport_container *tc, struct device *dev, struct device *cdev) { struct Scsi_Host *shost = dev_to_shost(dev); struct sas_host_attrs *sas_host = to_sas_host_attrs(shost); - struct device *dma_dev = shost->dma_dev; INIT_LIST_HEAD(&sas_host->rphy_list); mutex_init(&sas_host->lock); @@ -239,10 +271,7 @@ static int sas_host_setup(struct transport_container *tc, struct device *dev, dev_printk(KERN_ERR, dev, "fail to a bsg device %d\n", shost->host_no); - if (dma_dev->dma_mask) { - shost->opt_sectors = min_t(unsigned int, shost->max_sectors, - dma_opt_mapping_size(dma_dev) >> SECTOR_SHIFT); - } + sas_dma_setup_opt_sectors(shost); return 0; } @@ -1685,22 +1714,6 @@ int scsi_is_sas_rphy(const struct device *dev) } EXPORT_SYMBOL(scsi_is_sas_rphy); -static void scan_channel_zero(struct Scsi_Host *shost, uint id, u64 lun) -{ - struct sas_host_attrs *sas_host = to_sas_host_attrs(shost); - struct sas_rphy *rphy; - - list_for_each_entry(rphy, &sas_host->rphy_list, list) { - if (rphy->identify.device_type != SAS_END_DEVICE || - rphy->scsi_target_id == -1) - continue; - - if (id == SCAN_WILD_CARD || id == rphy->scsi_target_id) { - scsi_scan_target(&rphy->dev, 0, rphy->scsi_target_id, - lun, SCSI_SCAN_MANUAL); - } - } -} /* * SCSI scan helper @@ -1710,41 +1723,23 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel, uint id, u64 lun) { struct sas_host_attrs *sas_host = to_sas_host_attrs(shost); - int res = 0; - int i; - - switch (channel) { - case 0: - mutex_lock(&sas_host->lock); - scan_channel_zero(shost, id, lun); - mutex_unlock(&sas_host->lock); - break; - - case SCAN_WILD_CARD: - mutex_lock(&sas_host->lock); - scan_channel_zero(shost, id, lun); - mutex_unlock(&sas_host->lock); + struct sas_rphy *rphy; - for (i = 1; i <= shost->max_channel; i++) { - res = scsi_scan_host_selected(shost, i, id, lun, - SCSI_SCAN_MANUAL); - if (res) - goto exit_scan; - } - break; + mutex_lock(&sas_host->lock); + list_for_each_entry(rphy, &sas_host->rphy_list, list) { + if (rphy->identify.device_type != SAS_END_DEVICE || + rphy->scsi_target_id == -1) + continue; - default: - if (channel <= shost->max_channel) { - res = scsi_scan_host_selected(shost, channel, id, lun, - SCSI_SCAN_MANUAL); - } else { - res = -EINVAL; + if ((channel == SCAN_WILD_CARD || channel == 0) && + (id == SCAN_WILD_CARD || id == rphy->scsi_target_id)) { + scsi_scan_target(&rphy->dev, 0, rphy->scsi_target_id, + lun, SCSI_SCAN_MANUAL); } - break; } + mutex_unlock(&sas_host->lock); -exit_scan: - return res; + return 0; } diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 74cd4e8a61c2a..5408f002e6c01 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -863,10 +863,9 @@ sg_fill_request_table(Sg_fd *sfp, sg_req_info_t *rinfo) if (val >= SG_MAX_QUEUE) break; rinfo[val].req_state = srp->done + 1; - rinfo[val].problem = - srp->header.masked_status & - srp->header.host_status & - srp->header.driver_status; + rinfo[val].problem = srp->header.masked_status || + srp->header.host_status || + srp->header.driver_status; if (srp->done) rinfo[val].duration = srp->header.duration; diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 2026ac645d6ab..5ec583dc2e7df 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -2642,7 +2642,7 @@ static int pqi_scan_finished(struct Scsi_Host *shost, { struct pqi_ctrl_info *ctrl_info; - ctrl_info = shost_priv(shost); + ctrl_info = shost_to_hba(shost); return !mutex_is_locked(&ctrl_info->scan_mutex); } diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c index 70b6eddb867b9..3ecc4ce9cfa25 100644 --- a/drivers/soc/fsl/qe/qe.c +++ b/drivers/soc/fsl/qe/qe.c @@ -89,6 +89,9 @@ void qe_reset(void) if (qe_immr == NULL) qe_immr = ioremap(get_qe_base(), QE_IMMAP_SIZE); + if (!qe_immr) + panic("QE:ioremap failed!"); + qe_snums_init(); qe_issue_cmd(QE_RESET, QE_CR_SUBBLOCK_INVALID, diff --git a/drivers/soc/fsl/qe/qe_ports_ic.c b/drivers/soc/fsl/qe/qe_ports_ic.c index 8e2107e2cde5b..5e3fae19f3141 100644 --- a/drivers/soc/fsl/qe/qe_ports_ic.c +++ b/drivers/soc/fsl/qe/qe_ports_ic.c @@ -17,6 +17,7 @@ struct qepic_data { void __iomem *reg; struct irq_domain *host; + int irq; }; static void qepic_mask(struct irq_data *d) @@ -92,11 +93,18 @@ static const struct irq_domain_ops qepic_host_ops = { .map = qepic_host_map, }; +static void qepic_remove(void *res) +{ + struct qepic_data *data = res; + + irq_set_chained_handler_and_data(data->irq, NULL, NULL); + irq_domain_remove(data->host); +} + static int qepic_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct qepic_data *data; - int irq; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -106,17 +114,18 @@ static int qepic_probe(struct platform_device *pdev) if (IS_ERR(data->reg)) return PTR_ERR(data->reg); - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; + data->irq = platform_get_irq(pdev, 0); + if (data->irq < 0) + return data->irq; data->host = irq_domain_add_linear(dev->of_node, 32, &qepic_host_ops, data); if (!data->host) return -ENODEV; - irq_set_chained_handler_and_data(irq, qepic_cascade, data); + irq_set_chained_handler_and_data(data->irq, qepic_cascade, data); + + return devm_add_action_or_reset(dev, qepic_remove, data); - return 0; } static const struct of_device_id qepic_match[] = { diff --git a/drivers/soc/mediatek/mt8167-mmsys.h b/drivers/soc/mediatek/mt8167-mmsys.h index c468926561b47..eef14083c47b5 100644 --- a/drivers/soc/mediatek/mt8167-mmsys.h +++ b/drivers/soc/mediatek/mt8167-mmsys.h @@ -10,24 +10,29 @@ #define MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN 0x06c #define MT8167_DITHER_MOUT_EN_RDMA0 0x1 +#define MT8167_DITHER_MOUT_EN_MASK 0x7 + #define MT8167_RDMA0_SOUT_DSI0 0x2 +#define MT8167_RDMA0_SOUT_MASK 0x3 + #define MT8167_DSI0_SEL_IN_RDMA0 0x1 +#define MT8167_DSI0_SEL_IN_MASK 0x3 static const struct mtk_mmsys_routes mt8167_mmsys_routing_table[] = { MMSYS_ROUTE(OVL0, COLOR0, MT8167_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, OVL0_MOUT_EN_COLOR0, OVL0_MOUT_EN_COLOR0), MMSYS_ROUTE(DITHER0, RDMA0, - MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_RDMA0, + MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_MASK, MT8167_DITHER_MOUT_EN_RDMA0), MMSYS_ROUTE(OVL0, COLOR0, MT8167_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, COLOR0_SEL_IN_OVL0, COLOR0_SEL_IN_OVL0), MMSYS_ROUTE(RDMA0, DSI0, - MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_RDMA0, + MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_MASK, MT8167_DSI0_SEL_IN_RDMA0), MMSYS_ROUTE(RDMA0, DSI0, - MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_DSI0, + MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK, MT8167_RDMA0_SOUT_DSI0), }; diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c index 7602b8a909b05..e2ca380812d29 100644 --- a/drivers/soc/ti/k3-ringacc.c +++ b/drivers/soc/ti/k3-ringacc.c @@ -1012,7 +1012,7 @@ static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) { return k3_ringacc_ring_access_proxy(ring, elem, - K3_RINGACC_ACCESS_MODE_POP_HEAD); + K3_RINGACC_ACCESS_MODE_POP_TAIL); } static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, @@ -1083,7 +1083,7 @@ static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) { return k3_ringacc_ring_access_io(ring, elem, - K3_RINGACC_ACCESS_MODE_POP_HEAD); + K3_RINGACC_ACCESS_MODE_POP_TAIL); } /* diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c index 9085db1b480aa..370e61ac47d88 100644 --- a/drivers/soc/xilinx/zynqmp_power.c +++ b/drivers/soc/xilinx/zynqmp_power.c @@ -303,18 +303,18 @@ static int zynqmp_pm_probe(struct platform_device *pdev) * is not available to use) or -ENODEV(Xilinx Event Manager not compiled), * then use ipi-mailbox or interrupt method. */ + zynqmp_pm_init_suspend_work = devm_kzalloc(&pdev->dev, + sizeof(struct zynqmp_pm_work_struct), + GFP_KERNEL); + if (!zynqmp_pm_init_suspend_work) + return -ENOMEM; + + INIT_WORK(&zynqmp_pm_init_suspend_work->callback_work, + zynqmp_pm_init_suspend_work_fn); + ret = register_event(&pdev->dev, PM_INIT_SUSPEND_CB, 0, 0, false, suspend_event_callback); if (!ret) { - zynqmp_pm_init_suspend_work = devm_kzalloc(&pdev->dev, - sizeof(struct zynqmp_pm_work_struct), - GFP_KERNEL); - if (!zynqmp_pm_init_suspend_work) - return -ENOMEM; - - INIT_WORK(&zynqmp_pm_init_suspend_work->callback_work, - zynqmp_pm_init_suspend_work_fn); - ret = zynqmp_pm_get_family_info(&pm_family_code); if (ret < 0) return ret; @@ -326,14 +326,6 @@ static int zynqmp_pm_probe(struct platform_device *pdev) else return -ENODEV; - ret = register_event(&pdev->dev, PM_NOTIFY_CB, node_id, EVENT_SUBSYSTEM_RESTART, - false, subsystem_restart_event_callback); - if (ret) { - dev_err(&pdev->dev, "Failed to Register with Xilinx Event manager %d\n", - ret); - return ret; - } - zynqmp_pm_init_restart_work = devm_kzalloc(&pdev->dev, sizeof(struct zynqmp_pm_work_struct), GFP_KERNEL); @@ -342,19 +334,18 @@ static int zynqmp_pm_probe(struct platform_device *pdev) INIT_WORK(&zynqmp_pm_init_restart_work->callback_work, zynqmp_pm_subsystem_restart_work_fn); + + ret = register_event(&pdev->dev, PM_NOTIFY_CB, node_id, EVENT_SUBSYSTEM_RESTART, + false, subsystem_restart_event_callback); + if (ret) { + dev_err(&pdev->dev, "Failed to Register with Xilinx Event manager %d\n", + ret); + return ret; + } } else if (ret != -EACCES && ret != -ENODEV) { dev_err(&pdev->dev, "Failed to Register with Xilinx Event manager %d\n", ret); return ret; } else if (of_property_present(pdev->dev.of_node, "mboxes")) { - zynqmp_pm_init_suspend_work = - devm_kzalloc(&pdev->dev, - sizeof(struct zynqmp_pm_work_struct), - GFP_KERNEL); - if (!zynqmp_pm_init_suspend_work) - return -ENOMEM; - - INIT_WORK(&zynqmp_pm_init_suspend_work->callback_work, - zynqmp_pm_init_suspend_work_fn); client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL); if (!client) return -ENOMEM; @@ -398,8 +389,10 @@ static void zynqmp_pm_remove(struct platform_device *pdev) { sysfs_remove_file(&pdev->dev.kobj, &dev_attr_suspend_mode.attr); - if (!rx_chan) + if (rx_chan) { mbox_free_channel(rx_chan); + rx_chan = NULL; + } } static const struct of_device_id pm_of_match[] = { diff --git a/drivers/soundwire/generic_bandwidth_allocation.c b/drivers/soundwire/generic_bandwidth_allocation.c index fb3970e12dac9..f016ad088a1db 100644 --- a/drivers/soundwire/generic_bandwidth_allocation.c +++ b/drivers/soundwire/generic_bandwidth_allocation.c @@ -299,39 +299,36 @@ static int sdw_add_element_group_count(struct sdw_group *group, int num = group->count; int i; - for (i = 0; i <= num; i++) { + for (i = 0; i < num; i++) { if (rate == group->rates[i] && lane == group->lanes[i]) - break; - - if (i != num) - continue; - - if (group->count >= group->max_size) { - unsigned int *rates; - unsigned int *lanes; + return 0; + } - group->max_size += 1; - rates = krealloc(group->rates, - (sizeof(int) * group->max_size), - GFP_KERNEL); - if (!rates) - return -ENOMEM; + if (group->count >= group->max_size) { + unsigned int *rates; + unsigned int *lanes; - group->rates = rates; + group->max_size += 1; + rates = krealloc(group->rates, + (sizeof(int) * group->max_size), + GFP_KERNEL); + if (!rates) + return -ENOMEM; - lanes = krealloc(group->lanes, - (sizeof(int) * group->max_size), - GFP_KERNEL); - if (!lanes) - return -ENOMEM; + group->rates = rates; - group->lanes = lanes; - } + lanes = krealloc(group->lanes, + (sizeof(int) * group->max_size), + GFP_KERNEL); + if (!lanes) + return -ENOMEM; - group->rates[group->count] = rate; - group->lanes[group->count++] = lane; + group->lanes = lanes; } + group->rates[group->count] = rate; + group->lanes[group->count++] = lane; + return 0; } diff --git a/drivers/soundwire/intel_ace2x.c b/drivers/soundwire/intel_ace2x.c index 20422534baf19..6cd3a87323750 100644 --- a/drivers/soundwire/intel_ace2x.c +++ b/drivers/soundwire/intel_ace2x.c @@ -317,6 +317,7 @@ static void intel_ace2x_bpt_close_stream(struct sdw_intel *sdw, struct sdw_slave dev_err(cdns->dev, "%s: remove slave failed: %d\n", __func__, ret); + sdw_release_stream(cdns->bus.bpt_stream); cdns->bus.bpt_stream = NULL; } diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c index 4ed8fb7663ad4..0b96268380283 100644 --- a/drivers/soundwire/stream.c +++ b/drivers/soundwire/stream.c @@ -697,6 +697,13 @@ static int sdw_program_params(struct sdw_bus *bus, bool prepare) if (scale_index < 0) return scale_index; + /* Skip the unattached Peripherals */ + if (!completion_done(&slave->enumeration_complete)) { + dev_warn(&slave->dev, + "Not enumerated, skip programming BUSCLOCK_SCALE\n"); + continue; + } + ret = sdw_write_no_pm(slave, addr1, scale_index); if (ret < 0) { dev_err(&slave->dev, "SDW_SCP_BUSCLOCK_SCALE register write failed\n"); diff --git a/drivers/spi/spi-atcspi200.c b/drivers/spi/spi-atcspi200.c index 3832d9db3cbf8..c5cf1aa2d6743 100644 --- a/drivers/spi/spi-atcspi200.c +++ b/drivers/spi/spi-atcspi200.c @@ -575,12 +575,6 @@ static int atcspi_probe(struct platform_device *pdev) if (ret) goto free_controller; - ret = devm_spi_register_controller(&pdev->dev, host); - if (ret) { - dev_err_probe(spi->dev, ret, - "Failed to register SPI controller\n"); - goto free_controller; - } spi->use_dma = false; if (ATCSPI_DMA_SUPPORT) { ret = atcspi_configure_dma(spi); @@ -591,6 +585,13 @@ static int atcspi_probe(struct platform_device *pdev) spi->use_dma = true; } + ret = devm_spi_register_controller(&pdev->dev, host); + if (ret) { + dev_err_probe(spi->dev, ret, + "Failed to register SPI controller\n"); + goto free_controller; + } + return 0; free_controller: diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c index 42db85d7ff8e1..397bc819e7161 100644 --- a/drivers/spi/spi-atmel.c +++ b/drivers/spi/spi-atmel.c @@ -559,6 +559,38 @@ static int atmel_spi_dma_slave_config(struct atmel_spi *as, u8 bits_per_word) return err; } +static void atmel_spi_release_dma(void *data) +{ + struct spi_controller *host = data; + struct atmel_spi *as = spi_controller_get_devdata(host); + struct device *dev = &as->pdev->dev; + + if (host->dma_tx) { + dma_release_channel(host->dma_tx); + host->dma_tx = NULL; + } + + if (host->dma_rx) { + dma_release_channel(host->dma_rx); + host->dma_rx = NULL; + } + + if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { + if (as->addr_tx_bbuf) { + dma_free_coherent(dev, SPI_MAX_DMA_XFER, + as->addr_tx_bbuf, + as->dma_addr_tx_bbuf); + as->addr_tx_bbuf = NULL; + } + if (as->addr_rx_bbuf) { + dma_free_coherent(dev, SPI_MAX_DMA_XFER, + as->addr_rx_bbuf, + as->dma_addr_rx_bbuf); + as->addr_rx_bbuf = NULL; + } + } +} + static int atmel_spi_configure_dma(struct spi_controller *host, struct atmel_spi *as) { @@ -569,7 +601,8 @@ static int atmel_spi_configure_dma(struct spi_controller *host, if (IS_ERR(host->dma_tx)) { err = PTR_ERR(host->dma_tx); dev_dbg(dev, "No TX DMA channel, DMA is disabled\n"); - goto error_clear; + host->dma_tx = NULL; + return err; } host->dma_rx = dma_request_chan(dev, "rx"); @@ -580,26 +613,45 @@ static int atmel_spi_configure_dma(struct spi_controller *host, * requested tx channel. */ dev_dbg(dev, "No RX DMA channel, DMA is disabled\n"); - goto error; + host->dma_rx = NULL; + goto err_release_dma; } err = atmel_spi_dma_slave_config(as, 8); if (err) - goto error; + goto err_release_dma; + + if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { + as->addr_tx_bbuf = dma_alloc_coherent(dev, SPI_MAX_DMA_XFER, + &as->dma_addr_tx_bbuf, + GFP_KERNEL | GFP_DMA); + if (!as->addr_tx_bbuf) { + err = -ENOMEM; + goto err_release_dma; + } + + as->addr_rx_bbuf = dma_alloc_coherent(dev, SPI_MAX_DMA_XFER, + &as->dma_addr_rx_bbuf, + GFP_KERNEL | GFP_DMA); + if (!as->addr_rx_bbuf) { + err = -ENOMEM; + goto err_release_dma; + } + } + + err = devm_add_action_or_reset(dev, atmel_spi_release_dma, host); + if (err) + return err; dev_info(&as->pdev->dev, - "Using %s (tx) and %s (rx) for DMA transfers\n", - dma_chan_name(host->dma_tx), - dma_chan_name(host->dma_rx)); + "Using %s (tx) and %s (rx) for DMA transfers\n", + dma_chan_name(host->dma_tx), dma_chan_name(host->dma_rx)); return 0; -error: - if (!IS_ERR(host->dma_rx)) - dma_release_channel(host->dma_rx); - if (!IS_ERR(host->dma_tx)) - dma_release_channel(host->dma_tx); -error_clear: - host->dma_tx = host->dma_rx = NULL; + +err_release_dma: + atmel_spi_release_dma(host); + return err; } @@ -611,18 +663,6 @@ static void atmel_spi_stop_dma(struct spi_controller *host) dmaengine_terminate_all(host->dma_tx); } -static void atmel_spi_release_dma(struct spi_controller *host) -{ - if (host->dma_rx) { - dma_release_channel(host->dma_rx); - host->dma_rx = NULL; - } - if (host->dma_tx) { - dma_release_channel(host->dma_tx); - host->dma_tx = NULL; - } -} - /* This function is called by the DMA driver from tasklet context */ static void dma_callback(void *data) { @@ -1585,30 +1625,6 @@ static int atmel_spi_probe(struct platform_device *pdev) as->use_pdc = true; } - if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { - as->addr_rx_bbuf = dma_alloc_coherent(&pdev->dev, - SPI_MAX_DMA_XFER, - &as->dma_addr_rx_bbuf, - GFP_KERNEL | GFP_DMA); - if (!as->addr_rx_bbuf) { - as->use_dma = false; - } else { - as->addr_tx_bbuf = dma_alloc_coherent(&pdev->dev, - SPI_MAX_DMA_XFER, - &as->dma_addr_tx_bbuf, - GFP_KERNEL | GFP_DMA); - if (!as->addr_tx_bbuf) { - as->use_dma = false; - dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, - as->addr_rx_bbuf, - as->dma_addr_rx_bbuf); - } - } - if (!as->use_dma) - dev_info(host->dev.parent, - " can not allocate dma coherent memory\n"); - } - if (as->caps.has_dma_support && !as->use_dma) dev_info(&pdev->dev, "Atmel SPI Controller using PIO only\n"); @@ -1668,13 +1684,10 @@ static int atmel_spi_probe(struct platform_device *pdev) out_free_dma: pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); - - if (as->use_dma) - atmel_spi_release_dma(host); - spi_writel(as, CR, SPI_BIT(SWRST)); spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ - clk_disable_unprepare(as->gclk); + if (as->gclk) + clk_disable_unprepare(as->gclk); out_disable_clk: clk_disable_unprepare(clk); out_free_irq: @@ -1695,18 +1708,8 @@ static void atmel_spi_remove(struct platform_device *pdev) spi_unregister_controller(host); /* reset the hardware and block queue progress */ - if (as->use_dma) { + if (as->use_dma) atmel_spi_stop_dma(host); - atmel_spi_release_dma(host); - if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { - dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, - as->addr_tx_bbuf, - as->dma_addr_tx_bbuf); - dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, - as->addr_rx_bbuf, - as->dma_addr_rx_bbuf); - } - } spin_lock_irq(&as->lock); spi_writel(as, CR, SPI_BIT(SWRST)); diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h index 9cc79c566a70c..2f2debc64e737 100644 --- a/drivers/spi/spi-dw.h +++ b/drivers/spi/spi-dw.h @@ -282,6 +282,7 @@ static inline void dw_spi_shutdown_chip(struct dw_spi *dws) { dw_spi_enable_chip(dws, 0); dw_spi_set_clk(dws, 0); + dws->current_freq = 0; } extern void dw_spi_set_cs(struct spi_device *spi, bool enable); diff --git a/drivers/spi/spi-ep93xx.c b/drivers/spi/spi-ep93xx.c index f716c9607be4a..4cab0d97a68af 100644 --- a/drivers/spi/spi-ep93xx.c +++ b/drivers/spi/spi-ep93xx.c @@ -600,6 +600,7 @@ static int ep93xx_spi_setup_dma(struct device *dev, struct ep93xx_spi *espi) espi->dma_rx = NULL; fail_free_page: free_page((unsigned long)espi->zeropage); + espi->zeropage = NULL; return ret; } diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index e201309f8aae3..e14753144e19d 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -647,7 +647,7 @@ static int fsl_lpspi_dma_transfer(struct spi_controller *controller, tx->sgl, tx->nents, DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!desc_tx) { - dmaengine_terminate_all(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_rx); return -EINVAL; } @@ -668,8 +668,8 @@ static int fsl_lpspi_dma_transfer(struct spi_controller *controller, transfer_timeout); if (!time_left) { dev_err(fsl_lpspi->dev, "I/O Error in DMA TX\n"); - dmaengine_terminate_all(controller->dma_tx); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_rx); fsl_lpspi_reset(fsl_lpspi); return -ETIMEDOUT; } @@ -678,8 +678,8 @@ static int fsl_lpspi_dma_transfer(struct spi_controller *controller, transfer_timeout); if (!time_left) { dev_err(fsl_lpspi->dev, "I/O Error in DMA RX\n"); - dmaengine_terminate_all(controller->dma_tx); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_rx); fsl_lpspi_reset(fsl_lpspi); return -ETIMEDOUT; } @@ -688,8 +688,8 @@ static int fsl_lpspi_dma_transfer(struct spi_controller *controller, fsl_lpspi->target_aborted) { dev_dbg(fsl_lpspi->dev, "I/O Error in DMA TX interrupted\n"); - dmaengine_terminate_all(controller->dma_tx); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_rx); fsl_lpspi_reset(fsl_lpspi); return -EINTR; } @@ -698,8 +698,8 @@ static int fsl_lpspi_dma_transfer(struct spi_controller *controller, fsl_lpspi->target_aborted) { dev_dbg(fsl_lpspi->dev, "I/O Error in DMA RX interrupted\n"); - dmaengine_terminate_all(controller->dma_tx); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_rx); fsl_lpspi_reset(fsl_lpspi); return -EINTR; } diff --git a/drivers/spi/spi-hisi-kunpeng.c b/drivers/spi/spi-hisi-kunpeng.c index 046bd894040b1..395214b81179e 100644 --- a/drivers/spi/spi-hisi-kunpeng.c +++ b/drivers/spi/spi-hisi-kunpeng.c @@ -520,10 +520,8 @@ static int hisi_spi_probe(struct platform_device *pdev) } ret = spi_register_controller(host); - if (ret) { - dev_err(dev, "failed to register spi host, ret=%d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to register spi host\n"); if (hisi_spi_debugfs_init(hs)) dev_info(dev, "failed to create debugfs dir\n"); diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c index 480d1e8b281f1..79a6c1a60b0a0 100644 --- a/drivers/spi/spi-imx.c +++ b/drivers/spi/spi-imx.c @@ -1774,8 +1774,8 @@ static int spi_imx_dma_submit(struct spi_imx_data *spi_imx, transfer_timeout); if (!time_left) { dev_err(spi_imx->dev, "I/O Error in DMA TX\n"); - dmaengine_terminate_all(controller->dma_tx); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_rx); return -ETIMEDOUT; } @@ -1784,7 +1784,7 @@ static int spi_imx_dma_submit(struct spi_imx_data *spi_imx, if (!time_left) { dev_err(&controller->dev, "I/O Error in DMA RX\n"); spi_imx->devtype_data->reset(spi_imx); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_rx); return -ETIMEDOUT; } } else { @@ -1793,15 +1793,15 @@ static int spi_imx_dma_submit(struct spi_imx_data *spi_imx, if (wait_for_completion_interruptible(&spi_imx->dma_tx_completion) || READ_ONCE(spi_imx->target_aborted)) { dev_dbg(spi_imx->dev, "I/O Error in DMA TX interrupted\n"); - dmaengine_terminate_all(controller->dma_tx); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_rx); return -EINTR; } if (wait_for_completion_interruptible(&spi_imx->dma_rx_completion) || READ_ONCE(spi_imx->target_aborted)) { dev_dbg(spi_imx->dev, "I/O Error in DMA RX interrupted\n"); - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_rx); return -EINTR; } @@ -1818,9 +1818,9 @@ static int spi_imx_dma_submit(struct spi_imx_data *spi_imx, return 0; dmaengine_terminate_tx: - dmaengine_terminate_all(controller->dma_tx); + dmaengine_terminate_sync(controller->dma_tx); dmaengine_terminate_rx: - dmaengine_terminate_all(controller->dma_rx); + dmaengine_terminate_sync(controller->dma_rx); return -EINVAL; } @@ -2152,7 +2152,8 @@ static int spi_imx_transfer_one(struct spi_controller *controller, if (spi_imx->usedma) { ret = spi_imx_dma_transfer(spi_imx, transfer); if (transfer->error & SPI_TRANS_FAIL_NO_START) { - spi_imx->usedma = false; + controller->fallback = true; + spi_imx_setupxfer(spi, transfer); if (spi_imx->target_mode) return spi_imx_pio_transfer_target(spi, transfer); else diff --git a/drivers/spi/spi-meson-spifc.c b/drivers/spi/spi-meson-spifc.c index b818950a8cb7d..e2d19c3873f71 100644 --- a/drivers/spi/spi-meson-spifc.c +++ b/drivers/spi/spi-meson-spifc.c @@ -351,6 +351,7 @@ static void meson_spifc_remove(struct platform_device *pdev) { pm_runtime_get_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-rpc-if.c b/drivers/spi/spi-rpc-if.c index 6edc0c4db854d..c2f9c86c1d8d4 100644 --- a/drivers/spi/spi-rpc-if.c +++ b/drivers/spi/spi-rpc-if.c @@ -206,8 +206,12 @@ static int rpcif_spi_suspend(struct device *dev) static int rpcif_spi_resume(struct device *dev) { struct spi_controller *ctlr = dev_get_drvdata(dev); + struct rpcif *rpc = spi_controller_get_devdata(ctlr); + int ret; - rpcif_hw_init(dev, false); + ret = rpcif_hw_init(rpc->dev, false); + if (ret) + return ret; return spi_controller_resume(ctlr); } diff --git a/drivers/spi/spi-rzv2h-rspi.c b/drivers/spi/spi-rzv2h-rspi.c index 6ed3fad873b8c..190d7eb2034af 100644 --- a/drivers/spi/spi-rzv2h-rspi.c +++ b/drivers/spi/spi-rzv2h-rspi.c @@ -366,14 +366,14 @@ static int rzv2h_rspi_transfer_dma(struct rzv2h_rspi_priv *rspi, rzv2h_rspi_clear_all_irqs(rspi); ret = wait_event_interruptible_timeout(rspi->wait, rspi->dma_callbacked, HZ); - if (ret) { + if (ret > 0) { dmaengine_synchronize(rspi->controller->dma_tx); dmaengine_synchronize(rspi->controller->dma_rx); ret = 0; } else { dmaengine_terminate_sync(rspi->controller->dma_tx); dmaengine_terminate_sync(rspi->controller->dma_rx); - ret = -ETIMEDOUT; + ret = ret ?: -ETIMEDOUT; } enable_irq(rspi->irq_rx); diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c index f114b6313f4f6..5e49ce15ac4df 100644 --- a/drivers/spi/spi-sh-msiof.c +++ b/drivers/spi/spi-sh-msiof.c @@ -114,7 +114,7 @@ static irqreturn_t sh_msiof_spi_irq(int irq, void *data) return IRQ_HANDLED; } -static void sh_msiof_spi_reset_regs(struct sh_msiof_spi_priv *p) +static int sh_msiof_spi_reset_regs(struct sh_msiof_spi_priv *p) { u32 mask = SICTR_TXRST | SICTR_RXRST; u32 data; @@ -123,8 +123,8 @@ static void sh_msiof_spi_reset_regs(struct sh_msiof_spi_priv *p) data |= mask; sh_msiof_write(p, SICTR, data); - readl_poll_timeout_atomic(p->mapbase + SICTR, data, !(data & mask), 1, - 100); + return readl_poll_timeout_atomic(p->mapbase + SICTR, data, + !(data & mask), 1, 100); } static void sh_msiof_spi_set_clk_regs(struct sh_msiof_spi_priv *p, @@ -834,7 +834,9 @@ static int sh_msiof_transfer_one(struct spi_controller *ctlr, int ret; /* reset registers */ - sh_msiof_spi_reset_regs(p); + ret = sh_msiof_spi_reset_regs(p); + if (ret) + return ret; /* setup clocks (clock already enabled in chipselect()) */ if (!spi_controller_is_target(p->ctlr)) diff --git a/drivers/spi/spi-uniphier.c b/drivers/spi/spi-uniphier.c index eac6c3e8908bd..f2859e96003a2 100644 --- a/drivers/spi/spi-uniphier.c +++ b/drivers/spi/spi-uniphier.c @@ -659,6 +659,8 @@ static int uniphier_spi_probe(struct platform_device *pdev) priv->host = host; priv->is_save_param = false; + init_completion(&priv->xfer_done); + priv->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(priv->base)) { ret = PTR_ERR(priv->base); @@ -686,8 +688,6 @@ static int uniphier_spi_probe(struct platform_device *pdev) goto out_host_put; } - init_completion(&priv->xfer_done); - clk_rate = clk_get_rate(priv->clk); host->max_speed_hz = DIV_ROUND_UP(clk_rate, SSI_MIN_CLK_DIVIDER); diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c index 9f065d4e27d19..b95485710e2f7 100644 --- a/drivers/spi/spi-xilinx.c +++ b/drivers/spi/spi-xilinx.c @@ -371,11 +371,18 @@ static int xilinx_spi_find_buffer_size(struct xilinx_spi *xspi) xspi->regs + XIPIF_V123B_RESETR_OFFSET); /* Fill the Tx FIFO with as many words as possible */ - do { + while (1) { xspi->write_fn(0, xspi->regs + XSPI_TXD_OFFSET); sr = xspi->read_fn(xspi->regs + XSPI_SR_OFFSET); + if (sr & XSPI_SR_TX_FULL_MASK) + break; + n_words++; - } while (!(sr & XSPI_SR_TX_FULL_MASK)); + } + + /* Handle the NO FIFO case separately */ + if (!n_words) + return 1; return n_words; } diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 104279858f567..889e1eecc7573 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -5003,11 +5003,6 @@ static int of_spi_notify(struct notifier_block *nb, unsigned long action, return NOTIFY_OK; } - /* - * Clear the flag before adding the device so that fw_devlink - * doesn't skip adding consumers to this device. - */ - fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); spi = of_register_spi_device(ctlr, rd->dn); put_device(&ctlr->dev); diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c index d3414312e1de2..998c9f46bd068 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c @@ -809,7 +809,7 @@ static int gc2235_probe(struct i2c_client *client) ret = gc2235_s_config(&dev->sd, client->irq, gcpdev); if (ret) - goto out_free; + goto err_unregister_subdev; dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; dev->pad.flags = MEDIA_PAD_FL_SOURCE; @@ -818,18 +818,16 @@ static int gc2235_probe(struct i2c_client *client) ret = v4l2_ctrl_handler_init(&dev->ctrl_handler, ARRAY_SIZE(gc2235_controls)); - if (ret) { - gc2235_remove(client); - return ret; - } + if (ret) + goto err_csi_cfg; for (i = 0; i < ARRAY_SIZE(gc2235_controls); i++) v4l2_ctrl_new_custom(&dev->ctrl_handler, &gc2235_controls[i], NULL); if (dev->ctrl_handler.error) { - gc2235_remove(client); - return dev->ctrl_handler.error; + ret = dev->ctrl_handler.error; + goto err_ctrl_handler; } /* Use same lock for controls as for everything else. */ @@ -838,14 +836,23 @@ static int gc2235_probe(struct i2c_client *client) ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad); if (ret) - gc2235_remove(client); + goto err_ctrl_handler; + + ret = atomisp_register_i2c_module(&dev->sd, gcpdev); + if (ret) + goto err_media_cleanup; - return atomisp_register_i2c_module(&dev->sd, gcpdev); + return 0; -out_free: +err_media_cleanup: + media_entity_cleanup(&dev->sd.entity); +err_ctrl_handler: + v4l2_ctrl_handler_free(&dev->ctrl_handler); +err_csi_cfg: + dev->platform_data->csi_cfg(&dev->sd, 0); +err_unregister_subdev: v4l2_device_unregister_subdev(&dev->sd); kfree(dev); - return ret; } diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c index fec369575d883..d58aa4d594577 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c +++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c @@ -3364,10 +3364,8 @@ int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd, if (ret) return ret; - if (sh_css_set_black_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, - raw_black_frame) != 0) - return -ENOMEM; - + ret = sh_css_set_black_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, + raw_black_frame); ia_css_frame_free(raw_black_frame); return ret; } diff --git a/drivers/staging/media/atomisp/pci/sh_css.c b/drivers/staging/media/atomisp/pci/sh_css.c index 6cda5925fa45e..3d1903a988048 100644 --- a/drivers/staging/media/atomisp/pci/sh_css.c +++ b/drivers/staging/media/atomisp/pci/sh_css.c @@ -5020,7 +5020,6 @@ static int load_primary_binaries( struct ia_css_capture_settings *mycs; unsigned int i; bool need_extra_yuv_scaler = false; - struct ia_css_binary_descr prim_descr[MAX_NUM_PRIMARY_STAGES]; IA_CSS_ENTER_PRIVATE(""); assert(pipe); @@ -5189,15 +5188,16 @@ static int load_primary_binaries( /* Primary */ for (i = 0; i < mycs->num_primary_stage; i++) { + struct ia_css_binary_descr prim_descr; struct ia_css_frame_info *local_vf_info = NULL; if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] && (i == mycs->num_primary_stage - 1)) local_vf_info = &vf_info; - ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], + ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr, &prim_in_info, &prim_out_info, local_vf_info, i); - err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]); + err = ia_css_binary_find(&prim_descr, &mycs->primary_binary[i]); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -8192,7 +8192,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, int ia_css_stream_destroy(struct ia_css_stream *stream) { - int i; + int i, j; int err = 0; IA_CSS_ENTER_PRIVATE("stream = %p", stream); @@ -8223,10 +8223,10 @@ ia_css_stream_destroy(struct ia_css_stream *stream) sp_pipeline_input_terminal = &sh_css_sp_group.pipe_io[sp_thread_id].input; - for (i = 0; i < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; i++) { + for (j = 0; j < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; j++) { ia_css_isys_stream_h isys_stream = - &sp_pipeline_input_terminal->context.virtual_input_system_stream[i]; - if (stream->config.isys_config[i].valid && isys_stream->valid) + &sp_pipeline_input_terminal->context.virtual_input_system_stream[j]; + if (stream->config.isys_config[j].valid && isys_stream->valid) ia_css_isys_stream_destroy(isys_stream); } } diff --git a/drivers/staging/media/ipu3/ipu3-css-params.c b/drivers/staging/media/ipu3/ipu3-css-params.c index 2c48d57a3180f..92cce31e35c5d 100644 --- a/drivers/staging/media/ipu3/ipu3-css-params.c +++ b/drivers/staging/media/ipu3/ipu3-css-params.c @@ -1770,6 +1770,8 @@ static int imgu_css_cfg_acc_stripe(struct imgu_css *css, unsigned int pipe, acc->stripe.bds_out_stripes[0].width = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, f); } else { + u32 offset; + /* Image processing is divided into two stripes */ acc->stripe.bds_out_stripes[0].width = acc->stripe.bds_out_stripes[1].width = @@ -1788,8 +1790,10 @@ static int imgu_css_cfg_acc_stripe(struct imgu_css *css, unsigned int pipe, acc->stripe.bds_out_stripes[1].width += f; } /* Overlap between stripes is IPU3_UAPI_ISP_VEC_ELEMS * 4 */ - acc->stripe.bds_out_stripes[1].offset = - acc->stripe.bds_out_stripes[0].width - 2 * f; + offset = acc->stripe.bds_out_stripes[0].width - 2 * f; + if (offset > 65535) + return -EINVAL; + acc->stripe.bds_out_stripes[1].offset = offset; } acc->stripe.effective_stripes[0].height = diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c index c771e763f8c50..310e3f24e571f 100644 --- a/drivers/staging/media/ipu7/ipu7.c +++ b/drivers/staging/media/ipu7/ipu7.c @@ -2169,21 +2169,18 @@ ipu7_isys_init(struct pci_dev *pdev, struct device *parent, isys_adev->mmu = ipu7_mmu_init(dev, base, ISYS_MMID, &ipdata->hw_variant); if (IS_ERR(isys_adev->mmu)) { - dev_err_probe(dev, PTR_ERR(isys_adev->mmu), - "ipu7_mmu_init(isys_adev->mmu) failed\n"); + ret = dev_err_probe(dev, PTR_ERR(isys_adev->mmu), + "ipu7_mmu_init(isys_adev->mmu) failed\n"); put_device(&isys_adev->auxdev.dev); - kfree(pdata); - return ERR_CAST(isys_adev->mmu); + return ERR_PTR(ret); } isys_adev->mmu->dev = &isys_adev->auxdev.dev; isys_adev->subsys = IPU_IS; ret = ipu7_bus_add_device(isys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return isys_adev; } @@ -2216,21 +2213,18 @@ ipu7_psys_init(struct pci_dev *pdev, struct device *parent, psys_adev->mmu = ipu7_mmu_init(&pdev->dev, base, PSYS_MMID, &ipdata->hw_variant); if (IS_ERR(psys_adev->mmu)) { - dev_err_probe(&pdev->dev, PTR_ERR(psys_adev->mmu), - "ipu7_mmu_init(psys_adev->mmu) failed\n"); + ret = dev_err_probe(&pdev->dev, PTR_ERR(psys_adev->mmu), + "ipu7_mmu_init(psys_adev->mmu) failed\n"); put_device(&psys_adev->auxdev.dev); - kfree(pdata); - return ERR_CAST(psys_adev->mmu); + return ERR_PTR(ret); } psys_adev->mmu->dev = &psys_adev->auxdev.dev; psys_adev->subsys = IPU_PS; ret = ipu7_bus_add_device(psys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return psys_adev; } diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c index 6600245dff0e2..27e43af6c7bd0 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus.c @@ -476,7 +476,7 @@ static int cedrus_probe(struct platform_device *pdev) ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev); if (ret) { dev_err(&pdev->dev, "Failed to register V4L2 device\n"); - return ret; + goto err_hw; } vfd = &dev->vfd; @@ -537,6 +537,8 @@ static int cedrus_probe(struct platform_device *pdev) v4l2_m2m_release(dev->m2m_dev); err_v4l2: v4l2_device_unregister(&dev->v4l2_dev); +err_hw: + cedrus_hw_remove(dev); return ret; } diff --git a/drivers/staging/most/video/video.c b/drivers/staging/most/video/video.c index 04351f8ccccf1..aab57768f4f70 100644 --- a/drivers/staging/most/video/video.c +++ b/drivers/staging/most/video/video.c @@ -420,6 +420,7 @@ static int comp_register_videodev(struct most_video_dev *mdev) /* Fill the video capture device struct */ *mdev->vdev = comp_videodev_template; + mdev->vdev->release = video_device_release_empty; mdev->vdev->v4l2_dev = &mdev->v4l2_dev; mdev->vdev->lock = &mdev->lock; snprintf(mdev->vdev->name, sizeof(mdev->vdev->name), "MOST: %s", @@ -432,9 +433,13 @@ static int comp_register_videodev(struct most_video_dev *mdev) v4l2_err(&mdev->v4l2_dev, "video_register_device failed (%d)\n", ret); video_device_release(mdev->vdev); + return ret; } - return ret; + mdev->vdev->release = video_device_release; + + return 0; + } static void comp_unregister_videodev(struct most_video_dev *mdev) diff --git a/drivers/staging/nvec/nvec.c b/drivers/staging/nvec/nvec.c index 952c5a849a563..2a3499dd4d634 100644 --- a/drivers/staging/nvec/nvec.c +++ b/drivers/staging/nvec/nvec.c @@ -494,6 +494,8 @@ static void nvec_tx_completed(struct nvec_chip *nvec) static void nvec_rx_completed(struct nvec_chip *nvec) { if (nvec->rx->pos != nvec_msg_size(nvec->rx)) { + unsigned char msg_type = nvec->rx->data[0]; + dev_err(nvec->dev, "RX incomplete: Expected %u bytes, got %u\n", (uint)nvec_msg_size(nvec->rx), (uint)nvec->rx->pos); @@ -502,7 +504,7 @@ static void nvec_rx_completed(struct nvec_chip *nvec) nvec->state = 0; /* Battery quirk - Often incomplete, and likes to crash */ - if (nvec->rx->data[0] == NVEC_BAT) + if (msg_type == NVEC_BAT) complete(&nvec->ec_transfer); return; diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c index 72b7f731dd471..3c1f0068cd92d 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c +++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c @@ -583,9 +583,14 @@ int rtw_get_wapi_ie(u8 *in_ie, uint in_len, u8 *wapi_ie, u16 *wapi_len) cnt = (_TIMESTAMP_ + _BEACON_ITERVAL_ + _CAPABILITY_); while (cnt < in_len) { + if (cnt + 2 > in_len) + break; + if (cnt + 2 + in_ie[cnt + 1] > in_len) + break; authmode = in_ie[cnt]; if (authmode == WLAN_EID_BSS_AC_ACCESS_DELAY && + in_ie[cnt + 1] >= 8 && (!memcmp(&in_ie[cnt + 6], wapi_oui1, 4) || !memcmp(&in_ie[cnt + 6], wapi_oui2, 4))) { if (wapi_ie) @@ -616,9 +621,14 @@ void rtw_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len, u8 *wpa_ie cnt = (_TIMESTAMP_ + _BEACON_ITERVAL_ + _CAPABILITY_); while (cnt < in_len) { + if (cnt + 2 > in_len) + break; + if (cnt + 2 + in_ie[cnt + 1] > in_len) + break; authmode = in_ie[cnt]; if ((authmode == WLAN_EID_VENDOR_SPECIFIC) && + in_ie[cnt + 1] >= 4 && (!memcmp(&in_ie[cnt + 2], &wpa_oui[0], 4))) { if (wpa_ie) memcpy(wpa_ie, &in_ie[cnt], in_ie[cnt + 1] + 2); @@ -699,6 +709,9 @@ u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id, u8 *buf_att if (len_attr) *len_attr = 0; + if (wps_ielen < 6) + return attr_ptr; + if ((wps_ie[0] != WLAN_EID_VENDOR_SPECIFIC) || (memcmp(wps_ie + 2, wps_oui, 4))) { return attr_ptr; @@ -709,6 +722,8 @@ u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id, u8 *buf_att while (attr_ptr - wps_ie < wps_ielen) { /* 4 = 2(Attribute ID) + 2(Length) */ + if (attr_ptr + 4 > wps_ie + wps_ielen) + break; u16 attr_id = get_unaligned_be16(attr_ptr); u16 attr_data_len = get_unaligned_be16(attr_ptr + 2); u16 attr_len = attr_data_len + 4; diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c index 5f00fe282d1be..e78450402d193 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c @@ -687,6 +687,9 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame) if ((pmlmeinfo->state&0x03) != WIFI_FW_AP_STATE) return _FAIL; + if (len < WLAN_HDR_A3_LEN) + return _FAIL; + sa = GetAddr2Ptr(pframe); auth_mode = psecuritypriv->dot11AuthAlgrthm; @@ -698,6 +701,9 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame) prxattrib->hdrlen = WLAN_HDR_A3_LEN; prxattrib->encrypt = _WEP40_; + if (len < WLAN_HDR_A3_LEN + 8) + return _FAIL; + iv = pframe+prxattrib->hdrlen; prxattrib->key_index = ((iv[3]>>6)&0x3); @@ -802,7 +808,7 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame) p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + 4 + _AUTH_IE_OFFSET_, WLAN_EID_CHALLENGE, (int *)&ie_len, len - WLAN_HDR_A3_LEN - _AUTH_IE_OFFSET_ - 4); - if (!p || ie_len <= 0) { + if (!p || ie_len != 128) { status = WLAN_STATUS_CHALLENGE_FAIL; goto auth_fail; } @@ -1400,7 +1406,11 @@ unsigned int OnAssocRsp(struct adapter *padapter, union recv_frame *precv_frame) /* to handle HT, WMM, rate adaptive, update MAC reg */ /* for not to handle the synchronous IO in the tasklet */ for (i = (6 + WLAN_HDR_A3_LEN); i < pkt_len;) { + if (i + sizeof(*pIE) > pkt_len) + break; pIE = (struct ndis_80211_var_ie *)(pframe + i); + if (i + sizeof(*pIE) + pIE->length > pkt_len) + break; switch (pIE->element_id) { case WLAN_EID_VENDOR_SPECIFIC: @@ -2925,7 +2935,11 @@ void issue_assocreq(struct adapter *padapter) /* vendor specific IE, such as WPA, WMM, WPS */ for (i = sizeof(struct ndis_802_11_fix_ie); i < pmlmeinfo->network.ie_length;) { + if (i + sizeof(*pIE) > pmlmeinfo->network.ie_length) + break; pIE = (struct ndis_80211_var_ie *)(pmlmeinfo->network.ies + i); + if (i + sizeof(*pIE) + pIE->length > pmlmeinfo->network.ie_length) + break; switch (pIE->element_id) { case WLAN_EID_VENDOR_SPECIFIC: @@ -5318,7 +5332,11 @@ u8 join_cmd_hdl(struct adapter *padapter, u8 *pbuf) /* sizeof(struct ndis_802_11_fix_ie) */ for (i = _FIXED_IE_LENGTH_; i < pnetwork->ie_length;) { + if (i + sizeof(*pIE) > pnetwork->ie_length) + break; pIE = (struct ndis_80211_var_ie *)(pnetwork->ies + i); + if (i + sizeof(*pIE) + pIE->length > pnetwork->ie_length) + break; switch (pIE->element_id) { case WLAN_EID_VENDOR_SPECIFIC:/* Get WMM IE. */ diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c index a00504ff29109..f467cb5b1dca3 100644 --- a/drivers/staging/rtl8723bs/core/rtw_security.c +++ b/drivers/staging/rtl8723bs/core/rtw_security.c @@ -1212,7 +1212,7 @@ u32 rtw_aes_decrypt(struct adapter *padapter, u8 *precvframe) if (prxattrib->encrypt != _AES_) return _SUCCESS; stainfo = rtw_get_stainfo(&padapter->stapriv, &prxattrib->ta[0]); - if (stainfo) + if (!stainfo) return _FAIL; if (is_multicast_ether_addr(prxattrib->ra)) { static unsigned long start; diff --git a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c index 6a7c09db4cd93..94bbe7ac13ac7 100644 --- a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c +++ b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c @@ -936,7 +936,8 @@ void HT_caps_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE) pmlmeinfo->HT_caps_enable = 1; - for (i = 0; i < (pIE->length); i++) { + for (i = 0; i < umin(pIE->length, + sizeof(pmlmeinfo->HT_caps.u.HT_cap)); i++) { if (i != 2) { /* Commented by Albert 2010/07/12 */ /* Got the endian issue here. */ @@ -1289,7 +1290,11 @@ void update_beacon_info(struct adapter *padapter, u8 *pframe, uint pkt_len, stru len = pkt_len - (_BEACON_IE_OFFSET_ + WLAN_HDR_A3_LEN); for (i = 0; i < len;) { + if (i + sizeof(*pIE) > len) + break; pIE = (struct ndis_80211_var_ie *)(pframe + (_BEACON_IE_OFFSET_ + WLAN_HDR_A3_LEN) + i); + if (i + sizeof(*pIE) + pIE->length > len) + break; switch (pIE->element_id) { case WLAN_EID_VENDOR_SPECIFIC: @@ -1314,7 +1319,7 @@ void update_beacon_info(struct adapter *padapter, u8 *pframe, uint pkt_len, stru break; } - i += (pIE->length + 2); + i += sizeof(*pIE) + pIE->length; } } @@ -1330,15 +1335,23 @@ unsigned int is_ap_in_tkip(struct adapter *padapter) for (i = sizeof(struct ndis_802_11_fix_ie); i < pmlmeinfo->network.ie_length;) { pIE = (struct ndis_80211_var_ie *)(pmlmeinfo->network.ies + i); + if (i + sizeof(*pIE) > pmlmeinfo->network.ie_length) + break; + if (i + sizeof(*pIE) + pIE->length > pmlmeinfo->network.ie_length) + break; + switch (pIE->element_id) { case WLAN_EID_VENDOR_SPECIFIC: - if ((!memcmp(pIE->data, RTW_WPA_OUI, 4)) && (!memcmp((pIE->data + 12), WPA_TKIP_CIPHER, 4))) + if (pIE->length >= 16 && + !memcmp(pIE->data, RTW_WPA_OUI, 4) && + !memcmp((pIE->data + 12), WPA_TKIP_CIPHER, 4)) return true; break; case WLAN_EID_RSN: - if (!memcmp((pIE->data + 8), RSN_TKIP_CIPHER, 4)) + if (pIE->length >= 12 && + !memcmp((pIE->data + 8), RSN_TKIP_CIPHER, 4)) return true; break; @@ -1346,7 +1359,7 @@ unsigned int is_ap_in_tkip(struct adapter *padapter) break; } - i += (pIE->length + 2); + i += sizeof(*pIE) + pIE->length; } return false; diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c index fd3bae31b0edb..e7ba5ccfa03cf 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c @@ -1445,6 +1445,10 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel pwpa = rtw_get_wpa_ie(buf, &wpa_ielen, ielen); if (pwpa && wpa_ielen > 0) { + if (wpa_ielen + 2 > sizeof(padapter->securitypriv.supplicant_ie)) { + ret = -EINVAL; + goto exit; + } if (rtw_parse_wpa_ie(pwpa, wpa_ielen + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) { padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X; padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPAPSK; @@ -1454,6 +1458,10 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel pwpa2 = rtw_get_wpa2_ie(buf, &wpa2_ielen, ielen); if (pwpa2 && wpa2_ielen > 0) { + if (wpa2_ielen + 2 > sizeof(padapter->securitypriv.supplicant_ie)) { + ret = -EINVAL; + goto exit; + } if (rtw_parse_wpa2_ie(pwpa2, wpa2_ielen + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) { padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X; padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPA2PSK; diff --git a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c index dc0b77f38b1ae..9bdb67a8a408b 100644 --- a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c +++ b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c @@ -24,9 +24,11 @@ void _rtw_open_pktfile(struct sk_buff *pktptr, struct pkt_file *pfile) int _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, unsigned int rlen) { int ret; + unsigned int remain = rtw_remainder_len(pfile); - if (rtw_remainder_len(pfile) < rlen) - return -EINVAL; + /* clamp to bytes remaining; the coalesce loop relies on short reads */ + if (rlen > remain) + rlen = remain; if (rmem) { ret = skb_copy_bits(pfile->pkt, pfile->buf_len - pfile->pkt_len, rmem, rlen); diff --git a/drivers/staging/vme_user/vme_fake.c b/drivers/staging/vme_user/vme_fake.c index 8abaa3165fbb7..434cf760ade6c 100644 --- a/drivers/staging/vme_user/vme_fake.c +++ b/drivers/staging/vme_user/vme_fake.c @@ -1239,6 +1239,7 @@ static void __exit fake_exit(void) { struct list_head *pos = NULL; struct list_head *tmplist; + struct vme_lm_resource *lm; struct vme_master_resource *master_image; struct vme_slave_resource *slave_image; int i; @@ -1268,6 +1269,13 @@ static void __exit fake_exit(void) vme_unregister_bridge(fake_bridge); fake_crcsr_exit(fake_bridge); + /* resources are stored in link list */ + list_for_each_safe(pos, tmplist, &fake_bridge->lm_resources) { + lm = list_entry(pos, struct vme_lm_resource, list); + list_del(pos); + kfree(lm); + } + /* resources are stored in link list */ list_for_each_safe(pos, tmplist, &fake_bridge->slave_resources) { slave_image = list_entry(pos, struct vme_slave_resource, list); diff --git a/drivers/staging/vme_user/vme_tsi148.c b/drivers/staging/vme_user/vme_tsi148.c index 4cf3486646cee..c695ad9b4ca26 100644 --- a/drivers/staging/vme_user/vme_tsi148.c +++ b/drivers/staging/vme_user/vme_tsi148.c @@ -2534,6 +2534,7 @@ static void tsi148_remove(struct pci_dev *pdev) { struct list_head *pos = NULL; struct list_head *tmplist; + struct vme_lm_resource *lm; struct vme_master_resource *master_image; struct vme_slave_resource *slave_image; struct vme_dma_resource *dma_ctrlr; @@ -2590,6 +2591,13 @@ static void tsi148_remove(struct pci_dev *pdev) tsi148_crcsr_exit(tsi148_bridge, pdev); + /* resources are stored in link list */ + list_for_each_safe(pos, tmplist, &tsi148_bridge->lm_resources) { + lm = list_entry(pos, struct vme_lm_resource, list); + list_del(pos); + kfree(lm); + } + /* resources are stored in link list */ list_for_each_safe(pos, tmplist, &tsi148_bridge->dma_resources) { dma_ctrlr = list_entry(pos, struct vme_dma_resource, list); diff --git a/drivers/staging/vme_user/vme_user.c b/drivers/staging/vme_user/vme_user.c index 11e25c2f6b0aa..a472a38ef613a 100644 --- a/drivers/staging/vme_user/vme_user.c +++ b/drivers/staging/vme_user/vme_user.c @@ -156,6 +156,17 @@ static ssize_t buffer_to_user(unsigned int minor, char __user *buf, { void *image_ptr; + /* + * The slave window (image_size) can exceed the fixed kern_buf + * (size_buf == PCI_BUF_SIZE), so bound the copy to kern_buf. + * *ppos is >= 0 here (checked by the caller), so the + * subtraction below cannot wrap. + */ + if (*ppos >= image[minor].size_buf) + return 0; + if (count > image[minor].size_buf - *ppos) + count = image[minor].size_buf - *ppos; + image_ptr = image[minor].kern_buf + *ppos; if (copy_to_user(buf, image_ptr, (unsigned long)count)) return -EFAULT; @@ -168,6 +179,17 @@ static ssize_t buffer_from_user(unsigned int minor, const char __user *buf, { void *image_ptr; + /* + * The slave window (image_size) can exceed the fixed kern_buf + * (size_buf == PCI_BUF_SIZE), so bound the copy to kern_buf. + * *ppos is >= 0 here (checked by the caller), so the + * subtraction below cannot wrap. + */ + if (*ppos >= image[minor].size_buf) + return 0; + if (count > image[minor].size_buf - *ppos) + count = image[minor].size_buf - *ppos; + image_ptr = image[minor].kern_buf + *ppos; if (copy_from_user(image_ptr, buf, (unsigned long)count)) return -EFAULT; diff --git a/drivers/target/iscsi/iscsi_target_auth.c b/drivers/target/iscsi/iscsi_target_auth.c index a3ad2d244dbee..5858cc3089796 100644 --- a/drivers/target/iscsi/iscsi_target_auth.c +++ b/drivers/target/iscsi/iscsi_target_auth.c @@ -438,9 +438,11 @@ static int chap_server_compute_hash( } if (type == HEX) - ret = kstrtoul(&identifier[2], 0, &id); + ret = kstrtoul(identifier, 16, &id); + else if (type == DECIMAL) + ret = kstrtoul(identifier, 10, &id); else - ret = kstrtoul(identifier, 0, &id); + ret = -EINVAL; if (ret < 0) { pr_err("kstrtoul() failed for CHAP identifier: %d\n", ret); diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c index 1102973457513..d29830b951f75 100644 --- a/drivers/target/loopback/tcm_loop.c +++ b/drivers/target/loopback/tcm_loop.c @@ -270,69 +270,6 @@ static int tcm_loop_device_reset(struct scsi_cmnd *sc) return (ret == TMR_FUNCTION_COMPLETE) ? SUCCESS : FAILED; } -static bool tcm_loop_flush_work_iter(struct request *rq, void *data) -{ - struct scsi_cmnd *sc = blk_mq_rq_to_pdu(rq); - struct tcm_loop_cmd *tl_cmd = scsi_cmd_priv(sc); - struct se_cmd *se_cmd = &tl_cmd->tl_se_cmd; - - flush_work(&se_cmd->work); - return true; -} - -static int tcm_loop_target_reset(struct scsi_cmnd *sc) -{ - struct tcm_loop_hba *tl_hba; - struct tcm_loop_tpg *tl_tpg; - struct Scsi_Host *sh = sc->device->host; - int ret; - - /* - * Locate the tcm_loop_hba_t pointer - */ - tl_hba = *(struct tcm_loop_hba **)shost_priv(sh); - if (!tl_hba) { - pr_err("Unable to perform device reset without active I_T Nexus\n"); - return FAILED; - } - /* - * Locate the tl_tpg pointer from TargetID in sc->device->id - */ - tl_tpg = &tl_hba->tl_hba_tpgs[sc->device->id]; - if (!tl_tpg) - return FAILED; - - /* - * Issue a LUN_RESET to drain all commands that the target core - * knows about. This handles commands not yet marked CMD_T_COMPLETE. - */ - ret = tcm_loop_issue_tmr(tl_tpg, sc->device->lun, 0, TMR_LUN_RESET); - if (ret != TMR_FUNCTION_COMPLETE) - return FAILED; - - /* - * Flush any deferred target core completion work that may still be - * queued. Commands that already had CMD_T_COMPLETE set before the TMR - * are skipped by the TMR drain, but their async completion work - * (transport_lun_remove_cmd → percpu_ref_put, release_cmd → scsi_done) - * may still be pending in target_completion_wq. - * - * The SCSI EH will reuse in-flight scsi_cmnd structures for recovery - * commands (e.g. TUR) immediately after this handler returns SUCCESS — - * if deferred work is still pending, the memset in queuecommand would - * zero the se_cmd while the work accesses it, leaking the LUN - * percpu_ref and hanging configfs unlink forever. - * - * Use blk_mq_tagset_busy_iter() to find all started requests and - * flush_work() on each — the same pattern used by mpi3mr, scsi_debug, - * and other SCSI drivers to drain outstanding commands during reset. - */ - blk_mq_tagset_busy_iter(&sh->tag_set, tcm_loop_flush_work_iter, NULL); - - tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE; - return SUCCESS; -} - static const struct scsi_host_template tcm_loop_driver_template = { .show_info = tcm_loop_show_info, .proc_name = "tcm_loopback", @@ -341,7 +278,6 @@ static const struct scsi_host_template tcm_loop_driver_template = { .change_queue_depth = scsi_change_queue_depth, .eh_abort_handler = tcm_loop_abort_task, .eh_device_reset_handler = tcm_loop_device_reset, - .eh_target_reset_handler = tcm_loop_target_reset, .this_id = -1, .sg_tablesize = 256, .max_sectors = 0xFFFF, diff --git a/drivers/target/target_core_fabric_lib.c b/drivers/target/target_core_fabric_lib.c index 87c5d26a50895..2853b95b2c597 100644 --- a/drivers/target/target_core_fabric_lib.c +++ b/drivers/target/target_core_fabric_lib.c @@ -290,13 +290,24 @@ static void sbp_parse_pr_out_transport_id(char *buf, char *i_str) static bool iscsi_parse_pr_out_transport_id( struct se_portal_group *se_tpg, char *buf, + u32 buf_len, u32 *out_tid_len, char **port_nexus_ptr, char *i_str) { char *p; + u32 tid_len; int i; - u8 format_code = (buf[0] & 0xc0); + u8 format_code; + + /* + * The 4-byte iSCSI TransportID header (FORMAT CODE + 2-byte ADDITIONAL + * LENGTH) must be present before any of it can be parsed. + */ + if (buf_len < 4) + return false; + + format_code = buf[0] & 0xc0; /* * Check for FORMAT CODE 00b or 01b from spc4r17, section 7.5.4.6: * @@ -316,15 +327,17 @@ static bool iscsi_parse_pr_out_transport_id( return false; } /* - * If the caller wants the TransportID Length, we set that value for the - * entire iSCSI Tarnsport ID now. + * Reconstruct the self-described TransportID length from the ADDITIONAL + * LENGTH field plus the 4-byte header. Reject it if it is below the + * spc4r17 section 7.5.4.6 minimum (ADDITIONAL LENGTH shall be at least + * 20) or if it runs past the bytes actually received, so that every + * access below stays inside the TransportID. */ - if (out_tid_len) { - /* The shift works thanks to integer promotion rules */ - *out_tid_len = get_unaligned_be16(&buf[2]); - /* Add four bytes for iSCSI Transport ID header */ - *out_tid_len += 4; - } + tid_len = get_unaligned_be16(&buf[2]) + 4; + if (tid_len < 24 || tid_len > buf_len) + return false; + if (out_tid_len) + *out_tid_len = tid_len; /* * Check for ',i,0x' separator between iSCSI Name and iSCSI Initiator @@ -332,16 +345,32 @@ static bool iscsi_parse_pr_out_transport_id( * format. */ if (format_code == 0x40) { - p = strstr(&buf[4], ",i,0x"); + p = strnstr(&buf[4], ",i,0x", tid_len - 4); if (!p) { - pr_err("Unable to locate \",i,0x\" separator" - " for Initiator port identifier: %s\n", - &buf[4]); + pr_err("Unable to locate \",i,0x\" separator in iSCSI TransportID\n"); + return false; + } + /* + * The iSCSI name runs from &buf[4] up to the separator; reject it + * if it cannot fit in i_str[TRANSPORT_IQN_LEN]. + */ + if (p - &buf[4] >= TRANSPORT_IQN_LEN) { + pr_err("iSCSI Initiator port name too long in TransportID\n"); return false; } *p = '\0'; /* Terminate iSCSI Name */ p += 5; /* Skip over ",i,0x" separator */ + /* + * The ISID must follow the separator. A ",i,0x" sitting at the + * very end of the TransportID leaves no ISID and would point the + * port nexus at buf + tid_len, i.e. past the descriptor, which + * the registration code then reads as the ISID string. + */ + if (p >= buf + tid_len) { + pr_err("Missing ISID in iSCSI Initiator port TransportID\n"); + return false; + } *port_nexus_ptr = p; /* * Go ahead and do the lower case conversion of the received @@ -349,7 +378,7 @@ static bool iscsi_parse_pr_out_transport_id( * for comparison against the running iSCSI session's ISID from * iscsi_target.c:lio_sess_get_initiator_sid() */ - for (i = 0; i < 12; i++) { + for (i = 0; i < 12 && p < buf + tid_len; i++) { /* * The first ISCSI INITIATOR SESSION ID field byte * containing an ASCII null character terminates the @@ -367,10 +396,22 @@ static bool iscsi_parse_pr_out_transport_id( *p = tolower(*p); p++; } - } else + strscpy(i_str, &buf[4], TRANSPORT_IQN_LEN); + } else { *port_nexus_ptr = NULL; - - strscpy(i_str, &buf[4], TRANSPORT_IQN_LEN); + /* + * FORMAT CODE 00b: the name occupies buf[4..tid_len-1]. The + * declared length tid_len - 4 must fit in i_str[TRANSPORT_IQN_LEN]. + * (For 01b the same tid_len bound would be over-restrictive: the + * descriptor also carries the separator and ISID, so a legal + * <=223-byte name gives tid_len up to 244.) + */ + if (tid_len - 4 >= TRANSPORT_IQN_LEN) { + pr_err("iSCSI Initiator port name too long in TransportID\n"); + return false; + } + strscpy(i_str, &buf[4], tid_len - 4); + } return true; } @@ -420,8 +461,16 @@ int target_get_pr_transport_id(struct se_node_acl *nacl, } bool target_parse_pr_out_transport_id(struct se_portal_group *tpg, - char *buf, u32 *out_tid_len, char **port_nexus_ptr, char *i_str) + char *buf, u32 buf_len, u32 *out_tid_len, + char **port_nexus_ptr, char *i_str) { + /* + * The fixed-length SAS/SRP/FCP/SBP TransportIDs are 24 bytes; the iSCSI + * format is variable and bounds itself against buf_len below. + */ + if (tpg->proto_id != SCSI_PROTOCOL_ISCSI && buf_len < 24) + return false; + switch (tpg->proto_id) { case SCSI_PROTOCOL_SAS: /* @@ -440,8 +489,8 @@ bool target_parse_pr_out_transport_id(struct se_portal_group *tpg, sbp_parse_pr_out_transport_id(buf, i_str); break; case SCSI_PROTOCOL_ISCSI: - return iscsi_parse_pr_out_transport_id(tpg, buf, out_tid_len, - port_nexus_ptr, i_str); + return iscsi_parse_pr_out_transport_id(tpg, buf, buf_len, + out_tid_len, port_nexus_ptr, i_str); default: pr_err("Unknown proto_id: 0x%02x\n", tpg->proto_id); return false; diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h index 763e6d26e1877..f0886ea290345 100644 --- a/drivers/target/target_core_internal.h +++ b/drivers/target/target_core_internal.h @@ -104,7 +104,8 @@ int target_get_pr_transport_id(struct se_node_acl *nacl, struct t10_pr_registration *pr_reg, int *format_code, unsigned char *buf); bool target_parse_pr_out_transport_id(struct se_portal_group *tpg, - char *buf, u32 *out_tid_len, char **port_nexus_ptr, char *i_str); + char *buf, u32 buf_len, u32 *out_tid_len, + char **port_nexus_ptr, char *i_str); /* target_core_hba.c */ struct se_hba *core_alloc_hba(const char *, u32, u32); diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index 11790f2c5d80f..1a77b4bb62b00 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -1573,7 +1573,7 @@ core_scsi3_decode_spec_i_port( iport_ptr = NULL; tid_found = target_parse_pr_out_transport_id(tmp_tpg, - ptr, &tid_len, &iport_ptr, i_str); + ptr, tpdl, &tid_len, &iport_ptr, i_str); if (!tid_found) continue; /* @@ -3285,7 +3285,7 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key, goto out; } tid_found = target_parse_pr_out_transport_id(dest_se_tpg, - &buf[24], &tmp_tid_len, &iport_ptr, initiator_str); + &buf[24], tid_len, &tmp_tid_len, &iport_ptr, initiator_str); if (!tid_found) { pr_err("SPC-3 PR REGISTER_AND_MOVE: Unable to locate" " initiator_str from Transport ID\n"); @@ -3293,9 +3293,6 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key, goto out; } - transport_kunmap_data_sg(cmd); - buf = NULL; - pr_debug("SPC-3 PR [%s] Extracted initiator %s identifier: %s" " %s\n", dest_tf_ops->fabric_name, (iport_ptr != NULL) ? "port" : "device", initiator_str, (iport_ptr != NULL) ? @@ -3532,6 +3529,11 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key, core_scsi3_update_and_write_aptpl(cmd->se_dev, aptpl); core_scsi3_put_pr_reg(dest_pr_reg); + /* + * iport_ptr aliases the PR-OUT parameter list mapped above, so the + * buffer is unmapped only here on success (and at out: on error). + */ + transport_kunmap_data_sg(cmd); return 0; out: if (buf) diff --git a/drivers/thermal/intel/therm_throt.c b/drivers/thermal/intel/therm_throt.c index 44fa4dd15dd1c..45a8ef4a608b1 100644 --- a/drivers/thermal/intel/therm_throt.c +++ b/drivers/thermal/intel/therm_throt.c @@ -529,8 +529,13 @@ static int thermal_throttle_online(unsigned int cpu) { struct thermal_state *state = &per_cpu(thermal_state, cpu); struct device *dev = get_cpu_device(cpu); + int err; u32 l; + err = thermal_throttle_add_dev(dev, cpu); + if (err) + return err; + state->package_throttle.level = PACKAGE_LEVEL; state->core_throttle.level = CORE_LEVEL; @@ -548,7 +553,7 @@ static int thermal_throttle_online(unsigned int cpu) l = apic_read(APIC_LVTTHMR); apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED); - return thermal_throttle_add_dev(dev, cpu); + return err; } static int thermal_throttle_offline(unsigned int cpu) diff --git a/drivers/thermal/testing/command.c b/drivers/thermal/testing/command.c index 1159ecea57e7c..90e06c5933278 100644 --- a/drivers/thermal/testing/command.c +++ b/drivers/thermal/testing/command.c @@ -86,7 +86,10 @@ #include "thermal_testing.h" -struct dentry *d_testing; +struct workqueue_struct *tt_wq __ro_after_init; + +struct dentry *d_testing __ro_after_init; +static struct dentry *d_command __ro_after_init; #define TT_COMMAND_SIZE 16 @@ -116,18 +119,30 @@ static int tt_command_exec(int index, const char *arg) break; case TT_CMD_DELTZ: + if (!arg || !*arg) + return -EINVAL; + ret = tt_del_tz(arg); break; case TT_CMD_TZADDTRIP: + if (!arg || !*arg) + return -EINVAL; + ret = tt_zone_add_trip(arg); break; case TT_CMD_TZREG: + if (!arg || !*arg) + return -EINVAL; + ret = tt_zone_reg(arg); break; case TT_CMD_TZUNREG: + if (!arg || !*arg) + return -EINVAL; + ret = tt_zone_unreg(arg); break; @@ -191,17 +206,42 @@ static const struct file_operations tt_command_fops = { static int __init thermal_testing_init(void) { + int error; + + tt_wq = alloc_workqueue("thermal_testing", WQ_UNBOUND, 0); + if (!tt_wq) + return -ENOMEM; + d_testing = debugfs_create_dir("thermal-testing", NULL); - if (!IS_ERR(d_testing)) - debugfs_create_file("command", 0200, d_testing, NULL, - &tt_command_fops); + if (IS_ERR(d_testing)) { + error = PTR_ERR(d_testing); + goto destroy_wq; + } + + d_command = debugfs_create_file("command", 0200, d_testing, NULL, &tt_command_fops); + if (IS_ERR(d_command)) { + error = PTR_ERR(d_command); + goto remove_d_testing; + } return 0; + +remove_d_testing: + debugfs_remove(d_testing); +destroy_wq: + destroy_workqueue(tt_wq); + return error; } module_init(thermal_testing_init); static void __exit thermal_testing_exit(void) { + /* First, prevent new commands from being entered. */ + debugfs_remove(d_command); + /* Flush commands in progress (if any). */ + flush_workqueue(tt_wq); + destroy_workqueue(tt_wq); + /* Remove the directory structure and clean up. */ debugfs_remove(d_testing); tt_zone_cleanup(); } diff --git a/drivers/thermal/testing/thermal_testing.h b/drivers/thermal/testing/thermal_testing.h index c790a32aae4ed..5880c9a63dba9 100644 --- a/drivers/thermal/testing/thermal_testing.h +++ b/drivers/thermal/testing/thermal_testing.h @@ -1,4 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include + +extern struct workqueue_struct *tt_wq; + +static inline void tt_queue_work(struct work_struct *work) +{ + queue_work(tt_wq, work); +} extern struct dentry *d_testing; diff --git a/drivers/thermal/testing/zone.c b/drivers/thermal/testing/zone.c index 3c339242f52d4..51f1c0806d214 100644 --- a/drivers/thermal/testing/zone.c +++ b/drivers/thermal/testing/zone.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "thermal_testing.h" @@ -207,7 +206,7 @@ int tt_add_tz(void) INIT_WORK(&tt_work->work, tt_add_tz_work_fn); tt_work->tt_zone = no_free_ptr(tt_zone); - schedule_work(&(no_free_ptr(tt_work)->work)); + tt_queue_work(&(no_free_ptr(tt_work)->work)); return 0; } @@ -269,7 +268,7 @@ int tt_del_tz(const char *arg) INIT_WORK(&tt_work->work, tt_del_tz_work_fn); tt_work->tt_zone = tt_zone; - schedule_work(&(no_free_ptr(tt_work)->work)); + tt_queue_work(&(no_free_ptr(tt_work)->work)); return 0; } @@ -358,7 +357,7 @@ int tt_zone_add_trip(const char *arg) INIT_WORK(&tt_work->work, tt_zone_add_trip_work_fn); tt_work->tt_zone = no_free_ptr(tt_zone); tt_work->tt_trip = no_free_ptr(tt_trip); - schedule_work(&(no_free_ptr(tt_work)->work)); + tt_queue_work(&(no_free_ptr(tt_work)->work)); return 0; } diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index b624892bc6d66..223ae1571655b 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -19,27 +19,31 @@ #include "thermal_hwmon.h" #include "thermal_core.h" -/* hwmon sys I/F */ -/* thermal zone devices with the same type share one hwmon device */ -struct thermal_hwmon_device { - char type[THERMAL_NAME_LENGTH]; - struct device *device; - int count; - struct list_head tz_list; - struct list_head node; -}; +/* + * Needs to be large enough to hold a thermal zone type string followed by an + * underline character and a 32-bit integer in decimal representation. + */ +#define THERMAL_HWMON_NAME_LENGTH (THERMAL_NAME_LENGTH + 11) struct thermal_hwmon_attr { struct device_attribute attr; - char name[16]; }; /* one temperature input for each thermal zone */ struct thermal_hwmon_temp { - struct list_head hwmon_node; struct thermal_zone_device *tz; struct thermal_hwmon_attr temp_input; /* hwmon sys attr */ struct thermal_hwmon_attr temp_crit; /* hwmon sys attr */ + bool temp_crit_present; +}; + +/* hwmon sys I/F */ +/* thermal zone devices with the same type share one hwmon device */ +struct thermal_hwmon_device { + char name[THERMAL_HWMON_NAME_LENGTH]; + struct device *device; + struct list_head node; + struct thermal_hwmon_temp tz_temp; }; static LIST_HEAD(thermal_hwmon_list); @@ -87,45 +91,6 @@ temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf) return sysfs_emit(buf, "%d\n", temperature); } - -static struct thermal_hwmon_device * -thermal_hwmon_lookup_by_type(const struct thermal_zone_device *tz) -{ - struct thermal_hwmon_device *hwmon; - char type[THERMAL_NAME_LENGTH]; - - mutex_lock(&thermal_hwmon_list_lock); - list_for_each_entry(hwmon, &thermal_hwmon_list, node) { - strscpy(type, tz->type); - strreplace(type, '-', '_'); - if (!strcmp(hwmon->type, type)) { - mutex_unlock(&thermal_hwmon_list_lock); - return hwmon; - } - } - mutex_unlock(&thermal_hwmon_list_lock); - - return NULL; -} - -/* Find the temperature input matching a given thermal zone */ -static struct thermal_hwmon_temp * -thermal_hwmon_lookup_temp(const struct thermal_hwmon_device *hwmon, - const struct thermal_zone_device *tz) -{ - struct thermal_hwmon_temp *temp; - - mutex_lock(&thermal_hwmon_list_lock); - list_for_each_entry(temp, &hwmon->tz_list, hwmon_node) - if (temp->tz == tz) { - mutex_unlock(&thermal_hwmon_list_lock); - return temp; - } - mutex_unlock(&thermal_hwmon_list_lock); - - return NULL; -} - static bool thermal_zone_crit_temp_valid(struct thermal_zone_device *tz) { int temp; @@ -136,54 +101,39 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) { struct thermal_hwmon_device *hwmon; struct thermal_hwmon_temp *temp; - int new_hwmon_device = 1; int result; - hwmon = thermal_hwmon_lookup_by_type(tz); - if (hwmon) { - new_hwmon_device = 0; - goto register_sys_interface; - } - hwmon = kzalloc_obj(*hwmon); if (!hwmon) return -ENOMEM; - INIT_LIST_HEAD(&hwmon->tz_list); - strscpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH); - strreplace(hwmon->type, '-', '_'); + /* + * Append the thermal zone ID preceded by an underline character to the + * type to disambiguate the sensors command output. + */ + scnprintf(hwmon->name, THERMAL_HWMON_NAME_LENGTH, "%s_%d", tz->type, tz->id); + strreplace(hwmon->name, '-', '_'); hwmon->device = hwmon_device_register_for_thermal(&tz->device, - hwmon->type, hwmon); + hwmon->name, hwmon); if (IS_ERR(hwmon->device)) { result = PTR_ERR(hwmon->device); goto free_mem; } - register_sys_interface: - temp = kzalloc_obj(*temp); - if (!temp) { - result = -ENOMEM; - goto unregister_name; - } + temp = &hwmon->tz_temp; temp->tz = tz; - hwmon->count++; - snprintf(temp->temp_input.name, sizeof(temp->temp_input.name), - "temp%d_input", hwmon->count); - temp->temp_input.attr.attr.name = temp->temp_input.name; + temp->temp_input.attr.attr.name = "temp1_input"; temp->temp_input.attr.attr.mode = 0444; temp->temp_input.attr.show = temp_input_show; sysfs_attr_init(&temp->temp_input.attr.attr); result = device_create_file(hwmon->device, &temp->temp_input.attr); if (result) - goto free_temp_mem; + goto unregister_name; if (thermal_zone_crit_temp_valid(tz)) { - snprintf(temp->temp_crit.name, - sizeof(temp->temp_crit.name), - "temp%d_crit", hwmon->count); - temp->temp_crit.attr.attr.name = temp->temp_crit.name; + temp->temp_crit.attr.attr.name = "temp1_crit"; temp->temp_crit.attr.attr.mode = 0444; temp->temp_crit.attr.show = temp_crit_show; sysfs_attr_init(&temp->temp_crit.attr.attr); @@ -191,23 +141,21 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) &temp->temp_crit.attr); if (result) goto unregister_input; + + temp->temp_crit_present = true; } + /* The list is needed for hwmon lookup during removal. */ mutex_lock(&thermal_hwmon_list_lock); - if (new_hwmon_device) - list_add_tail(&hwmon->node, &thermal_hwmon_list); - list_add_tail(&temp->hwmon_node, &hwmon->tz_list); + list_add_tail(&hwmon->node, &thermal_hwmon_list); mutex_unlock(&thermal_hwmon_list_lock); return 0; unregister_input: device_remove_file(hwmon->device, &temp->temp_input.attr); - free_temp_mem: - kfree(temp); unregister_name: - if (new_hwmon_device) - hwmon_device_unregister(hwmon->device); + hwmon_device_unregister(hwmon->device); free_mem: kfree(hwmon); @@ -215,39 +163,37 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) } EXPORT_SYMBOL_GPL(thermal_add_hwmon_sysfs); +static struct thermal_hwmon_device * +thermal_hwmon_lookup(const struct thermal_zone_device *tz) +{ + struct thermal_hwmon_device *hwmon; + + list_for_each_entry(hwmon, &thermal_hwmon_list, node) { + if (hwmon->tz_temp.tz == tz) + return hwmon; + } + return NULL; +} + void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz) { struct thermal_hwmon_device *hwmon; struct thermal_hwmon_temp *temp; - hwmon = thermal_hwmon_lookup_by_type(tz); - if (unlikely(!hwmon)) { - /* Should never happen... */ - dev_dbg(&tz->device, "hwmon device lookup failed!\n"); - return; - } + scoped_guard(mutex, &thermal_hwmon_list_lock) { + hwmon = thermal_hwmon_lookup(tz); + if (!hwmon) + return; - temp = thermal_hwmon_lookup_temp(hwmon, tz); - if (unlikely(!temp)) { - /* Should never happen... */ - dev_dbg(&tz->device, "temperature input lookup failed!\n"); - return; + list_del(&hwmon->node); } + temp = &hwmon->tz_temp; + device_remove_file(hwmon->device, &temp->temp_input.attr); - if (thermal_zone_crit_temp_valid(tz)) + if (temp->temp_crit_present) device_remove_file(hwmon->device, &temp->temp_crit.attr); - mutex_lock(&thermal_hwmon_list_lock); - list_del(&temp->hwmon_node); - kfree(temp); - if (!list_empty(&hwmon->tz_list)) { - mutex_unlock(&thermal_hwmon_list_lock); - return; - } - list_del(&hwmon->node); - mutex_unlock(&thermal_hwmon_list_lock); - hwmon_device_unregister(hwmon->device); kfree(hwmon); } diff --git a/drivers/thunderbolt/debugfs.c b/drivers/thunderbolt/debugfs.c index 042f6a0d0f7f0..25f6ea6ea0947 100644 --- a/drivers/thunderbolt/debugfs.c +++ b/drivers/thunderbolt/debugfs.c @@ -956,7 +956,9 @@ margining_error_counter_write(struct file *file, const char __user *user_buf, else if (!strcmp(buf, "stop")) error_counter = USB4_MARGIN_SW_ERROR_COUNTER_STOP; else - return -EINVAL; + goto err_free; + + free_page((unsigned long)buf); scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &tb->lock) { if (!margining->software) @@ -966,6 +968,10 @@ margining_error_counter_write(struct file *file, const char __user *user_buf, } return count; + +err_free: + free_page((unsigned long)buf); + return -EINVAL; } static int margining_error_counter_show(struct seq_file *s, void *not_used) diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index 2af0c4d0ad823..035b46e136666 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -850,8 +850,10 @@ static int dw8250_probe(struct platform_device *pdev) */ if (data->clk) { err = clk_notifier_register(data->clk, &data->clk_notifier); - if (err) + if (err) { + serial8250_unregister_port(data->data.line); return dev_err_probe(dev, err, "Failed to set the clock notifier\n"); + } queue_work(system_dfl_wq, &data->clk_work); } diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c index 8ec03863606e4..f88809ff370b7 100644 --- a/drivers/tty/serial/8250/8250_mid.c +++ b/drivers/tty/serial/8250/8250_mid.c @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -368,8 +369,16 @@ static const struct mid8250_board dnv_board = { .freq = 133333333, .base_baud = 115200, .bar = 1, - .setup = dnv_setup, - .exit = dnv_exit, + /* + * Errata: + * HSUART May Stop Functioning when DMA is Active. + * + * - Denverton document #572409, rev 3.4, DNV60 + * - Ice Lake Xeon D document #714070, ICXD65 + * - Snowridge document #731931, SNR44 + */ + .setup = PTR_IF(false, dnv_setup), + .exit = PTR_IF(false, dnv_exit), }; static const struct pci_device_id pci_ids[] = { diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index c552c6b9a037f..3c7775df27eff 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -944,11 +944,12 @@ static void __dma_rx_do_complete(struct uart_8250_port *p) dev_err(p->port.dev, "teardown incomplete\n"); } } + + dma->rx_running = 0; if (!count) goto out; ret = tty_insert_flip_string(tty_port, dma->rx_buf, count); - dma->rx_running = 0; p->port.icount.rx += ret; p->port.icount.buf_overrun += count - ret; out: diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c index ac7d3f197c3a5..09b9ab57d2b44 100644 --- a/drivers/tty/serial/max310x.c +++ b/drivers/tty/serial/max310x.c @@ -1212,6 +1212,17 @@ static int max310x_gpio_set(struct gpio_chip *chip, unsigned int offset, return 0; } +static int max310x_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) +{ + struct max310x_port *s = gpiochip_get_data(chip); + struct uart_port *port = &s->p[offset / 4].port; + unsigned int val; + + val = max310x_port_read(port, MAX310X_GPIOCFG_REG); + + return val & BIT(offset % 4) ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + static int max310x_gpio_direction_input(struct gpio_chip *chip, unsigned int offset) { struct max310x_port *s = gpiochip_get_data(chip); @@ -1421,6 +1432,7 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty s->gpio.owner = THIS_MODULE; s->gpio.parent = dev; s->gpio.label = devtype->name; + s->gpio.get_direction = max310x_gpio_get_direction; s->gpio.direction_input = max310x_gpio_direction_input; s->gpio.get = max310x_gpio_get; s->gpio.direction_output= max310x_gpio_direction_output; diff --git a/drivers/tty/serial/msm_serial.c b/drivers/tty/serial/msm_serial.c index 2e999cb9c9744..bfa44b01c3e97 100644 --- a/drivers/tty/serial/msm_serial.c +++ b/drivers/tty/serial/msm_serial.c @@ -1228,7 +1228,8 @@ static int msm_startup(struct uart_port *port) data |= MSM_UART_MR1_AUTO_RFR_LEVEL0 & rfr_level; msm_write(port, data, MSM_UART_MR1); - if (msm_port->is_uartdm) { + /* Disable DMA for console to prevent PIO/DMA collisions */ + if (msm_port->is_uartdm && !uart_console(port)) { msm_request_tx_dma(msm_port, msm_port->uart.mapbase); msm_request_rx_dma(msm_port, msm_port->uart.mapbase); } diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index 17da115b1e78b..3b0f836a98222 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -905,12 +905,9 @@ static void qcom_geni_serial_handle_rx_dma(struct uart_port *uport, bool drop) port->rx_dma_addr = 0; rx_in = readl(uport->membase + SE_DMA_RX_LEN_IN); - if (!rx_in) { - dev_warn(uport->dev, "serial engine reports 0 RX bytes in!\n"); - return; - } - - if (!drop) + if (!rx_in) + dev_warn_ratelimited(uport->dev, "serial engine reports 0 RX bytes in!\n"); + else if (!drop) handle_rx_uart(uport, rx_in); ret = geni_se_rx_dma_prep(&port->se, port->rx_buf, diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c index dfdea08421495..763a3f1b7be08 100644 --- a/drivers/tty/vt/keyboard.c +++ b/drivers/tty/vt/keyboard.c @@ -765,16 +765,22 @@ static void k_fn(struct vc_data *vc, unsigned char value, char up_flag) /* * Compute xterm-style modifier parameter for CSI sequences. * Returns 1 + (shift ? 1 : 0) + (alt ? 2 : 0) + (ctrl ? 4 : 0) + * + * Only the canonical modifier weights are counted. The left/right variants + * (KG_SHIFTL, KG_SHIFTR, KG_CTRLL, KG_CTRLR) and KG_ALTGR are commonly + * repurposed as keymap layout-group or level selectors rather than as plain + * modifiers (for instance XKB-derived keymaps select the layout group with + * KG_SHIFTL/KG_SHIFTR), so counting them would encode a spurious modifier. */ static int csi_modifier_param(void) { int mod = 1; - if (shift_state & (BIT(KG_SHIFT) | BIT(KG_SHIFTL) | BIT(KG_SHIFTR))) + if (shift_state & BIT(KG_SHIFT)) mod += 1; - if (shift_state & (BIT(KG_ALT) | BIT(KG_ALTGR))) + if (shift_state & BIT(KG_ALT)) mod += 2; - if (shift_state & (BIT(KG_CTRL) | BIT(KG_CTRLL) | BIT(KG_CTRLR))) + if (shift_state & BIT(KG_CTRL)) mod += 4; return mod; } diff --git a/drivers/tty/vt/vc_screen.c b/drivers/tty/vt/vc_screen.c index 4d2d46c95fefd..7d40eacc21b3e 100644 --- a/drivers/tty/vt/vc_screen.c +++ b/drivers/tty/vt/vc_screen.c @@ -686,7 +686,7 @@ vcs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) } *ppos += written; ret = written; - if (written) + if (written && vc) vcs_scr_updated(vc); return ret; diff --git a/drivers/ufs/core/ufs_trace.h b/drivers/ufs/core/ufs_trace.h index 309ae51b49063..377a3c54b9f56 100644 --- a/drivers/ufs/core/ufs_trace.h +++ b/drivers/ufs/core/ufs_trace.h @@ -89,16 +89,18 @@ TRACE_EVENT(ufshcd_clk_gating, TP_STRUCT__entry( __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __field(int, state) ), TP_fast_assign( + __assign_str(dev_name); __entry->hba = hba; __entry->state = state; ), TP_printk("%s: gating state changed to %s", - dev_name(__entry->hba->dev), + __get_str(dev_name), __print_symbolic(__entry->state, UFSCHD_CLK_GATING_STATES)) ); @@ -111,6 +113,7 @@ TRACE_EVENT(ufshcd_clk_scaling, TP_STRUCT__entry( __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __string(state, state) __string(clk, clk) __field(u32, prev_state) @@ -119,6 +122,7 @@ TRACE_EVENT(ufshcd_clk_scaling, TP_fast_assign( __entry->hba = hba; + __assign_str(dev_name); __assign_str(state); __assign_str(clk); __entry->prev_state = prev_state; @@ -126,7 +130,7 @@ TRACE_EVENT(ufshcd_clk_scaling, ), TP_printk("%s: %s %s from %u to %u Hz", - dev_name(__entry->hba->dev), __get_str(state), __get_str(clk), + __get_str(dev_name), __get_str(state), __get_str(clk), __entry->prev_state, __entry->curr_state) ); @@ -138,16 +142,18 @@ TRACE_EVENT(ufshcd_auto_bkops_state, TP_STRUCT__entry( __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __string(state, state) ), TP_fast_assign( __entry->hba = hba; + __assign_str(dev_name); __assign_str(state); ), TP_printk("%s: auto bkops - %s", - dev_name(__entry->hba->dev), __get_str(state)) + __get_str(dev_name), __get_str(state)) ); DECLARE_EVENT_CLASS(ufshcd_profiling_template, @@ -158,6 +164,7 @@ DECLARE_EVENT_CLASS(ufshcd_profiling_template, TP_STRUCT__entry( __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __string(profile_info, profile_info) __field(s64, time_us) __field(int, err) @@ -165,13 +172,14 @@ DECLARE_EVENT_CLASS(ufshcd_profiling_template, TP_fast_assign( __entry->hba = hba; + __assign_str(dev_name); __assign_str(profile_info); __entry->time_us = time_us; __entry->err = err; ), TP_printk("%s: %s: took %lld usecs, err %d", - dev_name(__entry->hba->dev), __get_str(profile_info), + __get_str(dev_name), __get_str(profile_info), __entry->time_us, __entry->err) ); @@ -200,6 +208,7 @@ DECLARE_EVENT_CLASS(ufshcd_template, __field(s64, usecs) __field(int, err) __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __field(int, dev_state) __field(int, link_state) ), @@ -208,13 +217,14 @@ DECLARE_EVENT_CLASS(ufshcd_template, __entry->usecs = usecs; __entry->err = err; __entry->hba = hba; + __assign_str(dev_name); __entry->dev_state = dev_state; __entry->link_state = link_state; ), TP_printk( "%s: took %lld usecs, dev_state: %s, link_state: %s, err %d", - dev_name(__entry->hba->dev), + __get_str(dev_name), __entry->usecs, __print_symbolic(__entry->dev_state, UFS_PWR_MODES), __print_symbolic(__entry->link_state, UFS_LINK_STATES), @@ -279,6 +289,7 @@ TRACE_EVENT(ufshcd_command, TP_STRUCT__entry( __field(struct scsi_device *, sdev) __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(&sdev->sdev_dev)) __field(enum ufs_trace_str_t, str_t) __field(unsigned int, tag) __field(u32, doorbell) @@ -291,6 +302,7 @@ TRACE_EVENT(ufshcd_command, ), TP_fast_assign( + __assign_str(dev_name); __entry->sdev = sdev; __entry->hba = hba; __entry->str_t = str_t; @@ -307,7 +319,7 @@ TRACE_EVENT(ufshcd_command, TP_printk( "%s: %s: tag: %u, DB: 0x%x, size: %d, IS: %u, LBA: %llu, opcode: 0x%x (%s), group_id: 0x%x, hwq_id: %d", show_ufs_cmd_trace_str(__entry->str_t), - dev_name(&__entry->sdev->sdev_dev), __entry->tag, + __get_str(dev_name), __entry->tag, __entry->doorbell, __entry->transfer_len, __entry->intr, __entry->lba, (u32)__entry->opcode, str_opcode(__entry->opcode), (u32)__entry->group_id, __entry->hwq_id @@ -322,6 +334,7 @@ TRACE_EVENT(ufshcd_uic_command, TP_STRUCT__entry( __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __field(enum ufs_trace_str_t, str_t) __field(u32, cmd) __field(u32, arg1) @@ -331,6 +344,7 @@ TRACE_EVENT(ufshcd_uic_command, TP_fast_assign( __entry->hba = hba; + __assign_str(dev_name); __entry->str_t = str_t; __entry->cmd = cmd; __entry->arg1 = arg1; @@ -340,7 +354,7 @@ TRACE_EVENT(ufshcd_uic_command, TP_printk( "%s: %s: cmd: 0x%x, arg1: 0x%x, arg2: 0x%x, arg3: 0x%x", - show_ufs_cmd_trace_str(__entry->str_t), dev_name(__entry->hba->dev), + show_ufs_cmd_trace_str(__entry->str_t), __get_str(dev_name), __entry->cmd, __entry->arg1, __entry->arg2, __entry->arg3 ) ); @@ -353,6 +367,7 @@ TRACE_EVENT(ufshcd_upiu, TP_STRUCT__entry( __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __field(enum ufs_trace_str_t, str_t) __array(unsigned char, hdr, 12) __array(unsigned char, tsf, 16) @@ -361,6 +376,7 @@ TRACE_EVENT(ufshcd_upiu, TP_fast_assign( __entry->hba = hba; + __assign_str(dev_name); __entry->str_t = str_t; memcpy(__entry->hdr, hdr, sizeof(__entry->hdr)); memcpy(__entry->tsf, tsf, sizeof(__entry->tsf)); @@ -369,7 +385,7 @@ TRACE_EVENT(ufshcd_upiu, TP_printk( "%s: %s: HDR:%s, %s:%s", - show_ufs_cmd_trace_str(__entry->str_t), dev_name(__entry->hba->dev), + show_ufs_cmd_trace_str(__entry->str_t), __get_str(dev_name), __print_hex(__entry->hdr, sizeof(__entry->hdr)), show_ufs_cmd_trace_tsf(__entry->tsf_t), __print_hex(__entry->tsf, sizeof(__entry->tsf)) @@ -384,16 +400,18 @@ TRACE_EVENT(ufshcd_exception_event, TP_STRUCT__entry( __field(struct ufs_hba *, hba) + __string(dev_name, dev_name(hba->dev)) __field(u16, status) ), TP_fast_assign( __entry->hba = hba; + __assign_str(dev_name); __entry->status = status; ), TP_printk("%s: status 0x%x", - dev_name(__entry->hba->dev), __entry->status + __get_str(dev_name), __entry->status ) ); diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index c3f08957d179a..84e32957c332d 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -5634,7 +5634,7 @@ static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, default: dev_err(hba->dev, "Unexpected request response code = %x\n", - result); + ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)); result = DID_ERROR << 16; break; } @@ -9466,22 +9466,44 @@ static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) { struct ufs_hba *hba = shost_priv(scmd->device->host); - if (!hba->system_suspending) { + if (!hba->pm_op_in_progress) { /* Activate the error handler in the SCSI core. */ return SCSI_EH_NOT_HANDLED; } /* - * If we get here we know that no TMFs are outstanding and also that - * the only pending command is a START STOP UNIT command. Handle the - * timeout of that command directly to prevent a deadlock between + * Handle the timeout directly to prevent a deadlock between * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler(). */ ufshcd_link_recovery(hba); dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", __func__, hba->outstanding_tasks); - return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; + /* + * ufshcd_link_recovery() may already have completed @scmd, e.g. via + * the existing MCQ force-completion path. + */ + if (!test_bit(SCMD_STATE_COMPLETE, &scmd->state)) { + if (!hba->mcq_enabled) { + unsigned long flags; + struct request *rq = scsi_cmd_to_rq(scmd); + + spin_lock_irqsave(&hba->outstanding_lock, flags); + __clear_bit(rq->tag, &hba->outstanding_reqs); + spin_unlock_irqrestore(&hba->outstanding_lock, flags); + } + + if (ufshcd_is_scsi_cmd(scmd)) { + set_host_byte(scmd, DID_REQUEUE); + ufshcd_release_scsi_cmd(hba, scmd); + } else { + set_host_byte(scmd, DID_TIME_OUT); + } + + scsi_done(scmd); + } + + return SCSI_EH_DONE; } static const struct attribute_group *ufshcd_driver_groups[] = { @@ -10518,7 +10540,6 @@ static int ufshcd_wl_suspend(struct device *dev) hba = shost_priv(sdev->host); down(&hba->host_sem); - hba->system_suspending = true; if (pm_runtime_suspended(dev)) goto out; @@ -10560,7 +10581,6 @@ static int ufshcd_wl_resume(struct device *dev) hba->curr_dev_pwr_mode, hba->uic_link_state); if (!ret) hba->is_sys_suspended = false; - hba->system_suspending = false; up(&hba->host_sem); return ret; } diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c index f3ae72feb5bfc..1e12fc19f8726 100644 --- a/drivers/usb/atm/ueagle-atm.c +++ b/drivers/usb/atm/ueagle-atm.c @@ -51,12 +51,6 @@ "[ueagle-atm vdbg] " format, ##args); \ } while (0) -#define uea_enters(usb_dev) \ - uea_vdbg(usb_dev, "entering %s\n" , __func__) - -#define uea_leaves(usb_dev) \ - uea_vdbg(usb_dev, "leaving %s\n" , __func__) - #define uea_err(usb_dev, format, args...) \ dev_err(&(usb_dev)->dev , "[UEAGLE-ATM] " format , ##args) @@ -600,13 +594,14 @@ static int uea_send_modem_cmd(struct usb_device *usb, static void uea_upload_pre_firmware(const struct firmware *fw_entry, void *context) { - struct usb_device *usb = context; + struct usb_interface *intf = context; + struct usb_device *usb = interface_to_usbdev(intf); + struct completion *fw_done = usb_get_intfdata(intf); const u8 *pfw; u8 value; u32 crc = 0; int ret, size; - uea_enters(usb); if (!fw_entry) { uea_err(usb, "firmware is not available\n"); goto err; @@ -670,18 +665,18 @@ static void uea_upload_pre_firmware(const struct firmware *fw_entry, uea_err(usb, "firmware is corrupted\n"); err: release_firmware(fw_entry); - uea_leaves(usb); + complete(fw_done); } /* * uea_load_firmware - Load usb firmware for pre-firmware devices. */ -static int uea_load_firmware(struct usb_device *usb, unsigned int ver) +static int uea_load_firmware(struct usb_interface *intf, unsigned int ver) { int ret; char *fw_name = EAGLE_FIRMWARE; + struct usb_device *usb = interface_to_usbdev(intf); - uea_enters(usb); uea_info(usb, "pre-firmware device, uploading firmware\n"); switch (ver) { @@ -703,14 +698,13 @@ static int uea_load_firmware(struct usb_device *usb, unsigned int ver) } ret = request_firmware_nowait(THIS_MODULE, 1, fw_name, &usb->dev, - GFP_KERNEL, usb, + GFP_KERNEL, intf, uea_upload_pre_firmware); if (ret) uea_err(usb, "firmware %s is not available\n", fw_name); else uea_info(usb, "loading firmware %s\n", fw_name); - uea_leaves(usb); return ret; } @@ -1137,7 +1131,6 @@ static int uea_cmv_e1(struct uea_softc *sc, struct cmv_e1 cmv; int ret; - uea_enters(INS_TO_USBDEV(sc)); uea_vdbg(INS_TO_USBDEV(sc), "Function : %d-%d, Address : %c%c%c%c, " "offset : 0x%04x, data : 0x%08x\n", E1_FUNCTION_TYPE(function), @@ -1164,9 +1157,8 @@ static int uea_cmv_e1(struct uea_softc *sc, sizeof(cmv), &cmv); if (ret < 0) return ret; - ret = wait_cmv_ack(sc); - uea_leaves(INS_TO_USBDEV(sc)); - return ret; + + return wait_cmv_ack(sc); } static int uea_cmv_e4(struct uea_softc *sc, @@ -1175,7 +1167,6 @@ static int uea_cmv_e4(struct uea_softc *sc, struct cmv_e4 cmv; int ret; - uea_enters(INS_TO_USBDEV(sc)); memset(&cmv, 0, sizeof(cmv)); uea_vdbg(INS_TO_USBDEV(sc), "Function : %d-%d, Group : 0x%04x, " @@ -1199,9 +1190,8 @@ static int uea_cmv_e4(struct uea_softc *sc, sizeof(cmv), &cmv); if (ret < 0) return ret; - ret = wait_cmv_ack(sc); - uea_leaves(INS_TO_USBDEV(sc)); - return ret; + + return wait_cmv_ack(sc); } static inline int uea_read_cmv_e1(struct uea_softc *sc, @@ -1295,7 +1285,6 @@ static int uea_stat_e1(struct uea_softc *sc) u32 data; int ret; - uea_enters(INS_TO_USBDEV(sc)); data = sc->stats.phy.state; ret = uea_read_cmv_e1(sc, E1_SA_STAT, 0, &sc->stats.phy.state); @@ -1438,7 +1427,6 @@ static int uea_stat_e4(struct uea_softc *sc) u32 tmp_arr[2]; int ret; - uea_enters(INS_TO_USBDEV(sc)); data = sc->stats.phy.state; /* XXX only need to be done before operationnal... */ @@ -1805,7 +1793,6 @@ static int uea_start_reset(struct uea_softc *sc) u16 zero = 0; /* ;-) */ int ret; - uea_enters(INS_TO_USBDEV(sc)); uea_info(INS_TO_USBDEV(sc), "(re)booting started\n"); /* mask interrupt */ @@ -1873,7 +1860,6 @@ static int uea_start_reset(struct uea_softc *sc) return ret; sc->reset = 0; - uea_leaves(INS_TO_USBDEV(sc)); return ret; } @@ -1889,7 +1875,6 @@ static int uea_kthread(void *data) int ret = -EAGAIN; set_freezable(); - uea_enters(INS_TO_USBDEV(sc)); while (!kthread_should_stop()) { if (ret < 0 || sc->reset) ret = uea_start_reset(sc); @@ -1898,7 +1883,7 @@ static int uea_kthread(void *data) if (ret != -EAGAIN) uea_wait(sc, 0, msecs_to_jiffies(1000)); } - uea_leaves(INS_TO_USBDEV(sc)); + return ret; } @@ -1911,8 +1896,6 @@ static int load_XILINX_firmware(struct uea_softc *sc) u8 value; char *fw_name = FPGA930_FIRMWARE; - uea_enters(INS_TO_USBDEV(sc)); - ret = request_firmware(&fw_entry, fw_name, &sc->usb_dev->dev); if (ret) { uea_err(INS_TO_USBDEV(sc), "firmware %s is not available\n", @@ -1956,7 +1939,6 @@ static int load_XILINX_firmware(struct uea_softc *sc) err1: release_firmware(fw_entry); err0: - uea_leaves(INS_TO_USBDEV(sc)); return ret; } @@ -1966,7 +1948,6 @@ static void uea_dispatch_cmv_e1(struct uea_softc *sc, struct intr_pkt *intr) struct cmv_dsc_e1 *dsc = &sc->cmv_dsc.e1; struct cmv_e1 *cmv = &intr->u.e1.s2.cmv; - uea_enters(INS_TO_USBDEV(sc)); if (le16_to_cpu(cmv->wPreamble) != E1_PREAMBLE) goto bad1; @@ -1990,7 +1971,6 @@ static void uea_dispatch_cmv_e1(struct uea_softc *sc, struct intr_pkt *intr) if (cmv->bFunction == E1_MAKEFUNCTION(E1_ADSLDIRECTIVE, E1_MODEMREADY)) { wake_up_cmv_ack(sc); - uea_leaves(INS_TO_USBDEV(sc)); return; } @@ -2004,7 +1984,6 @@ static void uea_dispatch_cmv_e1(struct uea_softc *sc, struct intr_pkt *intr) sc->data = sc->data << 16 | sc->data >> 16; wake_up_cmv_ack(sc); - uea_leaves(INS_TO_USBDEV(sc)); return; bad2: @@ -2012,14 +1991,12 @@ static void uea_dispatch_cmv_e1(struct uea_softc *sc, struct intr_pkt *intr) "Function : %d, Subfunction : %d\n", E1_FUNCTION_TYPE(cmv->bFunction), E1_FUNCTION_SUBTYPE(cmv->bFunction)); - uea_leaves(INS_TO_USBDEV(sc)); return; bad1: uea_err(INS_TO_USBDEV(sc), "invalid cmv received, " "wPreamble %d, bDirection %d\n", le16_to_cpu(cmv->wPreamble), cmv->bDirection); - uea_leaves(INS_TO_USBDEV(sc)); } /* The modem send us an ack. First with check if it right */ @@ -2028,7 +2005,6 @@ static void uea_dispatch_cmv_e4(struct uea_softc *sc, struct intr_pkt *intr) struct cmv_dsc_e4 *dsc = &sc->cmv_dsc.e4; struct cmv_e4 *cmv = &intr->u.e4.s2.cmv; - uea_enters(INS_TO_USBDEV(sc)); uea_dbg(INS_TO_USBDEV(sc), "cmv %x %x %x %x %x %x\n", be16_to_cpu(cmv->wGroup), be16_to_cpu(cmv->wFunction), be16_to_cpu(cmv->wOffset), be16_to_cpu(cmv->wAddress), @@ -2040,7 +2016,6 @@ static void uea_dispatch_cmv_e4(struct uea_softc *sc, struct intr_pkt *intr) if (be16_to_cpu(cmv->wFunction) == E4_MAKEFUNCTION(E4_ADSLDIRECTIVE, E4_MODEMREADY, 1)) { wake_up_cmv_ack(sc); - uea_leaves(INS_TO_USBDEV(sc)); return; } @@ -2053,7 +2028,6 @@ static void uea_dispatch_cmv_e4(struct uea_softc *sc, struct intr_pkt *intr) sc->data = be32_to_cpu(cmv->dwData[0]); sc->data1 = be32_to_cpu(cmv->dwData[1]); wake_up_cmv_ack(sc); - uea_leaves(INS_TO_USBDEV(sc)); return; bad2: @@ -2061,7 +2035,6 @@ static void uea_dispatch_cmv_e4(struct uea_softc *sc, struct intr_pkt *intr) "Function : %d, Subfunction : %d\n", E4_FUNCTION_TYPE(cmv->wFunction), E4_FUNCTION_SUBTYPE(cmv->wFunction)); - uea_leaves(INS_TO_USBDEV(sc)); return; } @@ -2089,8 +2062,6 @@ static void uea_intr(struct urb *urb) struct intr_pkt *intr = urb->transfer_buffer; int status = urb->status; - uea_enters(INS_TO_USBDEV(sc)); - if (unlikely(status < 0)) { uea_err(INS_TO_USBDEV(sc), "uea_intr() failed with %d\n", status); @@ -2130,8 +2101,6 @@ static int uea_boot(struct uea_softc *sc, struct usb_interface *intf) int ret = -ENOMEM; int size; - uea_enters(INS_TO_USBDEV(sc)); - if (UEA_CHIP_VERSION(sc) == EAGLE_IV) { size = E4_INTR_PKT_SIZE; sc->dispatch_cmv = uea_dispatch_cmv_e4; @@ -2188,7 +2157,6 @@ static int uea_boot(struct uea_softc *sc, struct usb_interface *intf) goto err2; } - uea_leaves(INS_TO_USBDEV(sc)); return 0; err2: @@ -2198,7 +2166,6 @@ static int uea_boot(struct uea_softc *sc, struct usb_interface *intf) sc->urb_int = NULL; kfree(intr); err0: - uea_leaves(INS_TO_USBDEV(sc)); return ret; } @@ -2208,7 +2175,7 @@ static int uea_boot(struct uea_softc *sc, struct usb_interface *intf) static void uea_stop(struct uea_softc *sc) { int ret; - uea_enters(INS_TO_USBDEV(sc)); + ret = kthread_stop(sc->kthread); uea_dbg(INS_TO_USBDEV(sc), "kthread finish with status %d\n", ret); @@ -2222,7 +2189,6 @@ static void uea_stop(struct uea_softc *sc) flush_work(&sc->task); release_firmware(sc->dsp_firm); - uea_leaves(INS_TO_USBDEV(sc)); } /* syfs interface */ @@ -2495,8 +2461,6 @@ static int uea_bind(struct usbatm_data *usbatm, struct usb_interface *intf, int ret, ifnum = intf->altsetting->desc.bInterfaceNumber; unsigned int alt; - uea_enters(usb); - /* interface 0 is for firmware/monitoring */ if (ifnum != UEA_INTR_IFACE_NO) return -ENODEV; @@ -2589,8 +2553,7 @@ static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id) struct usb_device *usb = interface_to_usbdev(intf); int ret; - uea_enters(usb); - uea_info(usb, "ADSL device founded vid (%#X) pid (%#X) Rev (%#X): %s\n", + uea_dbg(usb, "ADSL device found with vid (%#X) pid (%#X) Rev (%#X): %s\n", le16_to_cpu(usb->descriptor.idVendor), le16_to_cpu(usb->descriptor.idProduct), le16_to_cpu(usb->descriptor.bcdDevice), @@ -2598,8 +2561,23 @@ static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id) usb_reset_device(usb); - if (UEA_IS_PREFIRM(id)) - return uea_load_firmware(usb, UEA_CHIP_VERSION(id)); + if (UEA_IS_PREFIRM(id)) { + struct completion *fw_done; + + /* Wait for the firmware load to be done, in .disconnect() */ + fw_done = kzalloc_obj(*fw_done); + if (!fw_done) + return -ENOMEM; + + init_completion(fw_done); + usb_set_intfdata(intf, fw_done); + + ret = uea_load_firmware(intf, UEA_CHIP_VERSION(id)); + if (ret) + kfree(fw_done); + + return ret; + } ret = usbatm_usb_probe(intf, id, &uea_usbatm_driver); if (ret == 0) { @@ -2620,7 +2598,6 @@ static void uea_disconnect(struct usb_interface *intf) { struct usb_device *usb = interface_to_usbdev(intf); int ifnum = intf->altsetting->desc.bInterfaceNumber; - uea_enters(usb); /* ADI930 has 2 interfaces and eagle 3 interfaces. * Pre-firmware device has one interface @@ -2630,9 +2607,14 @@ static void uea_disconnect(struct usb_interface *intf) usbatm_usb_disconnect(intf); mutex_unlock(&uea_mutex); uea_info(usb, "ADSL device removed\n"); - } + } else if (usb->config->desc.bNumInterfaces == 1) { + struct completion *fw_done = usb_get_intfdata(intf); - uea_leaves(usb); + uea_dbg(usb, "pre-firmware device, waiting firmware upload\n"); + wait_for_completion(fw_done); + uea_dbg(usb, "pre-firmware device, finished waiting\n"); + kfree(fw_done); + } } /* diff --git a/drivers/usb/cdns3/cdnsp-mem.c b/drivers/usb/cdns3/cdnsp-mem.c index a2a1b21f2ef87..880097f1007d7 100644 --- a/drivers/usb/cdns3/cdnsp-mem.c +++ b/drivers/usb/cdns3/cdnsp-mem.c @@ -631,6 +631,8 @@ int cdnsp_alloc_stream_info(struct cdnsp_device *pdev, } } + cdnsp_free_stream_ctx(pdev, pep); + cleanup_stream_rings: kfree(pep->stream_info.stream_rings); diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index ddf0b59638595..3cb372721282a 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -1810,6 +1810,9 @@ static const struct usb_device_id acm_ids[] = { { USB_DEVICE(0x1901, 0x0006), /* GE Healthcare Patient Monitor UI Controller */ .driver_info = DISABLE_ECHO, /* DISABLE ECHO in termios flag */ }, + { USB_DEVICE(0x1965, 0x0017), /* Uniden BC125AT */ + .driver_info = NO_UNION_NORMAL, /* has no union descriptor */ + }, { USB_DEVICE(0x1965, 0x0018), /* Uniden UBC125XLT */ .driver_info = NO_UNION_NORMAL, /* has no union descriptor */ }, diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c index 9b69148128e5b..7e43429e996e7 100644 --- a/drivers/usb/common/ulpi.c +++ b/drivers/usb/common/ulpi.c @@ -281,28 +281,24 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi) ulpi->dev.parent = dev; /* needed early for ops */ ulpi->dev.bus = &ulpi_bus; ulpi->dev.type = &ulpi_dev_type; + + device_initialize(&ulpi->dev); + dev_set_name(&ulpi->dev, "%s.ulpi", dev_name(dev)); ACPI_COMPANION_SET(&ulpi->dev, ACPI_COMPANION(dev)); ret = ulpi_of_register(ulpi); - if (ret) { - kfree(ulpi); + if (ret) return ret; - } ret = ulpi_read_id(ulpi); - if (ret) { - of_node_put(ulpi->dev.of_node); - kfree(ulpi); + if (ret) return ret; - } - ret = device_register(&ulpi->dev); - if (ret) { - put_device(&ulpi->dev); + ret = device_add(&ulpi->dev); + if (ret) return ret; - } root = debugfs_create_dir(dev_name(&ulpi->dev), ulpi_root); debugfs_create_file("regs", 0444, root, ulpi, &ulpi_regs_fops); @@ -334,9 +330,10 @@ struct ulpi *ulpi_register_interface(struct device *dev, ulpi->ops = ops; ret = ulpi_register(dev, ulpi); - if (ret) + if (ret) { + put_device(&ulpi->dev); return ERR_PTR(ret); - + } return ulpi; } diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index 87810eff974ef..87ee2d938bc03 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -296,6 +296,9 @@ static const struct usb_device_id usb_quirk_list[] = { /* CarrolTouch 4500U */ { USB_DEVICE(0x04e7, 0x0030), .driver_info = USB_QUIRK_RESET_RESUME }, + /* Samsung T5 EVO Portable SSD */ + { USB_DEVICE(0x04e8, 0x6200), .driver_info = USB_QUIRK_NO_LPM }, + /* Samsung Android phone modem - ID conflict with SPH-I500 */ { USB_DEVICE(0x04e8, 0x6601), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS }, @@ -576,6 +579,9 @@ static const struct usb_device_id usb_quirk_list[] = { /* VLI disk */ { USB_DEVICE(0x2109, 0x0711), .driver_info = USB_QUIRK_NO_LPM }, + /* VIA Labs, Inc. USB2.0 Hub */ + { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, + /* Raydium Touchscreen */ { USB_DEVICE(0x2386, 0x3114), .driver_info = USB_QUIRK_NO_LPM }, diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 65213896de998..6f3c00b089371 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -793,9 +793,9 @@ static void dwc3_ulpi_setup(struct dwc3 *dwc) if (dwc->enable_usb2_transceiver_delay) { for (index = 0; index < dwc->num_usb2_ports; index++) { - reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(index)); + reg = dwc3_readl(dwc, DWC3_GUSB2PHYCFG(index)); reg |= DWC3_GUSB2PHYCFG_XCVRDLY; - dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(index), reg); + dwc3_writel(dwc, DWC3_GUSB2PHYCFG(index), reg); } } } diff --git a/drivers/usb/dwc3/dwc3-meson-g12a.c b/drivers/usb/dwc3/dwc3-meson-g12a.c index 55e144ba8cfc6..4d611c08e8a4c 100644 --- a/drivers/usb/dwc3/dwc3-meson-g12a.c +++ b/drivers/usb/dwc3/dwc3-meson-g12a.c @@ -907,35 +907,39 @@ static int __maybe_unused dwc3_meson_g12a_resume(struct device *dev) ret = priv->drvdata->usb_init(priv); if (ret) - return ret; + goto err_rearm; /* Init PHYs */ for (i = 0 ; i < PHY_COUNT ; ++i) { ret = phy_init(priv->phys[i]); if (ret) - return ret; + goto err_rearm; } /* Set PHY Power */ for (i = 0 ; i < PHY_COUNT ; ++i) { ret = phy_power_on(priv->phys[i]); if (ret) - return ret; + goto err_rearm; } if (priv->vbus && priv->otg_phy_mode == PHY_MODE_USB_HOST) { ret = regulator_enable(priv->vbus); if (ret) - return ret; + goto err_rearm; } if (priv->drvdata->usb_post_init) { ret = priv->drvdata->usb_post_init(priv); if (ret) - return ret; + goto err_rearm; } return 0; + +err_rearm: + reset_control_rearm(priv->reset); + return ret; } static const struct dev_pm_ops dwc3_meson_g12a_dev_pm_ops = { diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 3d4ca68e584c7..1082e9c9afaa8 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -3934,15 +3934,48 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc, } } +static bool dwc3_prepare_disconnect_gadget(struct dwc3 *dwc, + struct usb_gadget_driver **driver, + struct usb_gadget **gadget) +{ + if (!dwc->async_callbacks || !dwc->gadget_driver || + !dwc->gadget_driver->disconnect) + return false; + + *driver = dwc->gadget_driver; + *gadget = dwc->gadget; + + return true; +} + static void dwc3_disconnect_gadget(struct dwc3 *dwc) { - if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { + struct usb_gadget_driver *driver; + struct usb_gadget *gadget; + + if (dwc3_prepare_disconnect_gadget(dwc, &driver, &gadget)) { spin_unlock(&dwc->lock); - dwc->gadget_driver->disconnect(dwc->gadget); + driver->disconnect(gadget); spin_lock(&dwc->lock); } } +static void dwc3_disconnect_gadget_sleepable(struct dwc3 *dwc) +{ + struct usb_gadget_driver *driver; + struct usb_gadget *gadget; + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + if (!dwc3_prepare_disconnect_gadget(dwc, &driver, &gadget)) { + spin_unlock_irqrestore(&dwc->lock, flags); + return; + } + + spin_unlock_irqrestore(&dwc->lock, flags); + driver->disconnect(gadget); +} + static void dwc3_suspend_gadget(struct dwc3 *dwc) { if (dwc->async_callbacks && dwc->gadget_driver->suspend) { @@ -4838,7 +4871,6 @@ EXPORT_SYMBOL_GPL(dwc3_gadget_exit); int dwc3_gadget_suspend(struct dwc3 *dwc) { - unsigned long flags; int ret; ret = dwc3_gadget_soft_disconnect(dwc); @@ -4852,10 +4884,7 @@ int dwc3_gadget_suspend(struct dwc3 *dwc) return -EAGAIN; } - spin_lock_irqsave(&dwc->lock, flags); - if (dwc->gadget_driver) - dwc3_disconnect_gadget(dwc); - spin_unlock_irqrestore(&dwc->lock, flags); + dwc3_disconnect_gadget_sleepable(dwc); return 0; } diff --git a/drivers/usb/fotg210/fotg210-hcd.c b/drivers/usb/fotg210/fotg210-hcd.c index 1a48329a4e089..956be5b565102 100644 --- a/drivers/usb/fotg210/fotg210-hcd.c +++ b/drivers/usb/fotg210/fotg210-hcd.c @@ -4267,8 +4267,6 @@ static int iso_stream_schedule(struct fotg210_hcd *fotg210, struct urb *urb, return 0; fail: - iso_sched_free(stream, sched); - urb->hcpriv = NULL; return status; } @@ -4562,6 +4560,10 @@ static int itd_submit(struct fotg210_hcd *fotg210, struct urb *urb, else usb_hcd_unlink_urb_from_ep(fotg210_to_hcd(fotg210), urb); done_not_linked: + if (status < 0) { + iso_sched_free(stream, urb->hcpriv); + urb->hcpriv = NULL; + } spin_unlock_irqrestore(&fotg210->lock, flags); done: return status; diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index dc36643745963..df39e3487c1f3 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -1863,9 +1863,10 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl) if (cdev->config) config = cdev->config; else - config = list_first_entry( + config = list_first_entry_or_null( &cdev->configs, - struct usb_configuration, list); + struct usb_configuration, + list); if (!config) goto done; diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 75912ce6ab556..44218be1e6768 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -288,6 +288,7 @@ static int ffs_acquire_dev(const char *dev_name, struct ffs_data *ffs_data); static void ffs_release_dev(struct ffs_dev *ffs_dev); static int ffs_ready(struct ffs_data *ffs); static void ffs_closed(struct ffs_data *ffs); +static void ffs_reset_work(struct work_struct *work); /* Misc helper functions ****************************************************/ @@ -1374,7 +1375,6 @@ ffs_epfile_release(struct inode *inode, struct file *file) mutex_unlock(&epfile->dmabufs_mutex); - __ffs_epfile_read_buffer_free(epfile); ffs_data_closed(epfile->ffs); return 0; @@ -1704,6 +1704,7 @@ static int ffs_dmabuf_transfer(struct file *file, resv_dir = epfile->in ? DMA_RESV_USAGE_READ : DMA_RESV_USAGE_WRITE; dma_resv_add_fence(dmabuf->resv, &fence->base, resv_dir); + dma_fence_put(&fence->base); dma_resv_unlock(dmabuf->resv); /* Now that the dma_fence is in place, queue the transfer. */ @@ -2221,6 +2222,7 @@ static struct ffs_data *ffs_data_new(const char *dev_name) init_waitqueue_head(&ffs->ev.waitq); init_waitqueue_head(&ffs->wait); init_completion(&ffs->ep0req_completion); + INIT_WORK(&ffs->reset_work, ffs_reset_work); /* XXX REVISIT need to update it in some places, or do we? */ ffs->ev.can_stall = 1; @@ -2364,6 +2366,7 @@ static int ffs_epfiles_create(struct ffs_data *ffs) sprintf(epfile->name, "ep%02x", ffs->eps_addrmap[i]); else sprintf(epfile->name, "ep%u", i); + epfile->in = (ffs->eps_addrmap[i] & USB_ENDPOINT_DIR_MASK) ? 1 : 0; err = ffs_sb_create_file(ffs->sb, epfile->name, epfile, &ffs_epfile_operations); if (err) { @@ -2389,6 +2392,7 @@ static void ffs_epfiles_destroy(struct super_block *sb, for (; count; --count, ++epfile) { BUG_ON(mutex_is_locked(&epfile->mutex)); + __ffs_epfile_read_buffer_free(epfile); simple_remove_by_name(root, epfile->name, clear_one); } @@ -2453,7 +2457,6 @@ static int ffs_func_eps_enable(struct ffs_function *func) ret = usb_ep_enable(ep->ep); if (!ret) { epfile->ep = ep; - epfile->in = usb_endpoint_dir_in(ep->ep->desc); epfile->isoc = usb_endpoint_xfer_isoc(ep->ep->desc); } else { break; @@ -3775,7 +3778,6 @@ static int ffs_func_set_alt(struct usb_function *f, if (ffs->state == FFS_DEACTIVATED) { ffs->state = FFS_CLOSING; spin_unlock_irqrestore(&ffs->eps_lock, flags); - INIT_WORK(&ffs->reset_work, ffs_reset_work); schedule_work(&ffs->reset_work); return -ENODEV; } @@ -3806,7 +3808,6 @@ static void ffs_func_disable(struct usb_function *f) if (ffs->state == FFS_DEACTIVATED) { ffs->state = FFS_CLOSING; spin_unlock_irqrestore(&ffs->eps_lock, flags); - INIT_WORK(&ffs->reset_work, ffs_reset_work); schedule_work(&ffs->reset_work); return; } diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c index e4f7828ae75df..837f753d0cae5 100644 --- a/drivers/usb/gadget/function/f_printer.c +++ b/drivers/usb/gadget/function/f_printer.c @@ -363,12 +363,11 @@ printer_open(struct inode *inode, struct file *fd) ret = 0; /* Change the printer status to show that it's on-line. */ dev->printer_status |= PRINTER_SELECTED; + kref_get(&dev->kref); } spin_unlock_irqrestore(&dev->lock, flags); - kref_get(&dev->kref); - return ret; } diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c index 3da54a7d7aba8..a2fd239b7ad33 100644 --- a/drivers/usb/gadget/function/rndis.c +++ b/drivers/usb/gadget/function/rndis.c @@ -591,6 +591,7 @@ static int rndis_init_response(struct rndis_params *params, static int rndis_query_response(struct rndis_params *params, rndis_query_msg_type *buf) { + u32 BufLength, BufOffset; rndis_query_cmplt_type *resp; rndis_resp_t *r; @@ -598,6 +599,13 @@ static int rndis_query_response(struct rndis_params *params, if (!params->dev) return -ENOTSUPP; + BufLength = le32_to_cpu(buf->InformationBufferLength); + BufOffset = le32_to_cpu(buf->InformationBufferOffset); + if ((BufLength > RNDIS_MAX_TOTAL_SIZE) || + (BufOffset > RNDIS_MAX_TOTAL_SIZE) || + (BufOffset + 8 >= RNDIS_MAX_TOTAL_SIZE)) + return -EINVAL; + /* * we need more memory: * gen_ndis_query_resp expects enough space for @@ -614,10 +622,8 @@ static int rndis_query_response(struct rndis_params *params, resp->RequestID = buf->RequestID; /* Still LE in msg buffer */ if (gen_ndis_query_resp(params, le32_to_cpu(buf->OID), - le32_to_cpu(buf->InformationBufferOffset) - + 8 + (u8 *)buf, - le32_to_cpu(buf->InformationBufferLength), - r)) { + BufOffset + 8 + (u8 *)buf, + BufLength, r)) { /* OID not supported */ resp->Status = cpu_to_le32(RNDIS_STATUS_NOT_SUPPORTED); resp->MessageLength = cpu_to_le32(sizeof *resp); @@ -1074,6 +1080,12 @@ int rndis_rm_hdr(struct gether *port, /* tmp points to a struct rndis_packet_msg_type */ __le32 *tmp = (void *)skb->data; + /* Need at least MessageType, MessageLength, DataOffset, DataLength */ + if (skb->len < 16) { + dev_kfree_skb_any(skb); + return -EINVAL; + } + /* MessageType, MessageLength */ if (cpu_to_le32(RNDIS_MSG_PACKET) != get_unaligned(tmp++)) { diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index e8861eaad9077..068423fc581ae 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -31,8 +31,9 @@ static const struct bus_type gadget_bus_type; /** * struct usb_udc - describes one usb device controller * @driver: the gadget driver pointer. For use by the class code - * @dev: the child device to the actual controller * @gadget: the gadget. For use by the class code + * @gadget_release: the gadget's release routine + * @dev: the child device to the actual controller * @list: for use by the udc class driver * @vbus: for udcs who care about vbus status, this value is real vbus status; * for udcs who do not care about vbus status, this value is always true @@ -53,6 +54,7 @@ static const struct bus_type gadget_bus_type; struct usb_udc { struct usb_gadget_driver *driver; struct usb_gadget *gadget; + void (*gadget_release)(struct device *dev); struct device dev; struct list_head list; bool vbus; @@ -1357,6 +1359,17 @@ static void usb_udc_nop_release(struct device *dev) dev_vdbg(dev, "%s\n", __func__); } +static void usb_gadget_release(struct device *dev) +{ + struct usb_gadget *gadget = dev_to_usb_gadget(dev); + struct usb_udc *udc = gadget->udc; + /* Cache the gadget's release routine to prevent UAF */ + void (*release)(struct device *dev) = udc->gadget_release; + + put_device(&udc->dev); + release(dev); +} + /** * usb_initialize_gadget - initialize a gadget and its embedded struct device * @parent: the parent device to this udc. Usually the controller driver's @@ -1413,6 +1426,14 @@ int usb_add_gadget(struct usb_gadget *gadget) mutex_init(&udc->connect_lock); udc->started = false; + /* + * Align decoupled lifecycles: take a UDC reference to ensure it + * remains allocated until the gadget is released, requiring an + * override of the gadget's release routine to drop it. + */ + udc->gadget_release = gadget->dev.release; + gadget->dev.release = usb_gadget_release; + get_device(&udc->dev); mutex_lock(&udc_lock); list_add_tail(&udc->list, &udc_list); @@ -1457,6 +1478,12 @@ int usb_add_gadget(struct usb_gadget *gadget) mutex_lock(&udc_lock); list_del(&udc->list); mutex_unlock(&udc_lock); + /* + * Revert the override and drop the UDC reference to prevent + * leaking the UDC if the gadget was statically allocated. + */ + gadget->dev.release = udc->gadget_release; + put_device(&udc->dev); err_put_udc: put_device(&udc->dev); diff --git a/drivers/usb/host/ehci-sched.c b/drivers/usb/host/ehci-sched.c index a241337c9af86..57d07d1c2dfa6 100644 --- a/drivers/usb/host/ehci-sched.c +++ b/drivers/usb/host/ehci-sched.c @@ -1623,6 +1623,7 @@ iso_stream_schedule( status = 1; /* and give it back immediately */ iso_sched_free(stream, sched); sched = NULL; + urb->hcpriv = NULL; } } urb->error_count = skip / period; @@ -1653,8 +1654,6 @@ iso_stream_schedule( return status; fail: - iso_sched_free(stream, sched); - urb->hcpriv = NULL; return status; } @@ -1966,6 +1965,10 @@ static int itd_submit(struct ehci_hcd *ehci, struct urb *urb, usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); } done_not_linked: + if (status < 0) { + iso_sched_free(stream, urb->hcpriv); + urb->hcpriv = NULL; + } spin_unlock_irqrestore(&ehci->lock, flags); done: return status; @@ -2343,6 +2346,10 @@ static int sitd_submit(struct ehci_hcd *ehci, struct urb *urb, usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); } done_not_linked: + if (status < 0) { + iso_sched_free(stream, urb->hcpriv); + urb->hcpriv = NULL; + } spin_unlock_irqrestore(&ehci->lock, flags); done: return status; diff --git a/drivers/usb/host/max3421-hcd.c b/drivers/usb/host/max3421-hcd.c index 0e17c988d36a2..73e76d0e69738 100644 --- a/drivers/usb/host/max3421-hcd.c +++ b/drivers/usb/host/max3421-hcd.c @@ -1685,6 +1685,8 @@ max3421_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, u16 index, case ClearHubFeature: break; case ClearPortFeature: + if (index != 1) + goto error; switch (value) { case USB_PORT_FEAT_SUSPEND: break; @@ -1694,6 +1696,8 @@ max3421_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, u16 index, !pdata->vbus_active_level); fallthrough; default: + if (value >= 32) + goto error; max3421_hcd->port_status &= ~(1 << value); } break; @@ -1726,6 +1730,8 @@ max3421_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, u16 index, break; case SetPortFeature: + if (index != 1) + goto error; switch (value) { case USB_PORT_FEAT_LINK_STATE: case USB_PORT_FEAT_U1_TIMEOUT: @@ -1747,6 +1753,8 @@ max3421_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, u16 index, max3421_reset_port(hcd); fallthrough; default: + if (value >= 32) + goto error; if ((max3421_hcd->port_status & USB_PORT_STAT_POWER) != 0) max3421_hcd->port_status |= (1 << value); diff --git a/drivers/usb/host/sl811-hcd.c b/drivers/usb/host/sl811-hcd.c index 4ae47edd4b8b4..b044977f6f567 100644 --- a/drivers/usb/host/sl811-hcd.c +++ b/drivers/usb/host/sl811-hcd.c @@ -1591,6 +1591,7 @@ sl811h_remove(struct platform_device *dev) remove_debug_file(sl811); usb_remove_hcd(hcd); + device_wakeup_disable(hcd->self.controller); /* some platforms may use IORESOURCE_IO */ res = platform_get_resource(dev, IORESOURCE_MEM, 1); diff --git a/drivers/usb/host/xhci-sideband.c b/drivers/usb/host/xhci-sideband.c index 23153e136d4b8..a5deeee4d5dce 100644 --- a/drivers/usb/host/xhci-sideband.c +++ b/drivers/usb/host/xhci-sideband.c @@ -58,6 +58,8 @@ xhci_ring_to_sgtable(struct xhci_sideband *sb, struct xhci_ring *ring) if (sg_alloc_table_from_pages(sgt, pages, n_pages, 0, sz, GFP_KERNEL)) goto err; + kvfree(pages); + /* * Save first segment dma address to sg dma_address field for the sideband * client to have access to the IOVA of the ring. diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index a54f5b57f2055..15346fdd66bc4 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -3788,6 +3788,7 @@ static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev, struct xhci_virt_device *vdev; struct xhci_command *command; struct xhci_input_control_ctx *ctrl_ctx; + struct xhci_stream_info *stream_info[EP_CTX_PER_DEV]; unsigned int ep_index; unsigned long flags; u32 changed_ep_bitmask; @@ -3848,10 +3849,15 @@ static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev, if (ret < 0) return ret; + /* + * dma_free_coherent() called by xhci_free_stream_info() may sleep, + * so save stream_info pointers and clear references under lock, + * then free the memory outside lock. + */ spin_lock_irqsave(&xhci->lock, flags); for (i = 0; i < num_eps; i++) { ep_index = xhci_get_endpoint_index(&eps[i]->desc); - xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info); + stream_info[i] = vdev->eps[ep_index].stream_info; vdev->eps[ep_index].stream_info = NULL; /* FIXME Unset maxPstreams in endpoint context and * update deq ptr to point to normal string ring. @@ -3861,6 +3867,9 @@ static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev, } spin_unlock_irqrestore(&xhci->lock, flags); + for (i = 0; i < num_eps; i++) + xhci_free_stream_info(xhci, stream_info[i]); + return 0; } diff --git a/drivers/usb/misc/chaoskey.c b/drivers/usb/misc/chaoskey.c index d8016540953fc..9c06f7775301f 100644 --- a/drivers/usb/misc/chaoskey.c +++ b/drivers/usb/misc/chaoskey.c @@ -320,7 +320,6 @@ static int chaoskey_release(struct inode *inode, struct file *file) mutex_unlock(&dev->lock); destruction: mutex_unlock(&chaoskey_list_lock); - usb_dbg(interface, "release success"); return rv; } diff --git a/drivers/usb/misc/idmouse.c b/drivers/usb/misc/idmouse.c index 0f6b3464c2d6a..3e37adf2bb573 100644 --- a/drivers/usb/misc/idmouse.c +++ b/drivers/usb/misc/idmouse.c @@ -63,6 +63,7 @@ MODULE_DEVICE_TABLE(usb, idmouse_table); /* structure to hold all of our device specific stuff */ struct usb_idmouse { + struct kref kref; struct usb_device *udev; /* save off the usb device pointer */ struct usb_interface *interface; /* the interface for this device */ @@ -209,8 +210,10 @@ static int idmouse_resume(struct usb_interface *intf) return 0; } -static inline void idmouse_delete(struct usb_idmouse *dev) +static inline void idmouse_delete(struct kref *kref) { + struct usb_idmouse *dev = container_of(kref, struct usb_idmouse, kref); + kfree(dev->bulk_in_buffer); kfree(dev); } @@ -254,6 +257,8 @@ static int idmouse_open(struct inode *inode, struct file *file) /* increment our usage count for the driver */ ++dev->open; + kref_get(&dev->kref); + /* save our object in the file's private structure */ file->private_data = dev; @@ -277,16 +282,11 @@ static int idmouse_release(struct inode *inode, struct file *file) /* lock our device */ mutex_lock(&dev->lock); - --dev->open; + mutex_unlock(&dev->lock); + + kref_put(&dev->kref, idmouse_delete); - if (!dev->present) { - /* the device was unplugged before the file was released */ - mutex_unlock(&dev->lock); - idmouse_delete(dev); - } else { - mutex_unlock(&dev->lock); - } return 0; } @@ -334,6 +334,7 @@ static int idmouse_probe(struct usb_interface *interface, if (dev == NULL) return -ENOMEM; + kref_init(&dev->kref); mutex_init(&dev->lock); dev->udev = udev; dev->interface = interface; @@ -342,8 +343,7 @@ static int idmouse_probe(struct usb_interface *interface, result = usb_find_bulk_in_endpoint(iface_desc, &endpoint); if (result) { dev_err(&interface->dev, "Unable to find bulk-in endpoint.\n"); - idmouse_delete(dev); - return result; + goto err_put_kref; } dev->orig_bi_size = usb_endpoint_maxp(endpoint); @@ -351,8 +351,8 @@ static int idmouse_probe(struct usb_interface *interface, dev->bulk_in_endpointAddr = endpoint->bEndpointAddress; dev->bulk_in_buffer = kmalloc(IMGSIZE + dev->bulk_in_size, GFP_KERNEL); if (!dev->bulk_in_buffer) { - idmouse_delete(dev); - return -ENOMEM; + result = -ENOMEM; + goto err_put_kref; } /* allow device read, write and ioctl */ @@ -364,14 +364,18 @@ static int idmouse_probe(struct usb_interface *interface, if (result) { /* something prevented us from registering this device */ dev_err(&interface->dev, "Unable to allocate minor number.\n"); - idmouse_delete(dev); - return result; + goto err_put_kref; } /* be noisy */ dev_info(&interface->dev,"%s now attached\n",DRIVER_DESC); return 0; + +err_put_kref: + kref_put(&dev->kref, idmouse_delete); + + return result; } static void idmouse_disconnect(struct usb_interface *interface) @@ -387,14 +391,9 @@ static void idmouse_disconnect(struct usb_interface *interface) /* prevent device read, write and ioctl */ dev->present = 0; - /* if the device is opened, idmouse_release will clean this up */ - if (!dev->open) { - mutex_unlock(&dev->lock); - idmouse_delete(dev); - } else { - /* unlock */ - mutex_unlock(&dev->lock); - } + mutex_unlock(&dev->lock); + + kref_put(&dev->kref, idmouse_delete); dev_info(&interface->dev, "disconnected\n"); } diff --git a/drivers/usb/misc/iowarrior.c b/drivers/usb/misc/iowarrior.c index 22504c0a28416..de2b236ef9030 100644 --- a/drivers/usb/misc/iowarrior.c +++ b/drivers/usb/misc/iowarrior.c @@ -72,6 +72,7 @@ static struct usb_driver iowarrior_driver; /* Structure to hold all of our device specific stuff */ struct iowarrior { + struct kref kref; struct mutex mutex; /* locks this structure */ struct usb_device *udev; /* save off the usb device pointer */ struct usb_interface *interface; /* the interface for this device */ @@ -240,8 +241,10 @@ static void iowarrior_write_callback(struct urb *urb) /* * iowarrior_delete */ -static inline void iowarrior_delete(struct iowarrior *dev) +static inline void iowarrior_delete(struct kref *kref) { + struct iowarrior *dev = container_of(kref, struct iowarrior, kref); + kfree(dev->int_in_buffer); usb_free_urb(dev->int_in_urb); kfree(dev->read_queue); @@ -637,6 +640,9 @@ static int iowarrior_open(struct inode *inode, struct file *file) } /* increment our usage count for the driver */ ++dev->opened; + + kref_get(&dev->kref); + /* save our object in the file's private structure */ file->private_data = dev; retval = 0; @@ -652,7 +658,6 @@ static int iowarrior_open(struct inode *inode, struct file *file) static int iowarrior_release(struct inode *inode, struct file *file) { struct iowarrior *dev; - int retval = 0; dev = file->private_data; if (!dev) @@ -660,29 +665,18 @@ static int iowarrior_release(struct inode *inode, struct file *file) /* lock our device */ mutex_lock(&dev->mutex); + dev->opened = 0; /* we're closing now */ - if (dev->opened <= 0) { - retval = -ENODEV; /* close called more than once */ - mutex_unlock(&dev->mutex); - } else { - dev->opened = 0; /* we're closing now */ - retval = 0; - if (dev->present) { - /* - The device is still connected so we only shutdown - pending read-/write-ops. - */ - usb_kill_urb(dev->int_in_urb); - wake_up_interruptible(&dev->read_wait); - wake_up_interruptible(&dev->write_wait); - mutex_unlock(&dev->mutex); - } else { - /* The device was unplugged, cleanup resources */ - mutex_unlock(&dev->mutex); - iowarrior_delete(dev); - } + if (dev->present) { + usb_kill_urb(dev->int_in_urb); + wake_up_interruptible(&dev->read_wait); + wake_up_interruptible(&dev->write_wait); } - return retval; + mutex_unlock(&dev->mutex); + + kref_put(&dev->kref, iowarrior_delete); + + return 0; } static __poll_t iowarrior_poll(struct file *file, poll_table * wait) @@ -767,6 +761,7 @@ static int iowarrior_probe(struct usb_interface *interface, if (!dev) return retval; + kref_init(&dev->kref); mutex_init(&dev->mutex); atomic_set(&dev->intr_idx, 0); @@ -885,7 +880,8 @@ static int iowarrior_probe(struct usb_interface *interface, return retval; error: - iowarrior_delete(dev); + kref_put(&dev->kref, iowarrior_delete); + return retval; } @@ -905,21 +901,18 @@ static void iowarrior_disconnect(struct usb_interface *interface) /* prevent device read, write and ioctl */ dev->present = 0; + /* write urbs are not stopped on close() so kill unconditionally */ + usb_kill_anchored_urbs(&dev->submitted); + if (dev->opened) { - /* There is a process that holds a filedescriptor to the device , - so we only shutdown read-/write-ops going on. - Deleting the device is postponed until close() was called. - */ usb_kill_urb(dev->int_in_urb); - usb_kill_anchored_urbs(&dev->submitted); wake_up_interruptible(&dev->read_wait); wake_up_interruptible(&dev->write_wait); - mutex_unlock(&dev->mutex); - } else { - /* no process is using the device, cleanup now */ - mutex_unlock(&dev->mutex); - iowarrior_delete(dev); } + + mutex_unlock(&dev->mutex); + + kref_put(&dev->kref, iowarrior_delete); } /* usb specific object needed to register this driver with the usb subsystem */ diff --git a/drivers/usb/misc/ldusb.c b/drivers/usb/misc/ldusb.c index c74f142f6637a..71132a15e7711 100644 --- a/drivers/usb/misc/ldusb.c +++ b/drivers/usb/misc/ldusb.c @@ -150,6 +150,7 @@ MODULE_PARM_DESC(min_interrupt_out_interval, "Minimum interrupt out interval in /* Structure to hold all of our device specific stuff */ struct ld_usb { + struct kref kref; struct mutex mutex; /* locks this structure */ struct usb_interface *intf; /* save off the usb interface pointer */ unsigned long disconnected:1; @@ -201,8 +202,10 @@ static void ld_usb_abort_transfers(struct ld_usb *dev) /* * ld_usb_delete */ -static void ld_usb_delete(struct ld_usb *dev) +static void ld_usb_delete(struct kref *kref) { + struct ld_usb *dev = container_of(kref, struct ld_usb, kref); + /* free data structures */ usb_free_urb(dev->interrupt_in_urb); usb_free_urb(dev->interrupt_out_urb); @@ -355,6 +358,8 @@ static int ld_usb_open(struct inode *inode, struct file *file) goto unlock_exit; } + kref_get(&dev->kref); + /* save device in the file's private structure */ file->private_data = dev; @@ -381,17 +386,8 @@ static int ld_usb_release(struct inode *inode, struct file *file) mutex_lock(&dev->mutex); - if (dev->open_count != 1) { - retval = -ENODEV; + if (dev->disconnected) goto unlock_exit; - } - if (dev->disconnected) { - /* the device was unplugged before the file was released */ - mutex_unlock(&dev->mutex); - /* unlock here as ld_usb_delete frees dev */ - ld_usb_delete(dev); - goto exit; - } /* wait until write transfer is finished */ if (dev->interrupt_out_busy) @@ -401,7 +397,7 @@ static int ld_usb_release(struct inode *inode, struct file *file) unlock_exit: mutex_unlock(&dev->mutex); - + kref_put(&dev->kref, ld_usb_delete); exit: return retval; } @@ -659,6 +655,8 @@ static int ld_usb_probe(struct usb_interface *intf, const struct usb_device_id * dev = kzalloc_obj(*dev); if (!dev) goto exit; + + kref_init(&dev->kref); mutex_init(&dev->mutex); spin_lock_init(&dev->rbsl); dev->intf = intf; @@ -740,7 +738,7 @@ static int ld_usb_probe(struct usb_interface *intf, const struct usb_device_id * return retval; error: - ld_usb_delete(dev); + kref_put(&dev->kref, ld_usb_delete); return retval; } @@ -768,18 +766,18 @@ static void ld_usb_disconnect(struct usb_interface *intf) mutex_lock(&dev->mutex); - /* if the device is not opened, then we clean up right now */ - if (!dev->open_count) { - mutex_unlock(&dev->mutex); - ld_usb_delete(dev); - } else { - dev->disconnected = 1; + dev->disconnected = 1; + + if (dev->open_count) { /* wake up pollers */ wake_up_interruptible_all(&dev->read_wait); wake_up_interruptible_all(&dev->write_wait); - mutex_unlock(&dev->mutex); } + mutex_unlock(&dev->mutex); + + kref_put(&dev->kref, ld_usb_delete); + dev_info(&intf->dev, "LD USB Device #%d now disconnected\n", (minor - USB_LD_MINOR_BASE)); } diff --git a/drivers/usb/misc/legousbtower.c b/drivers/usb/misc/legousbtower.c index 052ffc2e71eea..18dd4115befb2 100644 --- a/drivers/usb/misc/legousbtower.c +++ b/drivers/usb/misc/legousbtower.c @@ -185,6 +185,7 @@ MODULE_DEVICE_TABLE(usb, tower_table); /* Structure to hold all of our device specific stuff */ struct lego_usb_tower { + struct kref kref; struct mutex lock; /* locks this structure */ struct usb_device *udev; /* save off the usb device pointer */ unsigned char minor; /* the starting minor number for this device */ @@ -220,7 +221,6 @@ struct lego_usb_tower { /* local function prototypes */ static ssize_t tower_read(struct file *file, char __user *buffer, size_t count, loff_t *ppos); static ssize_t tower_write(struct file *file, const char __user *buffer, size_t count, loff_t *ppos); -static inline void tower_delete(struct lego_usb_tower *dev); static int tower_open(struct inode *inode, struct file *file); static int tower_release(struct inode *inode, struct file *file); static __poll_t tower_poll(struct file *file, poll_table *wait); @@ -286,8 +286,10 @@ static inline void lego_usb_tower_debug_data(struct device *dev, /* * tower_delete */ -static inline void tower_delete(struct lego_usb_tower *dev) +static inline void tower_delete(struct kref *kref) { + struct lego_usb_tower *dev = container_of(kref, struct lego_usb_tower, kref); + /* free data structures */ usb_free_urb(dev->interrupt_in_urb); usb_free_urb(dev->interrupt_out_urb); @@ -381,6 +383,8 @@ static int tower_open(struct inode *inode, struct file *file) dev->open_count = 1; + kref_get(&dev->kref); + unlock_exit: mutex_unlock(&dev->lock); @@ -404,14 +408,8 @@ static int tower_release(struct inode *inode, struct file *file) mutex_lock(&dev->lock); - if (dev->disconnected) { - /* the device was unplugged before the file was released */ - - /* unlock here as tower_delete frees dev */ - mutex_unlock(&dev->lock); - tower_delete(dev); - goto exit; - } + if (dev->disconnected) + goto out_unlock; /* wait until write transfer is finished */ if (dev->interrupt_out_busy) { @@ -425,7 +423,9 @@ static int tower_release(struct inode *inode, struct file *file) dev->open_count = 0; +out_unlock: mutex_unlock(&dev->lock); + kref_put(&dev->kref, tower_delete); exit: return retval; } @@ -752,6 +752,7 @@ static int tower_probe(struct usb_interface *interface, const struct usb_device_ if (!dev) goto exit; + kref_init(&dev->kref); mutex_init(&dev->lock); dev->udev = usb_get_dev(udev); spin_lock_init(&dev->read_buffer_lock); @@ -828,7 +829,7 @@ static int tower_probe(struct usb_interface *interface, const struct usb_device_ return retval; error: - tower_delete(dev); + kref_put(&dev->kref, tower_delete); return retval; } @@ -856,18 +857,18 @@ static void tower_disconnect(struct usb_interface *interface) mutex_lock(&dev->lock); - /* if the device is not opened, then we clean up right now */ - if (!dev->open_count) { - mutex_unlock(&dev->lock); - tower_delete(dev); - } else { - dev->disconnected = 1; + dev->disconnected = 1; + + if (dev->open_count) { /* wake up pollers */ wake_up_interruptible_all(&dev->read_wait); wake_up_interruptible_all(&dev->write_wait); - mutex_unlock(&dev->lock); } + mutex_unlock(&dev->lock); + + kref_put(&dev->kref, tower_delete); + dev_info(&interface->dev, "LEGO USB Tower #%d now disconnected\n", (minor - LEGO_USB_TOWER_MINOR_BASE)); } diff --git a/drivers/usb/misc/usbio.c b/drivers/usb/misc/usbio.c index 02d1e0760f0c4..3c2474dca8101 100644 --- a/drivers/usb/misc/usbio.c +++ b/drivers/usb/misc/usbio.c @@ -344,6 +344,10 @@ int usbio_bulk_msg(struct auxiliary_device *adev, u8 type, u8 cmd, bool last, if (ibuf_len < bpkt_len) return -ENOSPC; + /* The device must not claim more payload than it actually sent. */ + if (bpkt_len > act - sizeof(*bpkt)) + return -EPROTO; + memcpy(ibuf, bpkt->data, bpkt_len); return bpkt_len; @@ -518,7 +522,7 @@ static int usbio_resume(struct usb_interface *intf) static void usbio_disconnect(struct usb_interface *intf) { struct usbio_device *usbio = usb_get_intfdata(intf); - struct usbio_client *client; + struct usbio_client *client, *next; /* Wakeup any clients waiting for a reply */ usbio->rxdat_len = 0; @@ -535,7 +539,7 @@ static void usbio_disconnect(struct usb_interface *intf) usb_kill_urb(usbio->urb); usb_free_urb(usbio->urb); - list_for_each_entry_reverse(client, &usbio->cli_list, link) { + list_for_each_entry_safe_reverse(client, next, &usbio->cli_list, link) { auxiliary_device_delete(&client->auxdev); auxiliary_device_uninit(&client->auxdev); } diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c index b7d3c44b970e9..1ce48f5832d71 100644 --- a/drivers/usb/misc/uss720.c +++ b/drivers/usb/misc/uss720.c @@ -732,8 +732,11 @@ static int uss720_probe(struct usb_interface *intf, * here. */ ret = get_1284_register(pp, 0, ®, GFP_KERNEL); dev_dbg(&intf->dev, "reg: %7ph\n", priv->reg); - if (ret < 0) + if (ret < 0) { + priv->pp = NULL; + parport_del_port(pp); goto probe_abort; + } ret = usb_find_last_int_in_endpoint(interface, &epd); if (!ret) { diff --git a/drivers/usb/mtu3/mtu3_gadget.c b/drivers/usb/mtu3/mtu3_gadget.c index da29f467943f0..f224f2ee379ab 100644 --- a/drivers/usb/mtu3/mtu3_gadget.c +++ b/drivers/usb/mtu3/mtu3_gadget.c @@ -305,6 +305,7 @@ static int mtu3_gadget_queue(struct usb_ep *ep, if (mtu3_prepare_transfer(mep)) { ret = -EAGAIN; + usb_gadget_unmap_request(&mtu->g, req, mep->is_in); goto error; } diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c index c481208255ebe..c96e5b76f3137 100644 --- a/drivers/usb/serial/digi_acceleport.c +++ b/drivers/usb/serial/digi_acceleport.c @@ -394,12 +394,14 @@ static int digi_write_oob_command(struct usb_serial_port *port, len &= ~3; memcpy(oob_port->write_urb->transfer_buffer, buf, len); oob_port->write_urb->transfer_buffer_length = len; + ret = usb_submit_urb(oob_port->write_urb, GFP_ATOMIC); - if (ret == 0) { - oob_priv->dp_write_urb_in_use = 1; - count -= len; - buf += len; - } + if (ret) + break; + + oob_priv->dp_write_urb_in_use = 1; + count -= len; + buf += len; } spin_unlock_irqrestore(&oob_priv->dp_port_lock, flags); if (ret) @@ -429,20 +431,22 @@ static int digi_write_inb_command(struct usb_serial_port *port, int len; struct digi_port *priv = usb_get_serial_port_data(port); unsigned char *data = port->write_urb->transfer_buffer; + unsigned long expire; unsigned long flags; dev_dbg(&port->dev, "digi_write_inb_command: TOP: port=%d, count=%d\n", priv->dp_port_num, count); if (timeout) - timeout += jiffies; - else - timeout = ULONG_MAX; + expire = jiffies + timeout; spin_lock_irqsave(&priv->dp_port_lock, flags); while (count > 0 && ret == 0) { - while (priv->dp_write_urb_in_use && - time_before(jiffies, timeout)) { + while (priv->dp_write_urb_in_use) { + if (timeout && time_after(jiffies, expire)) { + ret = -ETIMEDOUT; + break; + } cond_wait_interruptible_timeout_irqrestore( &priv->write_wait, DIGI_RETRY_TIMEOUT, &priv->dp_port_lock, flags); @@ -451,6 +455,9 @@ static int digi_write_inb_command(struct usb_serial_port *port, spin_lock_irqsave(&priv->dp_port_lock, flags); } + if (ret) + break; + /* len must be a multiple of 4 and small enough to */ /* guarantee the write will send buffered data first, */ /* so commands are in order with data and not split */ @@ -1071,6 +1078,7 @@ static int digi_open(struct tty_struct *tty, struct usb_serial_port *port) unsigned char buf[32]; struct digi_port *priv = usb_get_serial_port_data(port); struct ktermios not_termios; + int throttled; /* be sure the device is started up */ if (digi_startup_device(port->serial) != 0) @@ -1098,6 +1106,21 @@ static int digi_open(struct tty_struct *tty, struct usb_serial_port *port) not_termios.c_iflag = ~tty->termios.c_iflag; digi_set_termios(tty, port, ¬_termios); } + + spin_lock_irq(&priv->dp_port_lock); + throttled = priv->dp_throttle_restart; + priv->dp_throttled = 0; + priv->dp_throttle_restart = 0; + spin_unlock_irq(&priv->dp_port_lock); + + if (throttled) { + ret = usb_submit_urb(port->read_urb, GFP_KERNEL); + if (ret) { + dev_err(&port->dev, "failed to submit read urb: %d\n", ret); + return ret; + } + } + return 0; } diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c index 5036600dd334c..377f5695cfc6a 100644 --- a/drivers/usb/serial/keyspan_pda.c +++ b/drivers/usb/serial/keyspan_pda.c @@ -518,7 +518,7 @@ static int keyspan_pda_write_start(struct usb_serial_port *port) if (count == room) schedule_work(&priv->unthrottle_work); - return count; + return 0; } static void keyspan_pda_write_bulk_callback(struct urb *urb) diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index a34e79cfd5b60..7a0f37feb9f22 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -1326,6 +1326,22 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_CC864_SINGLE) }, { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_DE910_DUAL) }, { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_UE910_V2) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0990, 0xff, 0xff, 0x30), /* Telit FE990D50 (RNDIS) */ + .driver_info = NCTRL(6) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0990, 0xff, 0xff, 0x40) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0990, 0xff, 0xff, 0x60) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0991, 0xff, 0xff, 0x30), /* Telit FE990D50 (rmnet) */ + .driver_info = NCTRL(5) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0991, 0xff, 0xff, 0x40) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0991, 0xff, 0xff, 0x60) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0992, 0xff, 0xff, 0x30), /* Telit FE990D50 (MBIM) */ + .driver_info = NCTRL(6) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0992, 0xff, 0xff, 0x40) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0992, 0xff, 0xff, 0x60) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0993, 0xff, 0xff, 0x30), /* Telit FE990D50 (ECM) */ + .driver_info = NCTRL(6) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0993, 0xff, 0xff, 0x40) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x0993, 0xff, 0xff, 0x60) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1031, 0xff), /* Telit LE910C1-EUX */ .driver_info = NCTRL(0) | RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1033, 0xff), /* Telit LE910C1-EUX (ECM) */ diff --git a/drivers/usb/storage/ene_ub6250.c b/drivers/usb/storage/ene_ub6250.c index 8770de01a384d..ed49a3bc859ce 100644 --- a/drivers/usb/storage/ene_ub6250.c +++ b/drivers/usb/storage/ene_ub6250.c @@ -2305,7 +2305,8 @@ static int ene_transport(struct scsi_cmnd *srb, struct us_data *us) /*US_DEBUG(usb_stor_show_command(us, srb)); */ scsi_set_resid(srb, 0); - if (unlikely(!(info->SD_Status & SD_Ready) || (info->MS_Status & MS_Ready))) + if (unlikely(!(info->SD_Status & SD_Ready) && + !(info->MS_Status & MS_Ready))) result = ene_init(us); if (result == USB_STOR_XFER_GOOD) { result = USB_STOR_TRANSPORT_ERROR; diff --git a/drivers/usb/storage/usb.c b/drivers/usb/storage/usb.c index fa83fe0defe2d..064c7fc8e368a 100644 --- a/drivers/usb/storage/usb.c +++ b/drivers/usb/storage/usb.c @@ -570,7 +570,7 @@ void usb_stor_adjust_quirks(struct usb_device *udev, u64 *fflags) US_FL_INITIAL_READ10 | US_FL_WRITE_CACHE | US_FL_NO_ATA_1X | US_FL_NO_REPORT_OPCODES | US_FL_MAX_SECTORS_240 | US_FL_NO_REPORT_LUNS | - US_FL_ALWAYS_SYNC); + US_FL_ALWAYS_SYNC | US_FL_NO_SAME); p = quirks; while (*p) { diff --git a/drivers/usb/typec/anx7411.c b/drivers/usb/typec/anx7411.c index 2e8ae1d2faf98..db917b05dcdeb 100644 --- a/drivers/usb/typec/anx7411.c +++ b/drivers/usb/typec/anx7411.c @@ -1537,7 +1537,9 @@ static int anx7411_i2c_probe(struct i2c_client *client) if (anx7411_typec_check_connection(plat)) dev_err(dev, "check status\n"); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + goto free_wq; return 0; diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index 0977581ad1b6e..0595e8cb83aa6 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -1619,6 +1619,7 @@ static ssize_t select_usb_power_delivery_store(struct device *dev, return -EINVAL; ret = port->ops->pd_set(port, pd); + put_device(&pd->dev); if (ret) return ret; diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index db5e4a4c0a996..9b908c46bd7df 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -275,9 +275,7 @@ static int mux_fwnode_match(struct device *dev, const void *fwnode) static void *typec_mux_match(const struct fwnode_handle *fwnode, const char *id, void *data) { - struct typec_mux_dev **mux_devs = data; struct device *dev; - int i; /* * Device graph (OF graph) does not give any means to identify the @@ -293,14 +291,6 @@ static void *typec_mux_match(const struct fwnode_handle *fwnode, dev = class_find_device(&typec_mux_class, NULL, fwnode, mux_fwnode_match); - /* Skip duplicates */ - for (i = 0; i < TYPEC_MUX_MAX_DEVS; i++) - if (to_typec_mux_dev(dev) == mux_devs[i]) { - put_device(dev); - return NULL; - } - - return dev ? to_typec_mux_dev(dev) : ERR_PTR(-EPROBE_DEFER); } @@ -326,8 +316,7 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) return ERR_PTR(-ENOMEM); count = fwnode_connection_find_matches(fwnode, "mode-switch", - (void **)mux_devs, - typec_mux_match, + NULL, typec_mux_match, (void **)mux_devs, ARRAY_SIZE(mux_devs)); if (count <= 0) { diff --git a/drivers/usb/typec/mux/ps883x.c b/drivers/usb/typec/mux/ps883x.c index 1256252eceedc..c9bf4b4c2ba1e 100644 --- a/drivers/usb/typec/mux/ps883x.c +++ b/drivers/usb/typec/mux/ps883x.c @@ -118,12 +118,12 @@ static int ps883x_set(struct ps883x_retimer *retimer, struct typec_retimer_state CONN_STATUS_1_DP_HPD_LEVEL; switch (state->mode) { + case TYPEC_DP_STATE_D: + cfg0 |= CONN_STATUS_0_USB_3_1_CONNECTED; + fallthrough; case TYPEC_DP_STATE_C: cfg1 |= CONN_STATUS_1_DP_SINK_REQUESTED | CONN_STATUS_1_DP_PIN_ASSIGNMENT_C_D; - fallthrough; - case TYPEC_DP_STATE_D: - cfg1 |= CONN_STATUS_0_USB_3_1_CONNECTED; break; default: /* MODE_E */ break; diff --git a/drivers/usb/typec/tcpm/tcpci_rt1711h.c b/drivers/usb/typec/tcpm/tcpci_rt1711h.c index 4b3e4e22a82e8..0af95c2d33fc4 100644 --- a/drivers/usb/typec/tcpm/tcpci_rt1711h.c +++ b/drivers/usb/typec/tcpm/tcpci_rt1711h.c @@ -295,6 +295,8 @@ static int rt1711h_sw_reset(struct rt1711h_chip *chip) return 0; } +static void rt1711h_unregister_tcpci_port(void *tcpci); + static int rt1711h_probe(struct i2c_client *client) { int ret; @@ -340,6 +342,10 @@ static int rt1711h_probe(struct i2c_client *client) if (IS_ERR_OR_NULL(chip->tcpci)) return PTR_ERR(chip->tcpci); + ret = devm_add_action_or_reset(chip->dev, rt1711h_unregister_tcpci_port, chip->tcpci); + if (ret) + return ret; + ret = devm_request_threaded_irq(chip->dev, client->irq, NULL, rt1711h_irq, IRQF_ONESHOT | IRQF_TRIGGER_LOW, @@ -357,11 +363,9 @@ static int rt1711h_probe(struct i2c_client *client) return 0; } -static void rt1711h_remove(struct i2c_client *client) +static void rt1711h_unregister_tcpci_port(void *tcpci) { - struct rt1711h_chip *chip = i2c_get_clientdata(client); - - tcpci_unregister_port(chip->tcpci); + tcpci_unregister_port(tcpci); } static const struct rt1711h_chip_info rt1711h = { @@ -394,7 +398,6 @@ static struct i2c_driver rt1711h_i2c_driver = { .of_match_table = rt1711h_of_match, }, .probe = rt1711h_probe, - .remove = rt1711h_remove, .id_table = rt1711h_id, }; module_i2c_driver(rt1711h_i2c_driver); diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 7ef746a90a177..89eec20a2064c 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -2000,6 +2000,11 @@ static void svdm_consume_modes(struct tcpm_port *port, const u32 *p, int cnt, return; } + if (pmdata->svid_index < 0 || pmdata->svid_index >= pmdata->nsvids) { + tcpm_log(port, "Invalid SVID index %d", pmdata->svid_index); + return; + } + for (i = 1; i < cnt; i++) { if (pmdata->altmodes >= ALTMODE_DISCOVERY_MAX) { /* Already logged in svdm_consume_svids() */ @@ -3088,7 +3093,7 @@ static int tcpm_altmode_enter(struct typec_altmode *altmode, u32 *vdo) if (svdm_version < 0) return svdm_version; - header = VDO(altmode->svid, vdo ? 2 : 1, svdm_version, CMD_ENTER_MODE); + header = VDO(altmode->svid, 1, svdm_version, CMD_ENTER_MODE); header |= VDO_OPOS(altmode->mode); return tcpm_queue_vdm_unlocked(port, header, vdo, vdo ? 1 : 0, TCPC_TX_SOP); @@ -3136,7 +3141,7 @@ static int tcpm_cable_altmode_enter(struct typec_altmode *altmode, enum typec_pl if (svdm_version < 0) return svdm_version; - header = VDO(altmode->svid, vdo ? 2 : 1, svdm_version, CMD_ENTER_MODE); + header = VDO(altmode->svid, 1, svdm_version, CMD_ENTER_MODE); header |= VDO_OPOS(altmode->mode); return tcpm_queue_vdm_unlocked(port, header, vdo, vdo ? 1 : 0, TCPC_TX_SOP_PRIME); diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c index 67a0991a7b769..7067f2561b846 100644 --- a/drivers/usb/typec/ucsi/displayport.c +++ b/drivers/usb/typec/ucsi/displayport.c @@ -166,12 +166,12 @@ static int ucsi_displayport_status_update(struct ucsi_dp *dp) * that Multi-function is preferred. */ if (DP_CAP_CAPABILITY(cap) & DP_CAP_UFP_D) { - dp->data.status |= DP_STATUS_CON_UFP_D; + dp->data.status |= DP_STATUS_CON_DFP_D; if (DP_CAP_UFP_D_PIN_ASSIGN(cap) & BIT(DP_PIN_ASSIGN_D)) dp->data.status |= DP_STATUS_PREFER_MULTI_FUNC; } else { - dp->data.status |= DP_STATUS_CON_DFP_D; + dp->data.status |= DP_STATUS_CON_UFP_D; if (DP_CAP_DFP_D_PIN_ASSIGN(cap) & BIT(DP_PIN_ASSIGN_D)) dp->data.status |= DP_STATUS_PREFER_MULTI_FUNC; @@ -185,13 +185,12 @@ static int ucsi_displayport_status_update(struct ucsi_dp *dp) static int ucsi_displayport_configure(struct ucsi_dp *dp) { - u32 pins = DP_CONF_GET_PIN_ASSIGN(dp->data.conf); u64 command; if (!dp->override) return 0; - command = UCSI_CMD_SET_NEW_CAM(dp->con->num, 1, dp->offset, pins); + command = UCSI_CMD_SET_NEW_CAM(dp->con->num, 1, dp->offset, dp->data.conf); return ucsi_send_command(dp->con->ucsi, command, NULL, 0); } diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 61cb24ed820f8..66b8799571659 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -1978,6 +1978,26 @@ static void ucsi_resume_work(struct work_struct *work) } } +int ucsi_suspend(struct ucsi *ucsi) +{ + int i; + + /* + * Cancel pending work so it cannot access the firmware after the ACPI + * EC is stopped for suspend; state is re-read on resume. + */ + cancel_delayed_work_sync(&ucsi->work); + + if (!ucsi->connector) + return 0; + + for (i = 0; i < ucsi->cap.num_connectors; i++) + cancel_work_sync(&ucsi->connector[i].work); + + return 0; +} +EXPORT_SYMBOL_GPL(ucsi_suspend); + int ucsi_resume(struct ucsi *ucsi) { if (ucsi->connector) diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h index cff9ddc2ae215..dd64ddc42118b 100644 --- a/drivers/usb/typec/ucsi/ucsi.h +++ b/drivers/usb/typec/ucsi/ucsi.h @@ -566,6 +566,7 @@ int ucsi_send_command(struct ucsi *ucsi, u64 command, void *retval, size_t size); void ucsi_altmode_update_active(struct ucsi_connector *con); +int ucsi_suspend(struct ucsi *ucsi); int ucsi_resume(struct ucsi *ucsi); void ucsi_notify_common(struct ucsi *ucsi, u32 cci); diff --git a/drivers/usb/typec/ucsi/ucsi_acpi.c b/drivers/usb/typec/ucsi/ucsi_acpi.c index 6b92f296e9850..94b52ccf6f426 100644 --- a/drivers/usb/typec/ucsi/ucsi_acpi.c +++ b/drivers/usb/typec/ucsi/ucsi_acpi.c @@ -245,6 +245,13 @@ static void ucsi_acpi_remove(struct platform_device *pdev) ucsi_acpi_notify); } +static int ucsi_acpi_suspend(struct device *dev) +{ + struct ucsi_acpi *ua = dev_get_drvdata(dev); + + return ucsi_suspend(ua->ucsi); +} + static int ucsi_acpi_resume(struct device *dev) { struct ucsi_acpi *ua = dev_get_drvdata(dev); @@ -252,7 +259,8 @@ static int ucsi_acpi_resume(struct device *dev) return ucsi_resume(ua->ucsi); } -static DEFINE_SIMPLE_DEV_PM_OPS(ucsi_acpi_pm_ops, NULL, ucsi_acpi_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(ucsi_acpi_pm_ops, ucsi_acpi_suspend, + ucsi_acpi_resume); static const struct acpi_device_id ucsi_acpi_match[] = { { "PNP0CA0", 0 }, diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c index 4463c1ae96bd4..51498329a4d50 100644 --- a/drivers/usb/typec/ucsi/ucsi_ccg.c +++ b/drivers/usb/typec/ucsi/ucsi_ccg.c @@ -1519,8 +1519,8 @@ static void ucsi_ccg_remove(struct i2c_client *client) cancel_work_sync(&uc->work); pm_runtime_disable(uc->dev); ucsi_unregister(uc->ucsi); - ucsi_destroy(uc->ucsi); free_irq(uc->irq, uc); + ucsi_destroy(uc->ucsi); } static const struct of_device_id ucsi_ccg_of_match_table[] = { diff --git a/drivers/usb/usbip/vudc.h b/drivers/usb/usbip/vudc.h index faf61c9c6a989..5ef0e7d9b23ad 100644 --- a/drivers/usb/usbip/vudc.h +++ b/drivers/usb/usbip/vudc.h @@ -38,7 +38,6 @@ struct vep { struct vrequest { struct usb_request req; - struct vudc *udc; struct list_head req_entry; /* Request queue */ }; diff --git a/drivers/usb/usbip/vudc_dev.c b/drivers/usb/usbip/vudc_dev.c index c5f079c5a1ea5..5ef88117965d1 100644 --- a/drivers/usb/usbip/vudc_dev.c +++ b/drivers/usb/usbip/vudc_dev.c @@ -333,7 +333,6 @@ static int vep_queue(struct usb_ep *_ep, struct usb_request *_req, static int vep_dequeue(struct usb_ep *_ep, struct usb_request *_req) { struct vep *ep; - struct vrequest *req; struct vudc *udc; struct vrequest *lst; unsigned long flags; @@ -343,8 +342,7 @@ static int vep_dequeue(struct usb_ep *_ep, struct usb_request *_req) return ret; ep = to_vep(_ep); - req = to_vrequest(_req); - udc = req->udc; + udc = ep_to_vudc(ep); if (!udc->driver) return -ESHUTDOWN; diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c index 31a02e7fd7f27..df8af6c1454cc 100644 --- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c +++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2024 Marvell. */ +#include #include #include #include @@ -47,7 +48,7 @@ static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev) static irqreturn_t octep_vdpa_intr_handler(int irq, void *data) { struct octep_hw *oct_hw = data; - int i; + int i, start_ring_idx = -1; /* Each device has multiple interrupts (nb_irqs) shared among rings * (nr_vring). Device interrupts are mapped to the rings in a @@ -60,7 +61,16 @@ static irqreturn_t octep_vdpa_intr_handler(int irq, void *data) * 7 -> 7, 15, 23, 31, 39, 47, 55, 63; */ - for (i = irq - oct_hw->irqs[0]; i < oct_hw->nr_vring; i += oct_hw->nb_irqs) { + for (i = 0; i < oct_hw->nb_irqs; i++) { + if (oct_hw->irqs[i] == irq) { + start_ring_idx = i; + break; + } + } + if (start_ring_idx == -1) + return IRQ_NONE; + + for (i = start_ring_idx; i < oct_hw->nr_vring; i += oct_hw->nb_irqs) { if (ioread8(oct_hw->vqs[i].cb_notify_addr)) { /* Acknowledge the per ring notification to the device */ iowrite8(0, oct_hw->vqs[i].cb_notify_addr); @@ -722,6 +732,8 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs) bool done = false; int index = 0; int ret, i; + u8 rpvf; + u64 val; ret = pci_enable_sriov(pdev, num_vfs); if (ret) @@ -741,9 +753,11 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs) } } + val = readq(addr + OCTEP_EPF_RINFO(0)); + rpvf = FIELD_GET(GENMASK_ULL(35, 32), val); if (done) { for (i = 0; i < pf->enabled_vfs; i++) - writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i)); + writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i * rpvf)); } return num_vfs; diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index 806cec32c4bc5..4dc76c0d0d13d 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -124,7 +124,7 @@ static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain, if (!map->bounce_page) { head_map = &domain->bounce_maps[(iova & PAGE_MASK) >> BOUNCE_MAP_SHIFT]; if (!head_map->bounce_page) { - tmp_page = alloc_page(GFP_ATOMIC); + tmp_page = alloc_page(GFP_ATOMIC | __GFP_ZERO); if (!tmp_page) return -ENOMEM; if (cmpxchg(&head_map->bounce_page, NULL, tmp_page)) diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 6202f6902fcdf..f15ad425e01fa 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -221,6 +221,12 @@ static void vduse_enqueue_msg(struct list_head *head, list_add_tail(&msg->list, head); } +static void vduse_enqueue_msg_head(struct list_head *head, + struct vduse_dev_msg *msg) +{ + list_add(&msg->list, head); +} + static void vduse_dev_broken(struct vduse_dev *dev) { struct vduse_dev_msg *msg, *tmp; @@ -358,6 +364,7 @@ static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) struct file *file = iocb->ki_filp; struct vduse_dev *dev = file->private_data; struct vduse_dev_msg *msg; + struct vduse_dev_request req; int size = sizeof(struct vduse_dev_request); ssize_t ret; @@ -369,12 +376,11 @@ static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) msg = vduse_dequeue_msg(&dev->send_list); if (msg) break; + spin_unlock(&dev->msg_lock); - ret = -EAGAIN; if (file->f_flags & O_NONBLOCK) - goto unlock; + return -EAGAIN; - spin_unlock(&dev->msg_lock); ret = wait_event_interruptible_exclusive(dev->waitq, !list_empty(&dev->send_list)); if (ret) @@ -382,17 +388,34 @@ static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) spin_lock(&dev->msg_lock); } + + memcpy(&req, &msg->req, sizeof(req)); + /* + * We must ensure vduse_msg is on send_list or recv_list before unlock + * dev->msg_lock. Because vduse_dev_msg_sync() may be timeout when we + * copy data to userspace, and will call list_del() for this msg. + */ + vduse_enqueue_msg(&dev->recv_list, msg); spin_unlock(&dev->msg_lock); - ret = copy_to_iter(&msg->req, size, to); - spin_lock(&dev->msg_lock); + + ret = copy_to_iter(&req, size, to); if (ret != size) { + /* + * Roll back: move msg back to send_list if still pending. + * + * NOTE: + * vduse_find_msg() must use req.request_id instead of `msg`. + * A malicious userspace may reply to this request, and wake up + * the caller, after which `msg` will have already been freed. + * And here vduse_find_msg() will return NULL then do nothing. + */ + spin_lock(&dev->msg_lock); + msg = vduse_find_msg(&dev->recv_list, req.request_id); + if (msg) + vduse_enqueue_msg_head(&dev->send_list, msg); + spin_unlock(&dev->msg_lock); ret = -EFAULT; - vduse_enqueue_msg(&dev->send_list, msg); - goto unlock; } - vduse_enqueue_msg(&dev->recv_list, msg); -unlock: - spin_unlock(&dev->msg_lock); return ret; } @@ -976,7 +999,7 @@ static void *vduse_dev_alloc_coherent(union virtio_map token, size_t size, if (!token.group) return NULL; - addr = alloc_pages_exact(size, flag); + addr = alloc_pages_exact(size, flag | __GFP_ZERO); if (!addr) return NULL; @@ -1618,6 +1641,127 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, return ret; } +#ifdef CONFIG_COMPAT_FOR_U64_ALIGNMENT +/* + * i386 has different alignment constraints than x86_64, + * so there are only 3 bytes of padding instead of 7. + */ +struct compat_vduse_iotlb_entry { + compat_u64 offset; + compat_u64 start; + compat_u64 last; + __u8 perm; + __u8 padding[3]; +}; +#define COMPAT_VDUSE_IOTLB_GET_FD _IOWR(VDUSE_BASE, 0x10, struct compat_vduse_iotlb_entry) + +struct compat_vduse_vq_info { + __u32 index; + __u32 num; + compat_u64 desc_addr; + compat_u64 driver_addr; + compat_u64 device_addr; + union { + struct vduse_vq_state_split split; + struct vduse_vq_state_packed packed; + }; + __u8 ready; + __u8 padding[3]; +}; +#define COMPAT_VDUSE_VQ_GET_INFO _IOWR(VDUSE_BASE, 0x15, struct compat_vduse_vq_info) + +static long vduse_dev_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct vduse_dev *dev = file->private_data; + void __user *argp = (void __user *)arg; + int ret; + + if (unlikely(dev->broken)) + return -EPERM; + + switch (cmd) { + case COMPAT_VDUSE_IOTLB_GET_FD: { + struct vduse_iotlb_entry_v2 entry = {0}; + struct file *f = NULL; + + ret = -EFAULT; + if (copy_from_user(&entry, argp, _IOC_SIZE(cmd))) + break; + + ret = vduse_dev_iotlb_entry(dev, &entry, &f, NULL); + if (ret) + break; + + ret = -EINVAL; + if (!f) + break; + + ret = copy_to_user(argp, &entry, _IOC_SIZE(cmd)); + if (ret) { + ret = -EFAULT; + fput(f); + break; + } + ret = receive_fd(f, NULL, perm_to_file_flags(entry.perm)); + fput(f); + break; + } + case COMPAT_VDUSE_VQ_GET_INFO: { + struct vduse_vq_info vq_info = {}; + struct vduse_virtqueue *vq; + u32 index; + + ret = -EFAULT; + if (copy_from_user(&vq_info, argp, + sizeof(struct compat_vduse_vq_info))) + break; + + ret = -EINVAL; + if (vq_info.index >= dev->vq_num) + break; + + index = array_index_nospec(vq_info.index, dev->vq_num); + vq = dev->vqs[index]; + vq_info.desc_addr = vq->desc_addr; + vq_info.driver_addr = vq->driver_addr; + vq_info.device_addr = vq->device_addr; + vq_info.num = vq->num; + + if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { + vq_info.packed.last_avail_counter = + vq->state.packed.last_avail_counter; + vq_info.packed.last_avail_idx = + vq->state.packed.last_avail_idx; + vq_info.packed.last_used_counter = + vq->state.packed.last_used_counter; + vq_info.packed.last_used_idx = + vq->state.packed.last_used_idx; + } else + vq_info.split.avail_index = + vq->state.split.avail_index; + + vq_info.ready = vq->ready; + + ret = -EFAULT; + if (copy_to_user(argp, &vq_info, + sizeof(struct compat_vduse_vq_info))) + break; + + ret = 0; + break; + } + default: + ret = -ENOIOCTLCMD; + break; + } + + return vduse_dev_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#else +#define vduse_dev_compat_ioctl compat_ptr_ioctl +#endif + static int vduse_dev_release(struct inode *inode, struct file *file) { struct vduse_dev *dev = file->private_data; @@ -1637,26 +1781,18 @@ static int vduse_dev_release(struct inode *inode, struct file *file) return 0; } -static struct vduse_dev *vduse_dev_get_from_minor(int minor) +static int vduse_dev_open(struct inode *inode, struct file *file) { + int ret = -EBUSY; struct vduse_dev *dev; mutex_lock(&vduse_lock); - dev = idr_find(&vduse_idr, minor); - mutex_unlock(&vduse_lock); - - return dev; -} - -static int vduse_dev_open(struct inode *inode, struct file *file) -{ - int ret; - struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode)); - - if (!dev) + dev = idr_find(&vduse_idr, iminor(inode)); + if (!dev) { + mutex_unlock(&vduse_lock); return -ENODEV; + } - ret = -EBUSY; mutex_lock(&dev->lock); if (dev->connected) goto unlock; @@ -1666,6 +1802,7 @@ static int vduse_dev_open(struct inode *inode, struct file *file) file->private_data = dev; unlock: mutex_unlock(&dev->lock); + mutex_unlock(&vduse_lock); return ret; } @@ -1678,7 +1815,7 @@ static const struct file_operations vduse_dev_fops = { .write_iter = vduse_dev_write_iter, .poll = vduse_dev_poll, .unlocked_ioctl = vduse_dev_ioctl, - .compat_ioctl = compat_ptr_ioctl, + .compat_ioctl = vduse_dev_compat_ioctl, .llseek = noop_llseek, }; diff --git a/drivers/vfio/debugfs.c b/drivers/vfio/debugfs.c index 8b0ca7a090640..8a2f1b0cce3f1 100644 --- a/drivers/vfio/debugfs.c +++ b/drivers/vfio/debugfs.c @@ -97,6 +97,7 @@ void vfio_device_debugfs_init(struct vfio_device *vdev) void vfio_device_debugfs_exit(struct vfio_device *vdev) { debugfs_remove_recursive(vdev->debug_root); + vdev->debug_root = NULL; } void vfio_debugfs_create_root(void) diff --git a/drivers/vfio/pci/mlx5/cmd.h b/drivers/vfio/pci/mlx5/cmd.h index deed0f132f39f..c86d8b243a524 100644 --- a/drivers/vfio/pci/mlx5/cmd.h +++ b/drivers/vfio/pci/mlx5/cmd.h @@ -158,26 +158,29 @@ struct mlx5_vhca_qp { struct mlx5_vhca_page_tracker { u32 id; u32 pdn; - u8 is_err:1; - u8 object_changed:1; + /* Flags modified at runtime - dedicated storage unit */ + u8 is_err; + u8 object_changed; + int status; struct mlx5_uars_page *uar; struct mlx5_vhca_cq cq; struct mlx5_vhca_qp *host_qp; struct mlx5_vhca_qp *fw_qp; struct mlx5_nb nb; - int status; }; struct mlx5vf_pci_core_device { struct vfio_pci_core_device core_device; int vf_id; u16 vhca_id; + /* Flags only modified on setup/release - bitfield ok */ u8 migrate_cap:1; - u8 deferred_reset:1; - u8 mdev_detach:1; - u8 log_active:1; u8 chunk_mode:1; u8 mig_state_cap:1; + /* Flags modified at runtime - dedicated storage unit */ + u8 mdev_detach; + u8 log_active; + u8 deferred_reset; struct completion tracker_comp; /* protect migration state */ struct mutex state_mutex; diff --git a/drivers/vfio/pci/qat/main.c b/drivers/vfio/pci/qat/main.c index ac9652539d66a..60ff907b6a67a 100644 --- a/drivers/vfio/pci/qat/main.c +++ b/drivers/vfio/pci/qat/main.c @@ -298,14 +298,18 @@ static ssize_t qat_vf_resume_write(struct file *filp, const char __user *buf, return -ESPIPE; offs = &filp->f_pos; - if (*offs < 0 || - check_add_overflow(len, *offs, &end)) - return -EOVERFLOW; + mutex_lock(&migf->lock); - if (end > mig_dev->state_size) - return -ENOMEM; + if (*offs < 0 || check_add_overflow(len, *offs, &end)) { + done = -EOVERFLOW; + goto out_unlock; + } + + if (end > mig_dev->state_size) { + done = -ENOMEM; + goto out_unlock; + } - mutex_lock(&migf->lock); if (migf->disabled) { done = -ENODEV; goto out_unlock; diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 050e7542952ed..beb1674c69450 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -271,8 +271,11 @@ int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t stat int ret; /* Prevent changing power state for PFs with VFs enabled */ - if (pci_num_vf(pdev) && state > PCI_D0) - return -EBUSY; + if (state > PCI_D0) { + lockdep_assert_held_write(&vdev->memory_lock); + if (vdev->sriov_active) + return -EBUSY; + } if (vdev->needs_pm_restore) { if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) { @@ -535,7 +538,7 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) u16 cmd; u8 msix_pos; - if (!disable_idle_d3) { + if (!vdev->disable_idle_d3) { ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; @@ -614,7 +617,7 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) out_disable_device: pci_disable_device(pdev); out_power: - if (!disable_idle_d3) + if (!vdev->disable_idle_d3) pm_runtime_put(&pdev->dev); return ret; } @@ -750,7 +753,7 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) vfio_pci_dev_set_try_reset(vdev->vdev.dev_set); /* Put the pm-runtime usage counter acquired during enable */ - if (!disable_idle_d3) + if (!vdev->disable_idle_d3) pm_runtime_put(&pdev->dev); } EXPORT_SYMBOL_GPL(vfio_pci_core_disable); @@ -2142,6 +2145,8 @@ int vfio_pci_core_init_dev(struct vfio_device *core_vdev) init_rwsem(&vdev->memory_lock); xa_init(&vdev->ctx); + vdev->disable_idle_d3 = disable_idle_d3; + return 0; } EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev); @@ -2237,7 +2242,7 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) dev->driver->pm = &vfio_pci_core_pm_ops; pm_runtime_allow(dev); - if (!disable_idle_d3) + if (!vdev->disable_idle_d3) pm_runtime_put(dev); ret = vfio_register_group_dev(&vdev->vdev); @@ -2246,10 +2251,11 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) return 0; out_power: - if (!disable_idle_d3) + if (!vdev->disable_idle_d3) pm_runtime_get_noresume(dev); pm_runtime_forbid(dev); + vfio_pci_vga_uninit(vdev); out_vf: vfio_pci_vf_uninit(vdev); return ret; @@ -2265,7 +2271,7 @@ void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) vfio_pci_vf_uninit(vdev); vfio_pci_vga_uninit(vdev); - if (!disable_idle_d3) + if (!vdev->disable_idle_d3) pm_runtime_get_noresume(&vdev->pdev->dev); pm_runtime_forbid(&vdev->pdev->dev); @@ -2327,8 +2333,9 @@ int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, down_write(&vdev->memory_lock); vfio_pci_set_power_state(vdev, PCI_D0); - ret = pci_enable_sriov(pdev, nr_virtfn); + vdev->sriov_active = true; up_write(&vdev->memory_lock); + ret = pci_enable_sriov(pdev, nr_virtfn); if (ret) { pm_runtime_put(&pdev->dev); goto out_del; @@ -2342,6 +2349,13 @@ int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, } out_del: + /* + * Avoid taking the memory_lock intentionally. A race with a power + * state transition would at most result in an -EBUSY, leaving the + * device in PCI_D0. + */ + vdev->sriov_active = false; + mutex_lock(&vfio_pci_sriov_pfs_mutex); list_del_init(&vdev->sriov_pfs_item); out_unlock: @@ -2589,7 +2603,7 @@ static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set) * state. Increment the usage count for all the devices in the dev_set * before reset and decrement the same after reset. */ - if (!disable_idle_d3 && vfio_pci_dev_set_pm_runtime_get(dev_set)) + if (vfio_pci_dev_set_pm_runtime_get(dev_set)) return; if (!pci_reset_bus(pdev)) @@ -2599,8 +2613,7 @@ static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set) if (reset_done) cur->needs_reset = false; - if (!disable_idle_d3) - pm_runtime_put(&cur->pdev->dev); + pm_runtime_put(&cur->pdev->dev); } } diff --git a/drivers/vfio/pci/xe/main.c b/drivers/vfio/pci/xe/main.c index 4ecadbbfd86ec..cbff5af385ef5 100644 --- a/drivers/vfio/pci/xe/main.c +++ b/drivers/vfio/pci/xe/main.c @@ -135,8 +135,6 @@ static void xe_vfio_pci_reset_done(struct pci_dev *pdev) } spin_unlock(&xe_vdev->reset_lock); xe_vfio_pci_state_mutex_unlock(xe_vdev); - - xe_vfio_pci_reset(xe_vdev); } static const struct pci_error_handlers xe_vfio_pci_err_handlers = { diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 6222376ab6ab5..ed538aebb0b8e 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -406,6 +406,13 @@ void vfio_unregister_group_dev(struct vfio_device *device) */ vfio_device_group_unregister(device); + /* + * Remove debugfs before device_del(), which releases devres. Some + * debugfs entries are created with debugfs_create_devm_seqfile() and + * therefore rely on devres-managed inode private data. + */ + vfio_device_debugfs_exit(device); + /* * Balances vfio_device_add() in register path, also prevents * new device opened by userspace in the cdev path. @@ -435,7 +442,6 @@ void vfio_unregister_group_dev(struct vfio_device *device) } } - vfio_device_debugfs_exit(device); /* Balances vfio_device_set_group in register path */ vfio_device_remove_group(device); } @@ -858,7 +864,8 @@ int vfio_mig_get_next_state(struct vfio_device *device, * logical state, as per the above comment. */ *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; - while ((state_flags_table[*next_fsm] & device->migration_flags) != + while (*next_fsm != VFIO_DEVICE_STATE_ERROR && + (state_flags_table[*next_fsm] & device->migration_flags) != state_flags_table[*next_fsm]) *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index c6536cad9c4f9..b9af63fb63060 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -390,13 +390,20 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net, static void vhost_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *ubuf_base, bool success) { - struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base); - struct vhost_net_ubuf_ref *ubufs = ubuf->ctx; - struct vhost_virtqueue *vq = ubufs->vq; + struct ubuf_info_msgzc *ubuf; + struct vhost_net_ubuf_ref *ubufs; + struct vhost_virtqueue *vq; int cnt; - rcu_read_lock_bh(); + /* Only the final cloned skb reference completes the vhost descriptor. */ + if (!refcount_dec_and_test(&ubuf_base->refcnt)) + return; + + ubuf = uarg_to_msgzc(ubuf_base); + ubufs = ubuf->ctx; + vq = ubufs->vq; + rcu_read_lock_bh(); /* set len to mark this desc buffers done DMA */ vq->heads[ubuf->desc].len = success ? VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN; diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 692564b1bcbbe..ac55275fa0d0a 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -1482,16 +1482,32 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) } #ifdef CONFIG_MMU -static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) +static int +vhost_vdpa_get_vq_notification(struct vhost_vdpa *v, unsigned long index, + struct vdpa_notification_area *notify) { - struct vhost_vdpa *v = vmf->vma->vm_file->private_data; struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; + + if (index > 65535 || index >= v->nvqs) + return -EINVAL; + + index = array_index_nospec(index, v->nvqs); + + *notify = ops->get_vq_notification(vdpa, index); + + return 0; +} + +static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) +{ + struct vhost_vdpa *v = vmf->vma->vm_file->private_data; struct vdpa_notification_area notify; struct vm_area_struct *vma = vmf->vma; - u16 index = vma->vm_pgoff; + unsigned long index = vma->vm_pgoff; - notify = ops->get_vq_notification(vdpa, index); + if (vhost_vdpa_get_vq_notification(v, index, ¬ify)) + return VM_FAULT_SIGBUS; return vmf_insert_pfn(vma, vmf->address & PAGE_MASK, PFN_DOWN(notify.addr)); } @@ -1514,8 +1530,6 @@ static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) return -EINVAL; if (vma->vm_flags & VM_READ) return -EINVAL; - if (index > 65535) - return -EINVAL; if (!ops->get_vq_notification) return -ENOTSUPP; @@ -1523,7 +1537,8 @@ static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) * support the doorbell which sits on the page boundary and * does not share the page with other registers. */ - notify = ops->get_vq_notification(vdpa, index); + if (vhost_vdpa_get_vq_notification(v, index, ¬ify)) + return -EINVAL; if (notify.addr & (PAGE_SIZE - 1)) return -EINVAL; if (vma->vm_end - vma->vm_start != notify.size) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 2f2c45d208832..db329a6f61458 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -1522,6 +1522,7 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq) { __virtio16 idx; + u16 avail_idx; int r; r = vhost_get_avail(vq, idx, &vq->avail->idx); @@ -1532,17 +1533,19 @@ static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq) } /* Check it isn't doing very strange thing with available indexes */ - vq->avail_idx = vhost16_to_cpu(vq, idx); - if (unlikely((u16)(vq->avail_idx - vq->last_avail_idx) > vq->num)) { + avail_idx = vhost16_to_cpu(vq, idx); + if (unlikely((u16)(avail_idx - vq->last_avail_idx) > vq->num)) { vq_err(vq, "Invalid available index change from %u to %u", - vq->last_avail_idx, vq->avail_idx); + vq->last_avail_idx, avail_idx); return -EINVAL; } /* We're done if there is nothing new */ - if (vq->avail_idx == vq->last_avail_idx) + if (avail_idx == vq->avail_idx) return 0; + vq->avail_idx = avail_idx; + /* * We updated vq->avail_idx so we need a memory barrier between * the index read above and the caller reading avail ring entries. diff --git a/drivers/video/backlight/ktd2801-backlight.c b/drivers/video/backlight/ktd2801-backlight.c index 17eac1b3bce4a..1b1307e03b204 100644 --- a/drivers/video/backlight/ktd2801-backlight.c +++ b/drivers/video/backlight/ktd2801-backlight.c @@ -53,6 +53,7 @@ static int ktd2801_update_status(struct backlight_device *bd) } static const struct backlight_ops ktd2801_backlight_ops = { + .options = BL_CORE_SUSPENDRESUME, .update_status = ktd2801_update_status, }; diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c index cb006484831b6..2bed904e71e3b 100644 --- a/drivers/video/fbdev/aty/radeon_base.c +++ b/drivers/video/fbdev/aty/radeon_base.c @@ -2476,6 +2476,7 @@ static int radeonfb_pci_register(struct pci_dev *pdev, return 0; err_unmap_fb: iounmap(rinfo->fb_base); + fb_destroy_modelist(&info->modelist); err_unmap_rom: kfree(rinfo->mon1_EDID); kfree(rinfo->mon2_EDID); diff --git a/drivers/video/fbdev/broadsheetfb.c b/drivers/video/fbdev/broadsheetfb.c index c8ba098a8c42f..582f1ee4c9b6d 100644 --- a/drivers/video/fbdev/broadsheetfb.c +++ b/drivers/video/fbdev/broadsheetfb.c @@ -1072,12 +1072,14 @@ static int broadsheetfb_probe(struct platform_device *dev) info->flags = FBINFO_VIRTFB; info->fbdefio = &broadsheetfb_defio; - fb_deferred_io_init(info); + retval = fb_deferred_io_init(info); + if (retval) + goto err_vfree; retval = fb_alloc_cmap(&info->cmap, 16, 0); if (retval < 0) { dev_err(&dev->dev, "Failed to allocate colormap\n"); - goto err_vfree; + goto err_fbdefio; } /* set cmap */ @@ -1121,6 +1123,8 @@ static int broadsheetfb_probe(struct platform_device *dev) board->cleanup(par); err_cmap: fb_dealloc_cmap(&info->cmap); +err_fbdefio: + fb_deferred_io_cleanup(info); err_vfree: vfree(videomemory); err_fb_rel: diff --git a/drivers/video/fbdev/carminefb.c b/drivers/video/fbdev/carminefb.c index bd4bff6a24844..18f02dd881e1c 100644 --- a/drivers/video/fbdev/carminefb.c +++ b/drivers/video/fbdev/carminefb.c @@ -589,6 +589,7 @@ static int alloc_carmine_fb(void __iomem *regs, void __iomem *smem_base, return 0; err_dealloc_cmap: + fb_destroy_modelist(&info->modelist); fb_dealloc_cmap(&info->cmap); err_free_fb: framebuffer_release(info); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index b0e3e765360d1..666293f0864f3 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1274,6 +1274,7 @@ static void fbcon_deinit(struct vc_data *vc) int idx; fbcon_free_font(p); + p->mode = NULL; idx = con2fb_map[vc->vc_num]; if (idx == -1) @@ -1440,19 +1441,18 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var, struct vc_data **default_mode, *vc; struct vc_data *svc; struct fbcon_par *par = info->fbcon_par; - int rows, cols; - unsigned long ret = 0; + int rows, cols, ret; p = &fb_display[unit]; - if (var_to_display(p, var, info)) - return; - vc = vc_cons[unit].d; if (!vc) return; + if (var_to_display(p, var, info)) + return; + default_mode = vc->vc_display_fg; svc = *default_mode; t = &fb_display[svc->vc_num]; @@ -2407,6 +2407,7 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount, int resize, ret, old_width, old_height, old_charcount; font_data_t *old_fontdata = p->fontdata; const u8 *old_data = vc->vc_font.data; + unsigned short old_hi_font_mask = vc->vc_hi_font_mask; font_data_get(data); @@ -2453,6 +2454,12 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount, vc->vc_font.height = old_height; vc->vc_font.charcount = old_charcount; + /* Restore the hi_font state and screen buffer */ + if (old_hi_font_mask && !vc->vc_hi_font_mask) + set_vc_hi_font(vc, true); + else if (!old_hi_font_mask && vc->vc_hi_font_mask) + set_vc_hi_font(vc, false); + font_data_put(data); return ret; diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c index 30f2b59c47bf9..0929deb952185 100644 --- a/drivers/video/fbdev/core/fbmem.c +++ b/drivers/video/fbdev/core/fbmem.c @@ -734,6 +734,18 @@ int fb_new_modelist(struct fb_info *info) if (list_empty(&info->modelist)) return 1; + /* + * The new modelist may not contain the current mode (info->var), and + * fbcon_new_modelist() below only re-points consoles mapped to this + * framebuffer. Add the current mode here so info->var keeps a match + * even when fbcon is unbound. + */ + if (!fb_match_mode(&info->var, &info->modelist)) { + fb_var_to_videomode(&mode, &info->var); + if (fb_add_videomode(&mode, &info->modelist)) + return 1; + } + fbcon_new_modelist(info); return 0; diff --git a/drivers/video/fbdev/core/fbsysfs.c b/drivers/video/fbdev/core/fbsysfs.c index baa2bae0fb5b3..fe8bd33e64ab1 100644 --- a/drivers/video/fbdev/core/fbsysfs.c +++ b/drivers/video/fbdev/core/fbsysfs.c @@ -11,6 +11,7 @@ #include #include "fb_internal.h" +#include "fbcon.h" static int activate(struct fb_info *fb_info, struct fb_var_screeninfo *var) { @@ -111,8 +112,15 @@ static ssize_t store_modes(struct device *device, if (fb_new_modelist(fb_info)) { fb_destroy_modelist(&fb_info->modelist); list_splice(&old_list, &fb_info->modelist); - } else + } else { + /* + * fb_display[i].mode and fb_info->mode both point into the old + * list. Clear them before it is freed. + */ + fbcon_delete_modelist(&old_list); + fb_info->mode = NULL; fb_destroy_modelist(&old_list); + } unlock_fb_info(fb_info); console_unlock(); diff --git a/drivers/video/fbdev/core/modedb.c b/drivers/video/fbdev/core/modedb.c index 703d0b7aec322..e1fd9298a702c 100644 --- a/drivers/video/fbdev/core/modedb.c +++ b/drivers/video/fbdev/core/modedb.c @@ -259,7 +259,7 @@ static const struct fb_videomode modedb[] = { FB_VMODE_DOUBLE }, /* 1920x1080 @ 60 Hz, 67.3 kHz hsync */ - { NULL, 60, 1920, 1080, 6734, 148, 88, 36, 4, 44, 5, 0, + { NULL, 60, 1920, 1080, 6734, 148, 88, 36, 4, 44, 5, FB_SYNC_HOR_HIGH_ACT | FB_SYNC_VERT_HIGH_ACT, FB_VMODE_NONINTERLACED }, @@ -626,7 +626,7 @@ int fb_find_mode(struct fb_var_screeninfo *var, const struct fb_videomode *default_mode, unsigned int default_bpp) { - char *mode_option_buf = NULL; + char *mode_option_buf __free(kfree) = NULL; int i; /* Set up defaults */ @@ -724,7 +724,6 @@ int fb_find_mode(struct fb_var_screeninfo *var, res_specified = 1; } done: - kfree(mode_option_buf); if (cvt) { struct fb_videomode cvt_mode; int ret; diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c index 47ebc0107209e..ad8dec7807c3a 100644 --- a/drivers/video/fbdev/efifb.c +++ b/drivers/video/fbdev/efifb.c @@ -377,6 +377,7 @@ static int efifb_probe(struct platform_device *dev) if (fb_get_options("efifb", &option)) return -ENODEV; efifb_setup(si, option); + kfree(option); /* We don't get linelength from UGA Draw Protocol, only from * EFI Graphics Protocol. So if it's not in DMI, and it's not diff --git a/drivers/video/fbdev/hecubafb.c b/drivers/video/fbdev/hecubafb.c index 3547d58a29cf2..dd2af980f3d8b 100644 --- a/drivers/video/fbdev/hecubafb.c +++ b/drivers/video/fbdev/hecubafb.c @@ -192,7 +192,9 @@ static int hecubafb_probe(struct platform_device *dev) info->flags = FBINFO_VIRTFB; info->fbdefio = &hecubafb_defio; - fb_deferred_io_init(info); + retval = fb_deferred_io_init(info); + if (retval) + goto err_fbdefio; retval = register_framebuffer(info); if (retval < 0) @@ -209,6 +211,8 @@ static int hecubafb_probe(struct platform_device *dev) return 0; err_fbreg: + fb_deferred_io_cleanup(info); +err_fbdefio: framebuffer_release(info); err_fballoc: vfree(videomemory); diff --git a/drivers/video/fbdev/i740fb.c b/drivers/video/fbdev/i740fb.c index 9b74dae71472c..c14a19382769a 100644 --- a/drivers/video/fbdev/i740fb.c +++ b/drivers/video/fbdev/i740fb.c @@ -1152,6 +1152,7 @@ static int i740fb_probe(struct pci_dev *dev, const struct pci_device_id *ent) fb_dealloc_cmap(&info->cmap); err_alloc_cmap: err_find_mode: + fb_destroy_modelist(&info->modelist); if (par->ddc_registered) i2c_del_adapter(&par->ddc_adapter); pci_iounmap(dev, par->regs); diff --git a/drivers/video/fbdev/metronomefb.c b/drivers/video/fbdev/metronomefb.c index 6f0942c6e5f1c..83c614963a0a6 100644 --- a/drivers/video/fbdev/metronomefb.c +++ b/drivers/video/fbdev/metronomefb.c @@ -645,12 +645,14 @@ static int metronomefb_probe(struct platform_device *dev) info->flags = FBINFO_VIRTFB; info->fbdefio = &metronomefb_defio; - fb_deferred_io_init(info); + retval = fb_deferred_io_init(info); + if (retval) + goto err_free_irq; retval = fb_alloc_cmap(&info->cmap, 8, 0); if (retval < 0) { dev_err(&dev->dev, "Failed to allocate colormap\n"); - goto err_free_irq; + goto err_fbdefio; } /* set cmap */ @@ -673,6 +675,8 @@ static int metronomefb_probe(struct platform_device *dev) err_cmap: fb_dealloc_cmap(&info->cmap); +err_fbdefio: + fb_deferred_io_cleanup(info); err_free_irq: board->cleanup(par); err_csum_table: diff --git a/drivers/video/fbdev/nvidia/nvidia.c b/drivers/video/fbdev/nvidia/nvidia.c index 72b85f475605d..c9dcc9e899487 100644 --- a/drivers/video/fbdev/nvidia/nvidia.c +++ b/drivers/video/fbdev/nvidia/nvidia.c @@ -1421,6 +1421,7 @@ static int nvidiafb_probe(struct pci_dev *pd, const struct pci_device_id *ent) err_out_iounmap_fb: iounmap(info->screen_base); + fb_destroy_modelist(&info->modelist); err_out_free_base1: fb_destroy_modedb(info->monspecs.modedb); nvidia_delete_i2c_busses(par); diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c index d70deb6a91508..046892682fc62 100644 --- a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c +++ b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c @@ -1099,7 +1099,11 @@ static int omapfb_mmap(struct fb_info *fbi, struct vm_area_struct *vma) rg = omapfb_get_mem_region(ofbi->region); - start = omapfb_get_region_paddr(ofbi); + if (ofbi->rotation_type == OMAP_DSS_ROT_VRFB) + start = rg->vrfb.paddr[0]; + else + start = rg->paddr; + len = fix->smem_len; DBG("user mmap region start %lx, len %d, off %lx\n", start, len, @@ -1109,6 +1113,8 @@ static int omapfb_mmap(struct fb_info *fbi, struct vm_area_struct *vma) vma->vm_ops = &mmap_user_ops; vma->vm_private_data = rg; + atomic_inc(&rg->map_count); + r = vm_iomap_memory(vma, start, len); if (r) goto error; @@ -1121,6 +1127,7 @@ static int omapfb_mmap(struct fb_info *fbi, struct vm_area_struct *vma) return 0; error: + atomic_dec(&rg->map_count); omapfb_put_mem_region(rg); return r; diff --git a/drivers/video/fbdev/s3fb.c b/drivers/video/fbdev/s3fb.c index ba30e5568cabc..d7ee2ff07c7c9 100644 --- a/drivers/video/fbdev/s3fb.c +++ b/drivers/video/fbdev/s3fb.c @@ -1446,6 +1446,7 @@ static int s3_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) err_alloc_cmap: err_find_mode: #ifdef CONFIG_FB_S3_DDC + fb_destroy_modelist(&info->modelist); if (par->ddc_registered) i2c_del_adapter(&par->ddc_adapter); if (par->mmio) diff --git a/drivers/video/fbdev/sm501fb.c b/drivers/video/fbdev/sm501fb.c index fee4b9f84592f..ea5375ed4ea6f 100644 --- a/drivers/video/fbdev/sm501fb.c +++ b/drivers/video/fbdev/sm501fb.c @@ -96,6 +96,7 @@ struct sm501fb_info { void __iomem *fbmem; /* remapped framebuffer */ size_t fbmem_len; /* length of remapped region */ u8 *edid_data; + char *fb_mode; }; /* per-framebuffer private data */ @@ -1793,12 +1794,11 @@ static int sm501fb_init_fb(struct fb_info *fb, enum sm501_controller head, fb->var.yres_virtual = fb->var.yres; } else { if (info->edid_data) { - ret = fb_find_mode(&fb->var, fb, fb_mode, + ret = fb_find_mode(&fb->var, fb, + info->fb_mode ?: fb_mode, fb->monspecs.modedb, fb->monspecs.modedb_len, &sm501_default_mode, default_bpp); - /* edid_data is no longer needed, free it */ - kfree(info->edid_data); } else { ret = fb_find_mode(&fb->var, fb, NULL, NULL, 0, NULL, 8); @@ -1974,7 +1974,7 @@ static int sm501fb_probe(struct platform_device *pdev) /* Get EDID */ cp = of_get_property(np, "mode", &len); if (cp) - strcpy(fb_mode, cp); + info->fb_mode = kstrdup(cp, GFP_KERNEL); prop = of_get_property(np, "edid", &len); if (prop && len == EDID_LENGTH) { info->edid_data = kmemdup(prop, EDID_LENGTH, @@ -2031,6 +2031,12 @@ static int sm501fb_probe(struct platform_device *pdev) goto err_started_crt; } + /* These aren't needed any more */ + kfree(info->edid_data); + kfree(info->fb_mode); + info->edid_data = NULL; + info->fb_mode = NULL; + /* we registered, return ok */ return 0; @@ -2048,6 +2054,8 @@ static int sm501fb_probe(struct platform_device *pdev) framebuffer_release(info->fb[HEAD_CRT]); err_alloc: + kfree(info->edid_data); + kfree(info->fb_mode); kfree(info); return ret; diff --git a/drivers/video/fbdev/sm712.h b/drivers/video/fbdev/sm712.h index c7ebf03b8d537..83fe25fc61f2c 100644 --- a/drivers/video/fbdev/sm712.h +++ b/drivers/video/fbdev/sm712.h @@ -101,7 +101,7 @@ struct modeinit { #define mmio_addr 0x00800000 #define seqw17() smtc_seqw(0x17, 0x30) #define big_pixel_depth(p, d) {if (p == 24) {p = 32; d = 32; } } -#define big_swap(p) ((p & 0xff00ff00 >> 8) | (p & 0x00ff00ff << 8)) +#define big_swap(p) (((p & 0xff00ff00) >> 8) | ((p & 0x00ff00ff) << 8)) #else #define pal_rgb(r, g, b, val) val #define big_addr 0 diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index 4c4e53aaea3ad..ac74ccda4108c 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -1551,6 +1551,7 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) out_err_iobase: #ifdef CONFIG_FB_3DFX_I2C + fb_destroy_modelist(&info->modelist); tdfxfb_delete_i2c_busses(default_par); #endif arch_phys_wc_del(default_par->wc_cookie); diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c index 17b7253b8fbec..5c7ccf76348e4 100644 --- a/drivers/video/fbdev/tridentfb.c +++ b/drivers/video/fbdev/tridentfb.c @@ -1706,6 +1706,7 @@ static int trident_pci_probe(struct pci_dev *dev, return 0; out_unmap2: + fb_destroy_modelist(&info->modelist); if (default_par->ddc_registered) i2c_del_adapter(&default_par->ddc_adapter); kfree(info->pixmap.addr); diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c index 88667fccc27b9..9d82326c744f2 100644 --- a/drivers/video/fbdev/uvesafb.c +++ b/drivers/video/fbdev/uvesafb.c @@ -1694,14 +1694,14 @@ static int uvesafb_probe(struct platform_device *dev) i = uvesafb_vbe_init_mode(info); if (i < 0) { err = -EINVAL; - goto out; + goto out_mode; } else { mode = &par->vbe_modes[i]; } if (fb_alloc_cmap(&info->cmap, 256, 0) < 0) { err = -ENXIO; - goto out; + goto out_mode; } uvesafb_init_info(info, mode); diff --git a/drivers/video/fbdev/vesafb.c b/drivers/video/fbdev/vesafb.c index f84f4db244bfe..f1902056bd733 100644 --- a/drivers/video/fbdev/vesafb.c +++ b/drivers/video/fbdev/vesafb.c @@ -269,6 +269,7 @@ static int vesafb_probe(struct platform_device *dev) /* ignore error return of fb_get_options */ fb_get_options("vesafb", &option); vesafb_setup(option); + kfree(option); if (si->orig_video_isVGA != VIDEO_TYPE_VLFB) return -ENODEV; diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 595c2274fbb53..510b7c4efdff8 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -662,9 +662,7 @@ static void virtio_mmio_remove(struct platform_device *pdev) #if defined(CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES) -static struct device vm_cmdline_parent = { - .init_name = "virtio-mmio-cmdline", -}; +static struct device *vm_cmdline_parent; static int vm_cmdline_parent_registered; static int vm_cmdline_id; @@ -672,7 +670,6 @@ static int vm_cmdline_id; static int vm_cmdline_set(const char *device, const struct kernel_param *kp) { - int err; struct resource resources[2] = {}; char *str; long long base, size; @@ -704,11 +701,10 @@ static int vm_cmdline_set(const char *device, resources[1].start = resources[1].end = irq; if (!vm_cmdline_parent_registered) { - err = device_register(&vm_cmdline_parent); - if (err) { - put_device(&vm_cmdline_parent); + vm_cmdline_parent = __root_device_register("virtio-mmio-cmdline", NULL); + if (IS_ERR(vm_cmdline_parent)) { pr_err("Failed to register parent device!\n"); - return err; + return PTR_ERR(vm_cmdline_parent); } vm_cmdline_parent_registered = 1; } @@ -719,7 +715,7 @@ static int vm_cmdline_set(const char *device, (unsigned long long)resources[0].end, (int)resources[1].start); - pdev = platform_device_register_resndata(&vm_cmdline_parent, + pdev = platform_device_register_resndata(vm_cmdline_parent, "virtio-mmio", vm_cmdline_id++, resources, ARRAY_SIZE(resources), NULL, 0); @@ -743,8 +739,12 @@ static int vm_cmdline_get_device(struct device *dev, void *data) static int vm_cmdline_get(char *buffer, const struct kernel_param *kp) { buffer[0] = '\0'; - device_for_each_child(&vm_cmdline_parent, buffer, - vm_cmdline_get_device); + + if (vm_cmdline_parent_registered) { + device_for_each_child(vm_cmdline_parent, buffer, + vm_cmdline_get_device); + } + return strlen(buffer) + 1; } @@ -766,9 +766,9 @@ static int vm_unregister_cmdline_device(struct device *dev, static void vm_unregister_cmdline_devices(void) { if (vm_cmdline_parent_registered) { - device_for_each_child(&vm_cmdline_parent, NULL, + device_for_each_child(vm_cmdline_parent, NULL, vm_unregister_cmdline_device); - device_unregister(&vm_cmdline_parent); + root_device_unregister(vm_cmdline_parent); vm_cmdline_parent_registered = 0; } } diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index da97b6a988deb..164f480b18a6f 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -423,10 +423,11 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, vqs[i] = NULL; continue; } - vqs[i] = vp_find_one_vq_msix(vdev, queue_idx++, vqi->callback, + vqs[i] = vp_find_one_vq_msix(vdev, queue_idx, vqi->callback, vqi->name, vqi->ctx, false, &allocated_vectors, vector_policy, - &vp_dev->vqs[i]); + &vp_dev->vqs[queue_idx]); + queue_idx++; if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto error_find; @@ -485,9 +486,10 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, vqs[i] = NULL; continue; } - vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback, + vqs[i] = vp_setup_vq(vdev, queue_idx, vqi->callback, vqi->name, vqi->ctx, - VIRTIO_MSI_NO_VECTOR, &vp_dev->vqs[i]); + VIRTIO_MSI_NO_VECTOR, &vp_dev->vqs[queue_idx]); + queue_idx++; if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto out_del_vqs; diff --git a/drivers/virtio/virtio_rtc_driver.c b/drivers/virtio/virtio_rtc_driver.c index a57d5e06e19d2..4419735b0f0dc 100644 --- a/drivers/virtio/virtio_rtc_driver.c +++ b/drivers/virtio/virtio_rtc_driver.c @@ -1257,6 +1257,15 @@ static int viortc_init_vqs(struct viortc_dev *viortc) return 0; } +static void __viortc_remove(struct viortc_dev *viortc) +{ + struct virtio_device *vdev = viortc->vdev; + + viortc_clocks_deinit(viortc); + virtio_reset_device(vdev); + vdev->config->del_vqs(vdev); +} + /** * viortc_probe() - probe a virtio_rtc virtio device * @vdev: virtio device @@ -1282,7 +1291,7 @@ static int viortc_probe(struct virtio_device *vdev) ret = viortc_init_vqs(viortc); if (ret) - return ret; + goto err_reset_vdev; virtio_device_ready(vdev); @@ -1329,10 +1338,7 @@ static void viortc_remove(struct virtio_device *vdev) { struct viortc_dev *viortc = vdev->priv; - viortc_clocks_deinit(viortc); - - virtio_reset_device(vdev); - vdev->config->del_vqs(vdev); + __viortc_remove(viortc); } static int viortc_freeze(struct virtio_device *dev) @@ -1353,9 +1359,11 @@ static int viortc_restore(struct virtio_device *dev) bool notify = false; int ret; + dev->config->del_vqs(dev); + ret = viortc_init_vqs(viortc); if (ret) - return ret; + goto err_remove; alarm_viortc_vq = &viortc->vqs[VIORTC_ALARMQ]; alarm_vq = alarm_viortc_vq->vq; @@ -1364,7 +1372,7 @@ static int viortc_restore(struct virtio_device *dev) ret = viortc_populate_vq(viortc, alarm_viortc_vq, VIORTC_ALARMQ_BUF_CAP, false); if (ret) - return ret; + goto err_remove; notify = virtqueue_kick_prepare(alarm_vq); } @@ -1372,8 +1380,12 @@ static int viortc_restore(struct virtio_device *dev) virtio_device_ready(dev); if (notify && !virtqueue_notify(alarm_vq)) - ret = -EIO; + return -EIO; + + return 0; +err_remove: + __viortc_remove(viortc); return ret; } diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c index 66a158f67a712..6b9b0f9b05ced 100644 --- a/drivers/watchdog/apple_wdt.c +++ b/drivers/watchdog/apple_wdt.c @@ -218,6 +218,7 @@ static int apple_wdt_suspend(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_wdt_pm_ops, apple_wdt_suspend, apple_wdt_resume); static const struct of_device_id apple_wdt_of_match[] = { + { .compatible = "apple,t8103-wdt" }, { .compatible = "apple,wdt" }, {}, }; diff --git a/drivers/watchdog/sama5d4_wdt.c b/drivers/watchdog/sama5d4_wdt.c index 13e72918338a2..704b786cc2ec6 100644 --- a/drivers/watchdog/sama5d4_wdt.c +++ b/drivers/watchdog/sama5d4_wdt.c @@ -30,6 +30,7 @@ struct sama5d4_wdt { void __iomem *reg_base; u32 mr; u32 ir; + u32 wddis_mask; unsigned long last_ping; bool need_irq; bool sam9x60_support; @@ -48,7 +49,10 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); -#define wdt_enabled (!(wdt->mr & AT91_WDT_WDDIS)) +static inline bool wdt_enabled(struct sama5d4_wdt *wdt) +{ + return !(wdt->mr & wdt->wddis_mask); +} #define wdt_read(wdt, field) \ readl_relaxed((wdt)->reg_base + (field)) @@ -81,12 +85,9 @@ static int sama5d4_wdt_start(struct watchdog_device *wdd) { struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd); - if (wdt->sam9x60_support) { + if (wdt->sam9x60_support) writel_relaxed(wdt->ir, wdt->reg_base + AT91_SAM9X60_IER); - wdt->mr &= ~AT91_SAM9X60_WDDIS; - } else { - wdt->mr &= ~AT91_WDT_WDDIS; - } + wdt->mr &= ~wdt->wddis_mask; wdt_write(wdt, AT91_WDT_MR, wdt->mr); return 0; @@ -96,12 +97,9 @@ static int sama5d4_wdt_stop(struct watchdog_device *wdd) { struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd); - if (wdt->sam9x60_support) { + if (wdt->sam9x60_support) writel_relaxed(wdt->ir, wdt->reg_base + AT91_SAM9X60_IDR); - wdt->mr |= AT91_SAM9X60_WDDIS; - } else { - wdt->mr |= AT91_WDT_WDDIS; - } + wdt->mr |= wdt->wddis_mask; wdt_write(wdt, AT91_WDT_MR, wdt->mr); return 0; @@ -117,7 +115,7 @@ static int sama5d4_wdt_ping(struct watchdog_device *wdd) } static int sama5d4_wdt_set_timeout(struct watchdog_device *wdd, - unsigned int timeout) + unsigned int timeout) { struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd); u32 value = WDT_SEC2TICKS(timeout); @@ -140,8 +138,8 @@ static int sama5d4_wdt_set_timeout(struct watchdog_device *wdd, * If the watchdog is enabled, then the timeout can be updated. Else, * wait that the user enables it. */ - if (wdt_enabled) - wdt_write(wdt, AT91_WDT_MR, wdt->mr & ~AT91_WDT_WDDIS); + if (wdt_enabled(wdt)) + wdt_write(wdt, AT91_WDT_MR, wdt->mr & ~wdt->wddis_mask); wdd->timeout = timeout; @@ -184,10 +182,7 @@ static int of_sama5d4_wdt_init(struct device_node *np, struct sama5d4_wdt *wdt) { const char *tmp; - if (wdt->sam9x60_support) - wdt->mr = AT91_SAM9X60_WDDIS; - else - wdt->mr = AT91_WDT_WDDIS; + wdt->mr = wdt->wddis_mask; if (!of_property_read_string(np, "atmel,watchdog-type", &tmp) && !strcmp(tmp, "software")) @@ -213,15 +208,11 @@ static int sama5d4_wdt_init(struct sama5d4_wdt *wdt) * If the watchdog is already running, we can safely update it. * Else, we have to disable it properly. */ - if (!wdt_enabled) { + if (!wdt_enabled(wdt)) { reg = wdt_read(wdt, AT91_WDT_MR); - if (wdt->sam9x60_support && (!(reg & AT91_SAM9X60_WDDIS))) - wdt_write_nosleep(wdt, AT91_WDT_MR, - reg | AT91_SAM9X60_WDDIS); - else if (!wdt->sam9x60_support && - (!(reg & AT91_WDT_WDDIS))) + if (!(reg & wdt->wddis_mask)) wdt_write_nosleep(wdt, AT91_WDT_MR, - reg | AT91_WDT_WDDIS); + reg | wdt->wddis_mask); } if (wdt->sam9x60_support) { @@ -273,6 +264,9 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) of_device_is_compatible(dev->of_node, "microchip,sama7g5-wdt")) wdt->sam9x60_support = true; + wdt->wddis_mask = wdt->sam9x60_support ? AT91_SAM9X60_WDDIS + : AT91_WDT_WDDIS; + watchdog_set_drvdata(wdd, wdt); regs = devm_platform_ioremap_resource(pdev, 0); @@ -306,8 +300,8 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) watchdog_init_timeout(wdd, wdt_timeout, dev); reg = wdt_read(wdt, AT91_WDT_MR); - if (!(reg & AT91_WDT_WDDIS)) { - wdt->mr &= ~AT91_WDT_WDDIS; + if (!(reg & wdt->wddis_mask)) { + wdt->mr &= ~wdt->wddis_mask; set_bit(WDOG_HW_RUNNING, &wdd->status); } diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c index 2bd3dc25cb030..7e99c3b1f3676 100644 --- a/drivers/watchdog/sp5100_tco.c +++ b/drivers/watchdog/sp5100_tco.c @@ -92,7 +92,8 @@ static enum tco_reg_layout tco_reg_layout(struct pci_dev *dev) dev->device == PCI_DEVICE_ID_ATI_SBX00_SMBUS && dev->revision < 0x40) { return sp5100; - } else if (dev->vendor == PCI_VENDOR_ID_AMD && + } else if ((dev->vendor == PCI_VENDOR_ID_AMD || + dev->vendor == PCI_VENDOR_ID_HYGON) && sp5100_tco_pci->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS && sp5100_tco_pci->revision >= AMD_ZEN_SMBUS_PCI_REV) { return efch_mmio; diff --git a/drivers/watchdog/sprd_wdt.c b/drivers/watchdog/sprd_wdt.c index 4e689b6ff1418..aacf04616fefe 100644 --- a/drivers/watchdog/sprd_wdt.c +++ b/drivers/watchdog/sprd_wdt.c @@ -320,10 +320,9 @@ static int sprd_wdt_probe(struct platform_device *pdev) watchdog_init_timeout(&wdt->wdd, 0, dev); ret = devm_watchdog_register_device(dev, &wdt->wdd); - if (ret) { - sprd_wdt_disable(wdt); + if (ret) return ret; - } + platform_set_drvdata(pdev, wdt); return 0; diff --git a/drivers/watchdog/watchdog_core.c b/drivers/watchdog/watchdog_core.c index 8300520688d07..6c087b4ef5dd5 100644 --- a/drivers/watchdog/watchdog_core.c +++ b/drivers/watchdog/watchdog_core.c @@ -391,6 +391,9 @@ static void __watchdog_unregister_device(struct watchdog_device *wdd) if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status)) unregister_reboot_notifier(&wdd->reboot_nb); + if (test_bit(WDOG_NO_PING_ON_SUSPEND, &wdd->status)) + unregister_pm_notifier(&wdd->pm_nb); + watchdog_dev_unregister(wdd); ida_free(&watchdog_ida, wdd->id); } diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 61ea855c45087..1dcc4675580ed 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -670,11 +670,15 @@ static long gntdev_ioctl_map_grant_ref(struct gntdev_priv *priv, mutex_lock(&priv->lock); gntdev_add_map(priv, map); op.index = map->index << PAGE_SHIFT; - mutex_unlock(&priv->lock); - if (copy_to_user(u, &op, sizeof(op)) != 0) + if (copy_to_user(u, &op, sizeof(op)) != 0) { + list_del(&map->next); + mutex_unlock(&priv->lock); + gntdev_put_map(priv, map); return -EFAULT; + } + mutex_unlock(&priv->lock); return 0; } diff --git a/drivers/xen/pvcalls-front.c b/drivers/xen/pvcalls-front.c index 50ce4820f7eeb..3e7aa807c3173 100644 --- a/drivers/xen/pvcalls-front.c +++ b/drivers/xen/pvcalls-front.c @@ -32,6 +32,7 @@ struct pvcalls_bedata { struct xen_pvcalls_front_ring ring; grant_ref_t ref; int irq; + bool disabled; struct list_head socket_mappings; spinlock_t socket_lock; @@ -131,6 +132,20 @@ static inline int get_request(struct pvcalls_bedata *bedata, int *req_id) return 0; } +/* + * Wait for the backend's response to req_id, or for the frontend to be + * disabled because the backend violated the wire protocol. Returns 0 once + * the response has arrived, or -EIO if the frontend was disabled. + */ +static int pvcalls_front_wait_rsp(struct pvcalls_bedata *bedata, u32 req_id) +{ + wait_event(bedata->inflight_req, + READ_ONCE(bedata->rsp[req_id].req_id) == req_id || + READ_ONCE(bedata->disabled)); + + return READ_ONCE(bedata->disabled) ? -EIO : 0; +} + static bool pvcalls_front_write_todo(struct sock_mapping *map) { struct pvcalls_data_intf *intf = map->active.ring; @@ -168,7 +183,8 @@ static irqreturn_t pvcalls_front_event_handler(int irq, void *dev_id) struct pvcalls_bedata *bedata; struct xen_pvcalls_response *rsp; uint8_t *src, *dst; - int req_id = 0, more = 0, done = 0; + u32 req_id = 0; + int more = 0, done = 0; if (dev == NULL) return IRQ_HANDLED; @@ -179,12 +195,31 @@ static irqreturn_t pvcalls_front_event_handler(int irq, void *dev_id) pvcalls_exit(); return IRQ_HANDLED; } + if (READ_ONCE(bedata->disabled)) { + pvcalls_exit(); + return IRQ_HANDLED; + } again: while (RING_HAS_UNCONSUMED_RESPONSES(&bedata->ring)) { rsp = RING_GET_RESPONSE(&bedata->ring, bedata->ring.rsp_cons); req_id = rsp->req_id; + if (req_id >= PVCALLS_NR_RSP_PER_RING) { + /* + * The backend supplied a req_id that would index + * bedata->rsp[] out of bounds: a protocol violation + * from a malicious or buggy backend. Log once, stop + * trusting this backend and disable the frontend rather + * than silently dropping the response and continuing. + */ + pr_err_once("pvcalls: backend sent out-of-range req_id %u, disabling frontend\n", + req_id); + WRITE_ONCE(bedata->disabled, true); + bedata->ring.rsp_cons++; + done = 1; + break; + } if (rsp->cmd == PVCALLS_POLL) { struct sock_mapping *map = (struct sock_mapping *)(uintptr_t) rsp->u.poll.id; @@ -217,7 +252,7 @@ static irqreturn_t pvcalls_front_event_handler(int irq, void *dev_id) } RING_FINAL_CHECK_FOR_RESPONSES(&bedata->ring, more); - if (more) + if (more && !READ_ONCE(bedata->disabled)) goto again; if (done) wake_up(&bedata->inflight_req); @@ -330,8 +365,11 @@ int pvcalls_front_socket(struct socket *sock) if (notify) notify_remote_via_irq(bedata->irq); - wait_event(bedata->inflight_req, - READ_ONCE(bedata->rsp[req_id].req_id) == req_id); + ret = pvcalls_front_wait_rsp(bedata, req_id); + if (ret) { + pvcalls_exit(); + return ret; + } /* read req_id, then the content */ smp_rmb(); @@ -477,8 +515,11 @@ int pvcalls_front_connect(struct socket *sock, struct sockaddr *addr, if (notify) notify_remote_via_irq(bedata->irq); - wait_event(bedata->inflight_req, - READ_ONCE(bedata->rsp[req_id].req_id) == req_id); + ret = pvcalls_front_wait_rsp(bedata, req_id); + if (ret) { + pvcalls_exit_sock(sock); + return ret; + } /* read req_id, then the content */ smp_rmb(); @@ -711,8 +752,11 @@ int pvcalls_front_bind(struct socket *sock, struct sockaddr *addr, int addr_len) if (notify) notify_remote_via_irq(bedata->irq); - wait_event(bedata->inflight_req, - READ_ONCE(bedata->rsp[req_id].req_id) == req_id); + ret = pvcalls_front_wait_rsp(bedata, req_id); + if (ret) { + pvcalls_exit_sock(sock); + return ret; + } /* read req_id, then the content */ smp_rmb(); @@ -761,8 +805,11 @@ int pvcalls_front_listen(struct socket *sock, int backlog) if (notify) notify_remote_via_irq(bedata->irq); - wait_event(bedata->inflight_req, - READ_ONCE(bedata->rsp[req_id].req_id) == req_id); + ret = pvcalls_front_wait_rsp(bedata, req_id); + if (ret) { + pvcalls_exit_sock(sock); + return ret; + } /* read req_id, then the content */ smp_rmb(); @@ -820,6 +867,14 @@ int pvcalls_front_accept(struct socket *sock, struct socket *newsock, } } + if (READ_ONCE(bedata->disabled)) { + clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT, + (void *)&map->passive.flags); + wake_up(&map->passive.inflight_accept_req); + pvcalls_exit_sock(sock); + return -EIO; + } + map2 = kzalloc_obj(*map2); if (map2 == NULL) { clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT, @@ -880,10 +935,18 @@ int pvcalls_front_accept(struct socket *sock, struct socket *newsock, } if (wait_event_interruptible(bedata->inflight_req, - READ_ONCE(bedata->rsp[req_id].req_id) == req_id)) { + READ_ONCE(bedata->rsp[req_id].req_id) == req_id || + READ_ONCE(bedata->disabled))) { pvcalls_exit_sock(sock); return -EINTR; } + if (READ_ONCE(bedata->disabled)) { + clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT, + (void *)&map->passive.flags); + wake_up(&map->passive.inflight_accept_req); + pvcalls_exit_sock(sock); + return -EIO; + } /* read req_id, then the content */ smp_rmb(); @@ -1054,7 +1117,8 @@ int pvcalls_front_release(struct socket *sock) notify_remote_via_irq(bedata->irq); wait_event(bedata->inflight_req, - READ_ONCE(bedata->rsp[req_id].req_id) == req_id); + READ_ONCE(bedata->rsp[req_id].req_id) == req_id || + READ_ONCE(bedata->disabled)); if (map->active_socket) { /* diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c index e33f95c91b096..c7036e0e41bda 100644 --- a/drivers/xen/xen-scsiback.c +++ b/drivers/xen/xen-scsiback.c @@ -611,6 +611,25 @@ static void scsiback_disconnect(struct vscsibk_info *info) xenbus_unmap_ring_vfree(info->dev, info->ring.sring); } +/* + * Send the error response for a request that did not reach the target core + * and return its tag. Free the tag before the response drops the v2p + * reference that keeps the session alive, and snapshot what the response + * needs since returning the tag can let the slot be reused. + */ +static void scsiback_resp_and_free(struct vscsibk_pend *pending_req, + int32_t result) +{ + struct vscsibk_info *info = pending_req->info; + struct v2p_entry *v2p = pending_req->v2p; + struct se_session *se_sess = v2p->tpg->tpg_nexus->tvn_se_sess; + u16 rqid = pending_req->rqid; + + target_free_tag(se_sess, &pending_req->se_cmd); + scsiback_send_response(info, NULL, result, 0, rqid); + kref_put(&v2p->kref, scsiback_free_translation_entry); +} + static void scsiback_device_action(struct vscsibk_pend *pending_req, enum tcm_tmreq_table act, int tag) { @@ -639,7 +658,7 @@ static void scsiback_device_action(struct vscsibk_pend *pending_req, return; err: - scsiback_do_resp_with_sense(NULL, err, 0, pending_req); + scsiback_resp_and_free(pending_req, err); } /* @@ -792,9 +811,8 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info, case VSCSIIF_ACT_SCSI_CDB: if (scsiback_gnttab_data_map(&ring_req, pending_req)) { scsiback_fast_flush_area(pending_req); - scsiback_do_resp_with_sense(NULL, - DID_ERROR << 16, 0, pending_req); - transport_generic_free_cmd(&pending_req->se_cmd, 0); + scsiback_resp_and_free(pending_req, + DID_ERROR << 16); } else { scsiback_cmd_exec(pending_req); } @@ -808,9 +826,7 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info, break; default: pr_err_ratelimited("invalid request\n"); - scsiback_do_resp_with_sense(NULL, DID_ERROR << 16, 0, - pending_req); - transport_generic_free_cmd(&pending_req->se_cmd, 0); + scsiback_resp_and_free(pending_req, DID_ERROR << 16); break; } diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index f468acb8ee7df..952792ce2dee9 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -488,10 +488,19 @@ static int v9fs_at_to_dotl_flags(int flags) * - ext4 (with dir_nlink feature enabled) sets nlink to 1 if a dir has more * than EXT4_LINK_MAX (65000) links. * + * In cacheless mode the server is the source of truth for nlink and the + * inode is going away immediately, so locally adjusting i_nlink buys + * nothing and races with concurrent metadata fetches that may already + * have observed the post-unlink value (nlink == 0). + * * @inode: inode whose nlink is being dropped */ static void v9fs_dec_count(struct inode *inode) { + struct v9fs_session_info *v9ses = v9fs_inode2v9ses(inode); + + if (!(v9ses->cache & (CACHE_META | CACHE_LOOSE))) + return; if (!S_ISDIR(inode->i_mode) || inode->i_nlink > 2) drop_nlink(inode); } @@ -672,27 +681,20 @@ v9fs_vfs_create(struct mnt_idmap *idmap, struct inode *dir, static struct dentry *v9fs_vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode) { - int err; u32 perm; struct p9_fid *fid; struct v9fs_session_info *v9ses; p9_debug(P9_DEBUG_VFS, "name %pd\n", dentry); - err = 0; v9ses = v9fs_inode2v9ses(dir); perm = unixmode2p9mode(v9ses, mode | S_IFDIR); fid = v9fs_create(v9ses, dir, dentry, NULL, perm, P9_OREAD); - if (IS_ERR(fid)) { - err = PTR_ERR(fid); - fid = NULL; - } else { - inc_nlink(dir); - v9fs_invalidate_inode_attr(dir); - } - - if (fid) - p9_fid_put(fid); - return ERR_PTR(err); + if (IS_ERR(fid)) + return ERR_CAST(fid); + inc_nlink(dir); + v9fs_invalidate_inode_attr(dir); + p9_fid_put(fid); + return NULL; } /** diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 141fb54db65d2..e90808808ea57 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -349,7 +349,7 @@ static struct dentry *v9fs_vfs_mkdir_dotl(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t omode) { - int err; + int err = 0; struct v9fs_session_info *v9ses; struct p9_fid *fid = NULL, *dfid = NULL; kgid_t gid; @@ -412,7 +412,7 @@ static struct dentry *v9fs_vfs_mkdir_dotl(struct mnt_idmap *idmap, p9_fid_put(fid); v9fs_put_acl(dacl, pacl); p9_fid_put(dfid); - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; } static int diff --git a/fs/afs/callback.c b/fs/afs/callback.c index 894d2bad6b6ce..61354003c006c 100644 --- a/fs/afs/callback.c +++ b/fs/afs/callback.c @@ -113,16 +113,12 @@ static struct afs_volume *afs_lookup_volume_rcu(struct afs_cell *cell, { struct afs_volume *volume = NULL; struct rb_node *p; - int seq = 1; - for (;;) { + scoped_seqlock_read(&cell->volume_lock, ss_lock) { /* Unfortunately, rbtree walking doesn't give reliable results * under just the RCU read lock, so we have to check for * changes. */ - seq++; /* 2 on the 1st/lockless path, otherwise odd */ - read_seqbegin_or_lock(&cell->volume_lock, &seq); - p = rcu_dereference_raw(cell->volumes.rb_node); while (p) { volume = rb_entry(p, struct afs_volume, cell_node); @@ -138,12 +134,9 @@ static struct afs_volume *afs_lookup_volume_rcu(struct afs_cell *cell, if (volume && afs_try_get_volume(volume, afs_volume_trace_get_callback)) break; - if (!need_seqretry(&cell->volume_lock, seq)) - break; - seq |= 1; /* Want a lock next time */ + volume = NULL; } - done_seqretry(&cell->volume_lock, seq); return volume; } @@ -221,7 +214,11 @@ static void afs_break_some_callbacks(struct afs_server *server, rcu_read_lock(); volume = afs_lookup_volume_rcu(server->cell, vid); - if (cbb->fid.vnode == 0 && cbb->fid.unique == 0) { + if (!volume) { + /* Ignore breaks on unknown volumes. */ + rcu_read_unlock(); + *_count = 0; + } else if (cbb->fid.vnode == 0 && cbb->fid.unique == 0) { afs_break_volume_callback(server, volume); *_count -= 1; if (*_count) diff --git a/fs/afs/cell.c b/fs/afs/cell.c index 9738684dbdd26..47a2645768d79 100644 --- a/fs/afs/cell.c +++ b/fs/afs/cell.c @@ -206,11 +206,6 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net, cell->dns_status = vllist->status; smp_store_release(&cell->dns_lookup_count, 1); /* vs source/status */ atomic_inc(&net->cells_outstanding); - ret = idr_alloc_cyclic(&net->cells_dyn_ino, cell, - 2, INT_MAX / 2, GFP_KERNEL); - if (ret < 0) - goto error; - cell->dynroot_ino = ret; cell->debug_id = atomic_inc_return(&cell_debug_id); trace_afs_cell(cell->debug_id, 1, 0, afs_cell_trace_alloc); @@ -304,6 +299,13 @@ struct afs_cell *afs_lookup_cell(struct afs_net *net, goto cell_already_exists; } + ret = idr_alloc_cyclic(&net->cells_dyn_ino, candidate, + 2, INT_MAX / 2, GFP_KERNEL); + if (ret < 0) + goto cant_alloc_ino; + candidate->dynroot_ino = ret; + set_bit(AFS_CELL_FL_HAVE_INO, &candidate->flags); + cell = candidate; candidate = NULL; afs_use_cell(cell, trace); @@ -378,6 +380,11 @@ struct afs_cell *afs_lookup_cell(struct afs_net *net, _leave(" = %p [cell]", cell); return cell; +cant_alloc_ino: + up_write(&net->cells_lock); + afs_put_cell(candidate, afs_cell_trace_put_candidate); + goto error_noput; + cell_already_exists: _debug("cell exists"); cell = cursor; @@ -547,6 +554,8 @@ static int afs_update_cell(struct afs_cell *cell) rcu_assign_pointer(cell->vl_servers, vllist); cell->dns_source = vllist->source; old = p; + } else { + old = vllist; } write_unlock(&cell->vl_servers_lock); afs_put_vlserverlist(cell->net, old); @@ -577,7 +586,6 @@ static void afs_cell_destroy(struct rcu_head *rcu) afs_put_vlserverlist(net, rcu_access_pointer(cell->vl_servers)); afs_unuse_cell(cell->alias_of, afs_cell_trace_unuse_alias); key_put(cell->anonymous_key); - idr_remove(&net->cells_dyn_ino, cell->dynroot_ino); kfree(cell->name - 1); kfree(cell); @@ -592,6 +600,13 @@ static void afs_destroy_cell_work(struct work_struct *work) afs_see_cell(cell, afs_cell_trace_destroy); timer_delete_sync(&cell->management_timer); cancel_work_sync(&cell->manager); + + if (test_bit(AFS_CELL_FL_HAVE_INO, &cell->flags)) { + down_write(&cell->net->cells_lock); + idr_remove(&cell->net->cells_dyn_ino, cell->dynroot_ino); + up_write(&cell->net->cells_lock); + } + call_rcu(&cell->rcu, afs_cell_destroy); } diff --git a/fs/afs/cm_security.c b/fs/afs/cm_security.c index edcbd249d2024..103168c70dd4d 100644 --- a/fs/afs/cm_security.c +++ b/fs/afs/cm_security.c @@ -101,7 +101,8 @@ void afs_process_oob_queue(struct work_struct *work) struct sk_buff *oob; enum rxrpc_oob_type type; - while ((oob = rxrpc_kernel_dequeue_oob(net->socket, &type))) { + while (READ_ONCE(net->live) && + (oob = rxrpc_kernel_dequeue_oob(net->socket, &type))) { switch (type) { case RXRPC_OOB_CHALLENGE: afs_respond_to_challenge(oob); diff --git a/fs/afs/cmservice.c b/fs/afs/cmservice.c index 5540ae1cad599..db394f101fc66 100644 --- a/fs/afs/cmservice.c +++ b/fs/afs/cmservice.c @@ -334,7 +334,6 @@ static int afs_deliver_cb_init_call_back_state3(struct afs_call *call) ret = afs_extract_data(call, false); switch (ret) { case 0: break; - case -EAGAIN: return 0; default: return ret; } @@ -364,6 +363,11 @@ static int afs_deliver_cb_init_call_back_state3(struct afs_call *call) if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) return afs_io_error(call, afs_io_error_cm_reply); + if (!call->server) { + trace_afs_cm_no_server_u(call, call->request); + return 0; + } + if (memcmp(call->request, &call->server->_uuid, sizeof(call->server->_uuid)) != 0) { pr_notice("Callback UUID does not match fileserver UUID\n"); trace_afs_cm_no_server_u(call, call->request); @@ -451,7 +455,6 @@ static int afs_deliver_cb_probe_uuid(struct afs_call *call) ret = afs_extract_data(call, false); switch (ret) { case 0: break; - case -EAGAIN: return 0; default: return ret; } diff --git a/fs/afs/dir.c b/fs/afs/dir.c index 498b99ccdf0e2..6df56fe9163fd 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -28,9 +28,11 @@ static int afs_d_revalidate(struct inode *dir, const struct qstr *name, static int afs_d_delete(const struct dentry *dentry); static void afs_d_iput(struct dentry *dentry, struct inode *inode); static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen, - loff_t fpos, u64 ino, unsigned dtype); + u64 ino, u32 uniquifier); +#define AFS_LOOKUP_ONE ((filldir_t)0x123UL) static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen, - loff_t fpos, u64 ino, unsigned dtype); + u64 ino, u32 uniquifier); +#define AFS_LOOKUP ((filldir_t)0x137UL) static int afs_create(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, bool excl); static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir, @@ -421,11 +423,18 @@ static int afs_dir_iterate_block(struct afs_vnode *dvnode, } /* found the next entry */ - if (!dir_emit(ctx, dire->u.name, nlen, - ntohl(dire->u.vnode), - (ctx->actor == afs_lookup_filldir || - ctx->actor == afs_lookup_one_filldir)? - ntohl(dire->u.unique) : DT_UNKNOWN)) { + if (ctx->actor == AFS_LOOKUP) { + if (!afs_lookup_filldir(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), + ntohl(dire->u.unique))) + return 0; + } else if (ctx->actor == AFS_LOOKUP_ONE) { + if (!afs_lookup_one_filldir(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), + ntohl(dire->u.unique))) + return 0; + } else if (!dir_emit(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), DT_UNKNOWN)) { _leave(" = 0 [full]"); return 0; } @@ -545,6 +554,7 @@ static int afs_readdir(struct file *file, struct dir_context *ctx) { afs_dataversion_t dir_version; + ctx->dt_flags_mask = UINT_MAX; return afs_dir_iterate(file_inode(file), ctx, file, &dir_version); } @@ -554,14 +564,14 @@ static int afs_readdir(struct file *file, struct dir_context *ctx) * uniquifier through dtype */ static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, - int nlen, loff_t fpos, u64 ino, unsigned dtype) + int nlen, u64 ino, u32 uniquifier) { struct afs_lookup_one_cookie *cookie = container_of(ctx, struct afs_lookup_one_cookie, ctx); _enter("{%s,%u},%s,%u,,%llu,%u", cookie->name.name, cookie->name.len, name, nlen, - (unsigned long long) ino, dtype); + (unsigned long long) ino, uniquifier); /* insanity checks first */ BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048); @@ -574,7 +584,7 @@ static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, } cookie->fid.vnode = ino; - cookie->fid.unique = dtype; + cookie->fid.unique = uniquifier; cookie->found = 1; _leave(" = false [found]"); @@ -591,7 +601,7 @@ static int afs_do_lookup_one(struct inode *dir, const struct qstr *name, { struct afs_super_info *as = dir->i_sb->s_fs_info; struct afs_lookup_one_cookie cookie = { - .ctx.actor = afs_lookup_one_filldir, + .ctx.actor = AFS_LOOKUP_ONE, .name = *name, .fid.vid = as->volume->vid }; @@ -622,14 +632,14 @@ static int afs_do_lookup_one(struct inode *dir, const struct qstr *name, * uniquifier through dtype */ static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, - int nlen, loff_t fpos, u64 ino, unsigned dtype) + int nlen, u64 ino, u32 uniquifier) { struct afs_lookup_cookie *cookie = container_of(ctx, struct afs_lookup_cookie, ctx); _enter("{%s,%u},%s,%u,,%llu,%u", cookie->name.name, cookie->name.len, name, nlen, - (unsigned long long) ino, dtype); + (unsigned long long) ino, uniquifier); /* insanity checks first */ BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048); @@ -637,7 +647,7 @@ static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, if (cookie->nr_fids < 50) { cookie->fids[cookie->nr_fids].vnode = ino; - cookie->fids[cookie->nr_fids].unique = dtype; + cookie->fids[cookie->nr_fids].unique = uniquifier; cookie->nr_fids++; } @@ -778,7 +788,7 @@ static struct inode *afs_do_lookup(struct inode *dir, struct dentry *dentry) for (i = 0; i < ARRAY_SIZE(cookie->fids); i++) cookie->fids[i].vid = dvnode->fid.vid; - cookie->ctx.actor = afs_lookup_filldir; + cookie->ctx.actor = AFS_LOOKUP; cookie->name = dentry->d_name; cookie->nr_fids = 2; /* slot 1 is saved for the fid we actually want * and slot 0 for the directory */ diff --git a/fs/afs/dynroot.c b/fs/afs/dynroot.c index 1d5e33bc75022..6e3c8c691ba97 100644 --- a/fs/afs/dynroot.c +++ b/fs/afs/dynroot.c @@ -278,7 +278,7 @@ static struct dentry *afs_lookup_atcell(struct inode *dir, struct dentry *dentry } /* - * Transcribe the cell database into readdir content under the RCU read lock. + * Transcribe the cell database into readdir content under net->cells_lock. * Each cell produces two entries, one prefixed with a dot and one not. */ static int afs_dynroot_readdir_cells(struct afs_net *net, struct dir_context *ctx) diff --git a/fs/afs/fs_operation.c b/fs/afs/fs_operation.c index c0dbbc6d3716e..20801b29521d1 100644 --- a/fs/afs/fs_operation.c +++ b/fs/afs/fs_operation.c @@ -348,7 +348,7 @@ int afs_put_operation(struct afs_operation *op) for (i = 0; i < op->nr_files - 2; i++) if (op->more_files[i].put_vnode) iput(&op->more_files[i].vnode->netfs.inode); - kfree(op->more_files); + kvfree(op->more_files); } if (op->estate) { diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 3f48458694baa..14f39a9bea6cf 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -52,9 +52,9 @@ static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *paren /* * Set parameters for the netfs library */ -static void afs_set_netfs_context(struct afs_vnode *vnode) +static void afs_set_netfs_context(struct afs_vnode *vnode, bool is_file) { - netfs_inode_init(&vnode->netfs, &afs_req_ops, true); + netfs_inode_init(&vnode->netfs, &afs_req_ops, is_file); } /* @@ -93,6 +93,10 @@ static int afs_inode_init_from_status(struct afs_operation *op, inode->i_gid = make_kgid(&init_user_ns, status->group); set_nlink(&vnode->netfs.inode, status->nlink); + i_size_write(inode, status->size); + inode_set_bytes(inode, status->size); + afs_set_netfs_context(vnode, status->type == AFS_FTYPE_FILE); + switch (status->type) { case AFS_FTYPE_FILE: inode->i_mode = S_IFREG | (status->mode & S_IALLUGO); @@ -126,7 +130,6 @@ static int afs_inode_init_from_status(struct afs_operation *op, } inode->i_mapping->a_ops = &afs_symlink_aops; inode_nohighmem(inode); - mapping_set_release_always(inode->i_mapping); break; default: dump_vnode(vnode, op->file[0].vnode != vnode ? op->file[0].vnode : NULL); @@ -134,10 +137,6 @@ static int afs_inode_init_from_status(struct afs_operation *op, return afs_protocol_error(NULL, afs_eproto_file_type); } - i_size_write(inode, status->size); - inode_set_bytes(inode, status->size); - afs_set_netfs_context(vnode); - vnode->invalid_before = status->data_version; trace_afs_set_dv(vnode, status->data_version); inode_set_iversion_raw(&vnode->netfs.inode, status->data_version); @@ -566,7 +565,6 @@ struct inode *afs_root_iget(struct super_block *sb, struct key *key) vnode = AFS_FS_I(inode); vnode->cb_v_check = atomic_read(&as->volume->cb_v_break); - afs_set_netfs_context(vnode); op = afs_alloc_operation(key, as->volume); if (IS_ERR(op)) { @@ -682,6 +680,7 @@ void afs_evict_inode(struct inode *inode) inode->i_mapping->a_ops->writepages(inode->i_mapping, &wbc); } + flush_delayed_work(&vnode->lock_work); netfs_wait_for_outstanding_io(inode); truncate_inode_pages_final(&inode->i_data); netfs_free_folioq_buffer(vnode->directory); diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 0b72a85662999..601f01e5c15fc 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -388,6 +388,7 @@ struct afs_cell { #define AFS_CELL_FL_NO_GC 0 /* The cell was added manually, don't auto-gc */ #define AFS_CELL_FL_DO_LOOKUP 1 /* DNS lookup requested */ #define AFS_CELL_FL_CHECK_ALIAS 2 /* Need to check for aliases */ +#define AFS_CELL_FL_HAVE_INO 3 /* Have dynroot_ino */ enum afs_cell_state state; short error; enum dns_record_source dns_source:8; /* Latest source of data from lookup */ @@ -750,8 +751,6 @@ static inline void afs_vnode_set_cache(struct afs_vnode *vnode, { #ifdef CONFIG_AFS_FSCACHE vnode->netfs.cache = cookie; - if (cookie) - mapping_set_release_always(vnode->netfs.inode.i_mapping); #endif } diff --git a/fs/afs/rxrpc.c b/fs/afs/rxrpc.c index 588f8de51167f..d82916657a3dd 100644 --- a/fs/afs/rxrpc.c +++ b/fs/afs/rxrpc.c @@ -127,8 +127,15 @@ void afs_close_socket(struct afs_net *net) { _enter(""); + cancel_work_sync(&net->charge_preallocation_work); + cancel_work_sync(&net->rx_oob_work); + /* Future work items should now see ->live is false. */ + kernel_listen(net->socket, 0); + + /* Make sure work items are no longer running. */ flush_workqueue(afs_async_calls); + cancel_work_sync(&net->charge_preallocation_work); if (net->spare_incoming_call) { afs_put_call(net->spare_incoming_call); @@ -142,6 +149,7 @@ void afs_close_socket(struct afs_net *net) kernel_sock_shutdown(net->socket, SHUT_RDWR); flush_workqueue(afs_async_calls); + cancel_work_sync(&net->rx_oob_work); net->socket->sk->sk_user_data = NULL; sock_release(net->socket); key_put(net->fs_cm_token_key); @@ -742,7 +750,7 @@ void afs_charge_preallocation(struct work_struct *work) container_of(work, struct afs_net, charge_preallocation_work); struct afs_call *call = net->spare_incoming_call; - for (;;) { + while (READ_ONCE(net->live)) { if (!call) { call = afs_alloc_call(net, &afs_RXCMxxxx, GFP_KERNEL); if (!call) @@ -792,7 +800,8 @@ static void afs_rx_new_call(struct sock *sk, struct rxrpc_call *rxcall, if (!call->server) trace_afs_cm_no_server(call, rxrpc_kernel_remote_srx(call->peer)); - queue_work(afs_wq, &net->charge_preallocation_work); + if (net->live) + queue_work(afs_wq, &net->charge_preallocation_work); } /* @@ -982,5 +991,6 @@ static void afs_rx_notify_oob(struct sock *sk, struct sk_buff *oob) { struct afs_net *net = sk->sk_user_data; - schedule_work(&net->rx_oob_work); + if (READ_ONCE(net->live)) + queue_work(afs_wq, &net->rx_oob_work); } diff --git a/fs/afs/super.c b/fs/afs/super.c index 942f3e9800d7d..82bb713825a0d 100644 --- a/fs/afs/super.c +++ b/fs/afs/super.c @@ -587,7 +587,8 @@ static int afs_get_tree(struct fs_context *fc) } fc->root = dget(sb->s_root); - trace_afs_get_tree(as->cell, as->volume); + if (!ctx->dyn_root) + trace_afs_get_tree(as->cell, as->volume); _leave(" = 0 [%p]", sb); return 0; @@ -659,7 +660,6 @@ static void afs_i_init_once(void *_vnode) INIT_LIST_HEAD(&vnode->wb_keys); INIT_LIST_HEAD(&vnode->pending_locks); INIT_LIST_HEAD(&vnode->granted_locks); - INIT_DELAYED_WORK(&vnode->lock_work, afs_lock_work); INIT_LIST_HEAD(&vnode->cb_mmap_link); seqlock_init(&vnode->cb_lock); } @@ -693,6 +693,7 @@ static struct inode *afs_alloc_inode(struct super_block *sb) init_rwsem(&vnode->rmdir_lock); INIT_WORK(&vnode->cb_work, afs_invalidate_mmap_work); + INIT_DELAYED_WORK(&vnode->lock_work, afs_lock_work); _leave(" = %p", &vnode->netfs.inode); return &vnode->netfs.inode; diff --git a/fs/afs/vl_list.c b/fs/afs/vl_list.c index 3e4966915ea4c..0dc1b005f554f 100644 --- a/fs/afs/vl_list.c +++ b/fs/afs/vl_list.c @@ -92,7 +92,7 @@ static struct afs_addr_list *afs_extract_vl_addrs(struct afs_net *net, { struct afs_addr_list *alist; const u8 *b = *_b; - int ret = -EINVAL; + int ret; alist = afs_alloc_addrlist(nr_addrs); if (!alist) @@ -110,6 +110,7 @@ static struct afs_addr_list *afs_extract_vl_addrs(struct afs_net *net, case DNS_ADDRESS_IS_IPV4: if (end - b < 4) { _leave(" = -EINVAL [short inet]"); + ret = -EINVAL; goto error; } memcpy(x, b, 4); @@ -122,6 +123,7 @@ static struct afs_addr_list *afs_extract_vl_addrs(struct afs_net *net, case DNS_ADDRESS_IS_IPV6: if (end - b < 16) { _leave(" = -EINVAL [short inet6]"); + ret = -EINVAL; goto error; } memcpy(x, b, 16); @@ -198,6 +200,8 @@ struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *cell, b += sizeof(*hdr); while (end - b >= sizeof(bs)) { + int nlen; + bs.name_len = afs_extract_le16(&b); bs.priority = afs_extract_le16(&b); bs.weight = afs_extract_le16(&b); @@ -207,10 +211,12 @@ struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *cell, bs.protocol = *b++; bs.nr_addrs = *b++; + nlen = min3(bs.name_len, end - b, 255); + _debug("extract %u %u %u %u %u %u %*.*s", bs.name_len, bs.priority, bs.weight, bs.port, bs.protocol, bs.nr_addrs, - bs.name_len, bs.name_len, b); + bs.name_len, nlen, b); if (end - b < bs.name_len) break; diff --git a/fs/afs/volume.c b/fs/afs/volume.c index 9ae5c8ad2e04b..4f79d25ec37f4 100644 --- a/fs/afs/volume.c +++ b/fs/afs/volume.c @@ -40,7 +40,7 @@ static struct afs_volume *afs_insert_volume_into_cell(struct afs_cell *cell, goto found; } - set_bit(AFS_VOLUME_RM_TREE, &volume->flags); + set_bit(AFS_VOLUME_RM_TREE, &p->flags); rb_replace_node_rcu(&p->cell_node, &volume->cell_node, &cell->volumes); } } diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c index e4e51a1d0de28..606319dd69e80 100644 --- a/fs/bpf_fs_kfuncs.c +++ b/fs/bpf_fs_kfuncs.c @@ -100,7 +100,7 @@ static bool match_security_bpf_prefix(const char *name__str) static int bpf_xattr_read_permission(const char *name, struct inode *inode) { - if (WARN_ON(!inode)) + if (!inode) return -EINVAL; /* Allow reading xattr with user. and security.bpf. prefix */ @@ -170,7 +170,7 @@ __bpf_kfunc_end_defs(); static int bpf_xattr_write_permission(const char *name, struct inode *inode) { - if (WARN_ON(!inode)) + if (!inode) return -EINVAL; /* Only allow setting and removing security.bpf. xattrs */ @@ -289,6 +289,9 @@ __bpf_kfunc int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__st struct inode *inode = d_inode(dentry); int ret; + if (!inode) + return -EINVAL; + inode_lock(inode); ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags); inode_unlock(inode); @@ -314,6 +317,9 @@ __bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name_ struct inode *inode = d_inode(dentry); int ret; + if (!inode) + return -EINVAL; + inode_lock(inode); ret = bpf_remove_dentry_xattr_locked(dentry, name__str); inode_unlock(inode); diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 55c272fe5d92a..5d8fda94fe221 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -569,6 +569,8 @@ int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, unsigned int extra_bits, struct extent_state **cached_state); +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state); struct btrfs_new_inode_args { /* Input */ diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index a02b62e0a8f33..2ceb5661e0718 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -1191,22 +1191,6 @@ void __cold btrfs_exit_compress(void) bioset_exit(&btrfs_compressed_bioset); } -/* - * The bvec is a single page bvec from a bio that contains folios from a filemap. - * - * Since the folio may be a large one, and if the bv_page is not a head page of - * a large folio, then page->index is unreliable. - * - * Thus we need this helper to grab the proper file offset. - */ -static u64 file_offset_from_bvec(const struct bio_vec *bvec) -{ - const struct page *page = bvec->bv_page; - const struct folio *folio = page_folio(page); - - return (page_pgoff(folio, page) << PAGE_SHIFT) + bvec->bv_offset; -} - /* * Copy decompressed data from working buffer to pages. * @@ -1259,7 +1243,7 @@ int btrfs_decompress_buf2page(const char *buf, u32 buf_len, * cb->start may underflow, but subtracting that value can still * give us correct offset inside the full decompressed extent. */ - bvec_offset = file_offset_from_bvec(&bvec) - cb->start; + bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start; /* Haven't reached the bvec range, exit */ if (decompressed + buf_len <= bvec_offset) diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c index 0970799d0aa44..4293a63834337 100644 --- a/fs/btrfs/delalloc-space.c +++ b/fs/btrfs/delalloc-space.c @@ -134,6 +134,8 @@ int btrfs_alloc_data_chunk_ondemand(const struct btrfs_inode *inode, u64 bytes) if (btrfs_is_free_space_inode(inode)) flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE; + else if (btrfs_is_zoned(fs_info) && btrfs_is_data_reloc_root(root)) + flush = BTRFS_RESERVE_FLUSH_ZONED_RELOCATION; return btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), bytes, flush); } diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 57167d56dc727..412309825d6fa 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -15,10 +15,12 @@ struct btrfs_dio_data { ssize_t submitted; + loff_t old_isize; struct extent_changeset *data_reserved; struct btrfs_ordered_extent *ordered; bool data_space_reserved; bool nocow_done; + bool updated_isize; }; struct btrfs_dio_private { @@ -228,6 +230,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, bool space_reserved = false; u64 len = *lenp; u64 prev_len; + loff_t old_isize; int ret = 0; /* @@ -341,8 +344,14 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, * Need to update the i_size under the extent lock so buffered * readers will get the updated i_size when we unlock. */ - if (start + len > i_size_read(inode)) + old_isize = i_size_read(inode); + if (start + len > old_isize) { + if (!dio_data->updated_isize) { + dio_data->old_isize = old_isize; + dio_data->updated_isize = true; + } i_size_write(inode, start + len); + } out: if (ret && space_reserved) { btrfs_delalloc_release_extents(BTRFS_I(inode), len); @@ -624,12 +633,55 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, if (submitted < length) { pos += submitted; length -= submitted; - if (write) + if (write) { + /* + * Got a short write and have updated the isize, need to + * revert the isize change. + * + * Normally we need to update isize with extent lock hold, + * but we're safe due to the following factors: + * + * - Only a single writer can be enlarging isize + * Enlarging isize will take the exclusive inode lock. + * + * - Buffered readers need to wait for the OE we're holding + * Buffered readers will lock extent and wait for OE + * of the folio range, and since page cache is invalidated + * the OE wait can not be skipped. + * + * So here we are safe to revert the isize before + * finishing the OE, and no reader of the remaining range + * can see the enlarged size. + * + * TODO: Extend the DIO_LOCKED lifespan for direct writes, + * and only enlarge isize after a successful write. + */ + if (dio_data->updated_isize) { + u64 new_isize; + + if (submitted == 0) + new_isize = dio_data->old_isize; + else + new_isize = max(dio_data->old_isize, pos); + i_size_write(inode, new_isize); + dio_data->updated_isize = false; + } + /* + * We have a short write, if there is any range + * that is submitted properly, that part will have + * its own OE split from the original one. + * + * So for the OE at dio_data->ordered, it's the part + * that is not submitted, and should be marked + * as fully truncated. + */ + btrfs_mark_ordered_extent_truncated(dio_data->ordered, 0); btrfs_finish_ordered_extent(dio_data->ordered, - pos, length, false); - else + pos, length, true); + } else { btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1, NULL); + } ret = -ENOTBLK; } if (write) { diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index c0a30bb213d7a..ab92b35fa3cc1 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1928,7 +1928,7 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) { u32 max_active = fs_info->thread_pool_size; unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND; - unsigned int ordered_flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_PERCPU; + unsigned int ordered_flags = WQ_MEM_RECLAIM | WQ_FREEZABLE; fs_info->workers = btrfs_alloc_workqueue(fs_info, "worker", flags, max_active, 16); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 391fad41c3b6d..70d1f7ead160f 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2108,7 +2108,8 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, locked_ref = btrfs_select_ref_head(fs_info, delayed_refs); if (IS_ERR_OR_NULL(locked_ref)) { if (PTR_ERR(locked_ref) == -EAGAIN) { - continue; + count++; + goto again; } else { break; } @@ -2156,7 +2157,7 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, * Either success case or btrfs_run_delayed_refs_for_head * returned -EAGAIN, meaning we need to select another head */ - +again: locked_ref = NULL; cond_resched(); } while ((min_bytes != U64_MAX && bytes_processed < min_bytes) || @@ -6613,12 +6614,16 @@ static int btrfs_trim_free_extents_throttle(struct btrfs_device *device, *trimmed = 0; - /* Discard not supported = nothing to do. */ - if (!bdev_max_discard_sectors(device->bdev)) + /* + * The caller only filters out MISSING devices, but a device that was + * missing at mount and later rescanned has MISSING cleared while bdev + * is still NULL and WRITEABLE is still unset. Skip those here. + */ + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) || !device->bdev) return 0; - /* Not writable = nothing to do. */ - if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) + /* Discard not supported = nothing to do. */ + if (!bdev_max_discard_sectors(device->bdev)) return 0; /* No free space = nothing to do. */ diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 8c171ed07008b..8636ebe1fcabd 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -93,16 +93,8 @@ int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos end_of_last_block = start_pos + num_bytes - 1; - /* - * The pages may have already been dirty, clear out old accounting so - * we can set things up properly - */ - btrfs_clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - cached); - - ret = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block, - extra_bits, cached); + ret = btrfs_reset_extent_delalloc(inode, start_pos, end_of_last_block, + extra_bits, cached); if (ret) return ret; @@ -1966,18 +1958,7 @@ static vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) } } - /* - * page_mkwrite gets called when the page is firstly dirtied after it's - * faulted in, but write(2) could also dirty a page and set delalloc - * bits, thus in this case for space account reason, we still need to - * clear any delalloc bits within this page range since we have to - * reserve data&meta space before lock_page() (see above comments). - */ - btrfs_clear_extent_bit(io_tree, page_start, end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | - EXTENT_DEFRAG, &cached_state); - - ret = btrfs_set_extent_delalloc(inode, page_start, end, 0, &cached_state); + ret = btrfs_reset_extent_delalloc(inode, page_start, end, 0, &cached_state); if (ret < 0) { btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state); goto out_unlock; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 1ca1cbdf25bcd..dc5148f176e77 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2317,7 +2317,7 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode, static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end) { if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) { - if (inode->defrag_bytes && + if (data_race(inode->defrag_bytes) && btrfs_test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG)) return false; return true; @@ -2810,7 +2810,13 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, unsigned int extra_bits, struct extent_state **cached_state) { - WARN_ON(PAGE_ALIGNED(end)); + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); if (start >= i_size_read(&inode->vfs_inode) && !(inode->flags & BTRFS_INODE_PREALLOC)) { @@ -3035,6 +3041,52 @@ int btrfs_writepage_cow_fixup(struct folio *folio) return -EAGAIN; } +/* + * Clear the old accounting flags and set EXTENT_DELALLOC for the range. + * + * Return <0 for error, in that case no range has EXTENT_DELALLOC bit cleared or set. + */ +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state) +{ + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* The @extra_bits can only be EXTENT_NORESERVE for now. */ + ASSERT(!(extra_bits & ~EXTENT_NORESERVE), "extra_bits=0x%x", extra_bits); + + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); + + /* + * Check and set DELALLOC_NEW flag, this needs to search tree thus can + * fail early. Thus we want to do this before clearing EXTENT_DELALLOC. + */ + if (start >= i_size_read(&inode->vfs_inode) && + !(inode->flags & BTRFS_INODE_PREALLOC)) { + /* + * There can't be any extents following EOF in this case so just + * set the delalloc new bit for the range directly. + */ + extra_bits |= EXTENT_DELALLOC_NEW; + } else { + int ret; + + ret = btrfs_find_new_delalloc_bytes(inode, start, end + 1 - start, + NULL); + if (unlikely(ret)) + return ret; + } + /* Clear the old accounting as the range may already be dirty. */ + btrfs_clear_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | + EXTENT_DEFRAG, cached_state); + return btrfs_set_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | extra_bits, cached_state); +} + static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, u64 file_pos, struct btrfs_file_extent_item *stack_fi, @@ -5181,12 +5233,7 @@ int btrfs_truncate_block(struct btrfs_inode *inode, u64 offset, u64 start, u64 e goto again; } - btrfs_clear_extent_bit(&inode->io_tree, block_start, block_end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - &cached_state); - - ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, - &cached_state); + ret = btrfs_reset_extent_delalloc(inode, block_start, block_end, 0, &cached_state); if (ret) { btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); goto out_unlock; @@ -7776,11 +7823,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, &cached_state); - spin_lock(&inode->ordered_tree_lock); - set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); - ordered->truncated_len = min(ordered->truncated_len, - cur - ordered->file_offset); - spin_unlock(&inode->ordered_tree_lock); + btrfs_mark_ordered_extent_truncated(ordered, cur - ordered->file_offset); /* * If the ordered extent has finished, we're safe to delete all diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c index 2de18c7b563af..6e4aa22853aba 100644 --- a/fs/btrfs/lzo.c +++ b/fs/btrfs/lzo.c @@ -491,6 +491,17 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb) return -EIO; } + /* The segment must not extend beyond the compressed input. */ + if (unlikely(cur_in + seg_len > compressed_len)) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(fs_info, + "lzo segment overflows compressed input, root %llu inode %llu offset %llu cur_in %u len %u compressed len %u", + btrfs_root_id(inode->root), btrfs_ino(inode), + cb->start, cur_in, seg_len, compressed_len); + return -EUCLEAN; + } + /* Copy the compressed segment payload into workspace */ copy_compressed_segment(cb, &fi, &cur_folio_index, workspace->cbuf, seg_len, &cur_in); diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index e5a24b3ff95e4..896de1c3c1474 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -357,6 +357,18 @@ void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered) mapping_set_error(ordered->inode->vfs_inode.i_mapping, -EIO); } +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len) +{ + struct btrfs_inode *inode = ordered->inode; + + ASSERT(truncate_len <= ordered->num_bytes); + spin_lock(&inode->ordered_tree_lock); + set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); + ordered->truncated_len = min(ordered->truncated_len, truncate_len); + spin_unlock(&inode->ordered_tree_lock); +} + static void finish_ordered_fn(struct btrfs_work *work) { struct btrfs_ordered_extent *ordered_extent; diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 03e12380a2fda..8d5d5ba1e02f0 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -226,6 +226,8 @@ bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end, struct btrfs_ordered_extent *btrfs_split_ordered_extent( struct btrfs_ordered_extent *ordered, u64 len); void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered); +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len); int __init ordered_data_init(void); void __cold ordered_data_exit(void); diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 49865a4637809..951824b033b78 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -94,9 +94,7 @@ static int copy_inline_to_page(struct btrfs_inode *inode, if (ret < 0) goto out_unlock; - btrfs_clear_extent_bit(&inode->io_tree, file_offset, range_end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, NULL); - ret = btrfs_set_extent_delalloc(inode, file_offset, range_end, 0, NULL); + ret = btrfs_reset_extent_delalloc(inode, file_offset, range_end, 0, NULL); if (ret) goto out_unlock; @@ -181,10 +179,12 @@ static int clone_copy_inline_extent(struct btrfs_inode *inode, struct btrfs_drop_extents_args drop_args = { 0 }; int ret; struct btrfs_key key; + bool copied_inline_to_page = false; if (new_key->offset > 0) { ret = copy_inline_to_page(inode, new_key->offset, inline_data, size, datal, comp_type); + copied_inline_to_page = (ret == 0); goto out; } @@ -290,6 +290,60 @@ static int clone_copy_inline_extent(struct btrfs_inode *inode, btrfs_abort_transaction(trans, ret); out: if (!ret && !trans) { + if (copied_inline_to_page && + new_key->offset + datal > i_size_read(&inode->vfs_inode)) { + /* + * If we copied the inline extent data to a page/folio + * beyond the i_size of the destination inode, then we + * need to increase the i_size before we start a + * transaction to update the inode item. This is to + * prevent a deadlock when the flushoncommit mount + * option is used, which happens like this: + * + * 1) Task A clones an inline extent from inode X to an + * offset of inode Y that is beyond Y's current + * i_size. This means we copied the inline extent's + * data to a folio of inode Y that is beyond its EOF, + * using the call above to copy_inline_to_page(); + * + * 2) Task B starts a transaction commit and calls + * btrfs_start_delalloc_flush() to flush delalloc; + * + * 3) The delalloc flushing sees the new dirty folio of + * inode Y and when it attempts to flush it, it ends + * up at extent_writepage() and sees that the offset + * of the folio is beyond the i_size of inode Y, so + * it attempts to invalidate the folio by calling + * folio_invalidate(), which ends up at btrfs' folio + * invalidate callback - btrfs_invalidate_folio(). + * There it tries to lock the folio's range in inode + * Y's extent io tree, but it blocks since it's + * currently locked by task A - during reflink we + * lock the inodes and the source and destination + * ranges after flushing all delalloc and waiting for + * ordered extent completion - after that we don't + * expect to have dirty folios in the ranges, the + * exception is if we have to copy an inline extent's + * data (because the destination offset is not zero); + * + * 4) Task A then does the 'goto out' below and attempts + * to start a transaction to update the inode item, + * and then it's blocked since the current + * transaction is in the TRANS_STATE_COMMIT_START + * state. Therefore task A has to wait for the + * current transaction to become unblocked (its + * state >= TRANS_STATE_UNBLOCKED). + * + * This leads to a deadlock - the task committing the + * transaction waiting for the delalloc flushing which + * is blocked during folio invalidation on the inode's + * extent lock and the reflink task waiting for the + * current transaction to be unblocked so that it can + * start a new one to update the inode item (while + * holding the extent lock). + */ + i_size_write(&inode->vfs_inode, new_key->offset + datal); + } /* * No transaction here means we copied the inline extent into a * page of the destination inode. @@ -322,50 +376,7 @@ static int clone_copy_inline_extent(struct btrfs_inode *inode, ret = copy_inline_to_page(inode, new_key->offset, inline_data, size, datal, comp_type); - - /* - * If we copied the inline extent data to a page/folio beyond the i_size - * of the destination inode, then we need to increase the i_size before - * we start a transaction to update the inode item. This is to prevent a - * deadlock when the flushoncommit mount option is used, which happens - * like this: - * - * 1) Task A clones an inline extent from inode X to an offset of inode - * Y that is beyond Y's current i_size. This means we copied the - * inline extent's data to a folio of inode Y that is beyond its EOF, - * using the call above to copy_inline_to_page(); - * - * 2) Task B starts a transaction commit and calls - * btrfs_start_delalloc_flush() to flush delalloc; - * - * 3) The delalloc flushing sees the new dirty folio of inode Y and when - * it attempts to flush it, it ends up at extent_writepage() and sees - * that the offset of the folio is beyond the i_size of inode Y, so - * it attempts to invalidate the folio by calling folio_invalidate(), - * which ends up at btrfs' folio invalidate callback - - * btrfs_invalidate_folio(). There it tries to lock the folio's range - * in inode Y's extent io tree, but it blocks since it's currently - * locked by task A - during reflink we lock the inodes and the - * source and destination ranges after flushing all delalloc and - * waiting for ordered extent completion - after that we don't expect - * to have dirty folios in the ranges, the exception is if we have to - * copy an inline extent's data (because the destination offset is - * not zero); - * - * 4) Task A then does the 'goto out' below and attempts to start a - * transaction to update the inode item, and then it's blocked since - * the current transaction is in the TRANS_STATE_COMMIT_START state. - * Therefore task A has to wait for the current transaction to become - * unblocked (its state >= TRANS_STATE_UNBLOCKED). - * - * This leads to a deadlock - the task committing the transaction - * waiting for the delalloc flushing which is blocked during folio - * invalidation on the inode's extent lock and the reflink task waiting - * for the current transaction to be unblocked so that it can start a - * a new one to update the inode item (while holding the extent lock). - */ - if (ret == 0 && new_key->offset + datal > i_size_read(&inode->vfs_inode)) - i_size_write(&inode->vfs_inode, new_key->offset + datal); + copied_inline_to_page = (ret == 0); goto out; } diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index f0436eea15445..e7a5cf50caa4b 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1698,6 +1698,7 @@ static int handle_reserve_ticket(struct btrfs_space_info *space_info, ARRAY_SIZE(evict_flush_states)); break; case BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE: + case BTRFS_RESERVE_FLUSH_ZONED_RELOCATION: priority_reclaim_data_space(space_info, ticket); break; default: @@ -1961,6 +1962,7 @@ int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes, ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA || flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE || + flush == BTRFS_RESERVE_FLUSH_ZONED_RELOCATION || flush == BTRFS_RESERVE_NO_FLUSH, "flush=%d", flush); ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA, "current->journal_info=0x%lx flush=%d", diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 24f45072ca4b1..aa836e8a9d4a6 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -77,6 +77,17 @@ enum btrfs_reserve_flush_enum { */ BTRFS_RESERVE_FLUSH_ALL_STEAL, + /* + * This is for relocation on zoned filesystems only. We need to use + * priority flushing for this, because otherwise we can deadlock on + * waiting for a ticket, that cannot be granted, because we cannot do + * any allocations. + * + * Apart from being specific to zoned relocation, it is equal to + * BTRFS_FLUSH_FREE_SPACE_INODE. + */ + BTRFS_RESERVE_FLUSH_ZONED_RELOCATION, + /* * This is for btrfs_use_block_rsv only. We have exhausted our block * rsv and our global block rsv. This can happen for things like diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index b26aa9169e838..ba70d727622e8 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -1740,7 +1740,8 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) int mixed = 0; list_for_each_entry(found, &fs_info->space_info, list) { - if (found->flags & BTRFS_BLOCK_GROUP_DATA) { + if (found->flags & BTRFS_BLOCK_GROUP_DATA && + found->subgroup_id != BTRFS_SUB_GROUP_DATA_RELOC) { int i; total_free_data += found->disk_total - found->disk_used; @@ -1873,6 +1874,7 @@ static int btrfs_get_tree_super(struct fs_context *fc) fs_info->fs_devices = fs_devices; mutex_unlock(&uuid_mutex); + fc->sb_flags |= SB_NOSEC; sb = sget_fc(fc, btrfs_fc_test_super, set_anon_super_fc); if (IS_ERR(sb)) { diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 16dd87aa06f20..0d590e81f3259 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -354,12 +354,33 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) return ret; } +static int btrfs_get_max_active_zones(struct btrfs_device *device, + struct btrfs_zoned_device_info *zone_info) +{ + struct block_device *bdev = device->bdev; + int max_active_zones; + + if (unlikely(zone_info->nr_zones < BTRFS_MIN_ACTIVE_ZONES)) { + btrfs_err(device->fs_info, "zoned: not enough zones to mount filesystem: %u < %d", + zone_info->nr_zones, BTRFS_MIN_ACTIVE_ZONES); + return -EINVAL; + } + + max_active_zones = min_not_zero(bdev_max_active_zones(bdev), + bdev_max_open_zones(bdev)); + if (max_active_zones == 0) + max_active_zones = min(zone_info->nr_zones / 4, + BTRFS_DEFAULT_MAX_ACTIVE_ZONES); + + zone_info->max_active_zones = max(max_active_zones, BTRFS_MIN_ACTIVE_ZONES); + return 0; +} + int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) { struct btrfs_fs_info *fs_info = device->fs_info; struct btrfs_zoned_device_info *zone_info = NULL; struct block_device *bdev = device->bdev; - unsigned int max_active_zones; unsigned int nactive; sector_t nr_sectors; sector_t sector = 0; @@ -424,19 +445,9 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) if (!IS_ALIGNED(nr_sectors, zone_sectors)) zone_info->nr_zones++; - max_active_zones = min_not_zero(bdev_max_active_zones(bdev), - bdev_max_open_zones(bdev)); - if (!max_active_zones && zone_info->nr_zones > BTRFS_DEFAULT_MAX_ACTIVE_ZONES) - max_active_zones = BTRFS_DEFAULT_MAX_ACTIVE_ZONES; - if (max_active_zones && max_active_zones < BTRFS_MIN_ACTIVE_ZONES) { - btrfs_err(fs_info, -"zoned: %s: max active zones %u is too small, need at least %u active zones", - rcu_dereference(device->name), max_active_zones, - BTRFS_MIN_ACTIVE_ZONES); - ret = -EINVAL; + ret = btrfs_get_max_active_zones(device, zone_info); + if (ret) goto out; - } - zone_info->max_active_zones = max_active_zones; zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); if (!zone_info->seq_zones) { @@ -517,26 +528,29 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) goto out; } - if (max_active_zones) { - if (unlikely(nactive > max_active_zones)) { - if (bdev_max_active_zones(bdev) == 0) { - max_active_zones = 0; - zone_info->max_active_zones = 0; - goto validate; - } + if (unlikely(nactive > zone_info->max_active_zones)) { + if (bdev_max_active_zones(bdev) > 0) { btrfs_err(device->fs_info, - "zoned: %u active zones on %s exceeds max_active_zones %u", - nactive, rcu_dereference(device->name), - max_active_zones); + "zoned: %u active zones on %s exceeds max_active_zones %u", + nactive, rcu_dereference(device->name), + zone_info->max_active_zones); ret = -EIO; goto out; } + + /* + * This is for backward compatibility with old filesystems that + * have a lot of active zones because the device doesn't report + * a maximum number of zones and we previously didn't care for + * the limit. + */ + zone_info->max_active_zones = 0; + } else { atomic_set(&zone_info->active_zones_left, - max_active_zones - nactive); + zone_info->max_active_zones - nactive); set_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags); } -validate: /* Validate superblock log */ nr_zones = BTRFS_NR_SB_LOG_ZONES; for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index 2937db690b40b..90200410dcfd4 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -209,7 +209,6 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, return ERR_PTR(ret); nomem_d_alloc: - inode_unlock(d_inode(dir)); _leave(" = -ENOMEM"); return ERR_PTR(-ENOMEM); } @@ -375,7 +374,7 @@ int cachefiles_bury_object(struct cachefiles_cache *cache, "Rename failed with error %d", ret); } - __cachefiles_unmark_inode_in_use(object, d_inode(rep)); + cachefiles_do_unmark_inode_in_use(object, d_inode(rep)); end_renaming(&rd); _leave(" = 0"); return 0; diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c index 362b6ff9b908f..eb991b2a9c341 100644 --- a/fs/configfs/dir.c +++ b/fs/configfs/dir.c @@ -235,15 +235,16 @@ static int configfs_dirent_exists(struct dentry *dentry) const unsigned char *new = dentry->d_name.name; struct configfs_dirent *sd; + spin_lock(&configfs_dirent_lock); list_for_each_entry(sd, &parent_sd->s_children, s_sibling) { if (sd->s_element) { - const unsigned char *existing = configfs_get_name(sd); - if (strcmp(existing, new)) - continue; - else + if (strcmp(configfs_get_name(sd), new) == 0) { + spin_unlock(&configfs_dirent_lock); return -EEXIST; + } } } + spin_unlock(&configfs_dirent_lock); return 0; } @@ -486,6 +487,9 @@ static struct dentry * configfs_lookup(struct inode *dir, inode = configfs_create(dentry, mode); if (IS_ERR(inode)) { + spin_lock(&configfs_dirent_lock); + sd->s_dentry = NULL; + spin_unlock(&configfs_dirent_lock); configfs_put(sd); return ERR_CAST(inode); } @@ -572,11 +576,28 @@ static void configfs_detach_rollback(struct dentry *dentry) configfs_detach_rollback(sd->s_dentry); } +/* + * Find the next non-cursor. configfs_dirent_lock held by caller. + */ +static struct configfs_dirent *next_dirent(struct configfs_dirent *parent, + struct configfs_dirent *last) +{ + struct configfs_dirent *s; + + s = list_prepare_entry(last, &parent->s_children, s_sibling); + + list_for_each_entry_continue(s, &parent->s_children, s_sibling) { + if (s->s_element) + return s; + } + return NULL; +} + static void detach_attrs(struct config_item * item) { struct dentry * dentry = dget(item->ci_dentry); - struct configfs_dirent * parent_sd; - struct configfs_dirent * sd, * tmp; + struct configfs_dirent *parent_sd; + struct configfs_dirent *sd, *next; if (!dentry) return; @@ -585,15 +606,19 @@ static void detach_attrs(struct config_item * item) dentry->d_name.name); parent_sd = dentry->d_fsdata; - list_for_each_entry_safe(sd, tmp, &parent_sd->s_children, s_sibling) { - if (!sd->s_element || !(sd->s_type & CONFIGFS_NOT_PINNED)) + + spin_lock(&configfs_dirent_lock); + for (sd = next_dirent(parent_sd, NULL); sd; sd = next) { + next = next_dirent(parent_sd, sd); + if (!(sd->s_type & CONFIGFS_NOT_PINNED)) continue; - spin_lock(&configfs_dirent_lock); list_del_init(&sd->s_sibling); spin_unlock(&configfs_dirent_lock); configfs_drop_dentry(sd, dentry); configfs_put(sd); + spin_lock(&configfs_dirent_lock); } + spin_unlock(&configfs_dirent_lock); /** * Drop reference from dget() on entrance. @@ -652,18 +677,20 @@ static void detach_groups(struct config_group *group) struct dentry * dentry = dget(group->cg_item.ci_dentry); struct dentry *child; struct configfs_dirent *parent_sd; - struct configfs_dirent *sd, *tmp; + struct configfs_dirent *sd, *next; if (!dentry) return; parent_sd = dentry->d_fsdata; - list_for_each_entry_safe(sd, tmp, &parent_sd->s_children, s_sibling) { - if (!sd->s_element || - !(sd->s_type & CONFIGFS_USET_DEFAULT)) + spin_lock(&configfs_dirent_lock); + for (sd = next_dirent(parent_sd, NULL); sd; sd = next) { + next = next_dirent(parent_sd, sd); + if (!(sd->s_type & CONFIGFS_USET_DEFAULT)) continue; child = sd->s_dentry; + spin_unlock(&configfs_dirent_lock); inode_lock(d_inode(child)); @@ -675,7 +702,9 @@ static void detach_groups(struct config_group *group) d_delete(child); dput(child); + spin_lock(&configfs_dirent_lock); } + spin_unlock(&configfs_dirent_lock); /** * Drop reference from dget() on entrance. @@ -1127,6 +1156,7 @@ configfs_find_subsys_dentry(struct configfs_dirent *root_sd, struct configfs_dirent *p; struct configfs_dirent *ret = NULL; + spin_lock(&configfs_dirent_lock); list_for_each_entry(p, &root_sd->s_children, s_sibling) { if (p->s_type & CONFIGFS_DIR && p->s_element == subsys_item) { @@ -1134,6 +1164,7 @@ configfs_find_subsys_dentry(struct configfs_dirent *root_sd, break; } } + spin_unlock(&configfs_dirent_lock); return ret; } diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index 8d3c278a7591e..0053b5c454123 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -236,7 +236,7 @@ struct fscrypt_symlink_data { * @tfm: crypto API transform object * @blk_key: key for blk-crypto * - * Normally only one of the fields will be non-NULL. + * Only one of the fields is non-NULL. */ struct fscrypt_prepared_key { struct crypto_sync_skcipher *tfm; @@ -245,6 +245,15 @@ struct fscrypt_prepared_key { #endif }; +/* An entry in the linked list ->mk_mode_keys */ +struct fscrypt_mode_key { + struct fscrypt_prepared_key key; + struct list_head link; + u8 hkdf_context; + u8 mode_num; + u8 data_unit_bits; +}; + /* * fscrypt_inode_info - the "encryption key" for an inode * @@ -430,20 +439,12 @@ int fscrypt_derive_sw_secret(struct super_block *sb, * @prep_key, depending on which encryption implementation the file will use. */ static inline bool -fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, +fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key, const struct fscrypt_inode_info *ci) { - /* - * The two smp_load_acquire()'s here pair with the smp_store_release()'s - * in fscrypt_prepare_inline_crypt_key() and fscrypt_prepare_key(). - * I.e., in some cases (namely, if this prep_key is a per-mode - * encryption key) another task can publish blk_key or tfm concurrently, - * executing a RELEASE barrier. We need to use smp_load_acquire() here - * to safely ACQUIRE the memory the other task published. - */ if (fscrypt_using_inline_encryption(ci)) - return smp_load_acquire(&prep_key->blk_key) != NULL; - return smp_load_acquire(&prep_key->tfm) != NULL; + return prep_key->blk_key != NULL; + return prep_key->tfm != NULL; } #else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ @@ -486,15 +487,28 @@ fscrypt_derive_sw_secret(struct super_block *sb, } static inline bool -fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, +fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key, const struct fscrypt_inode_info *ci) { - return smp_load_acquire(&prep_key->tfm) != NULL; + return prep_key->tfm != NULL; } #endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ /* keyring.c */ +/* + * fscrypt_master_key_user - a user's claim to a master key + */ +struct fscrypt_master_key_user { + struct list_head link; + kuid_t uid; + /* + * This 'struct key' contains no secret. It exists solely to charge the + * appropriate user's key quota. + */ + struct key *quota_key; +}; + /* * fscrypt_master_key_secret - secret key material of an in-use master key */ @@ -577,8 +591,8 @@ struct fscrypt_master_key { /* * Active and structural reference counts. An active ref guarantees * that the struct continues to exist, continues to be in the keyring - * ->s_master_keys, and that any embedded subkeys (e.g. - * ->mk_direct_keys) that have been prepared continue to exist. + * ->s_master_keys, and that any non-file-scoped subkeys (e.g. + * ->mk_mode_keys) that have been prepared continue to exist. * A structural ref only guarantees that the struct continues to exist. * * There is one active ref associated with ->mk_present being true, and @@ -610,19 +624,18 @@ struct fscrypt_master_key { struct fscrypt_key_specifier mk_spec; /* - * Keyring which contains a key of type 'key_type_fscrypt_user' for each - * user who has added this key. Normally each key will be added by just - * one user, but it's possible that multiple users share a key, and in - * that case we need to keep track of those users so that one user can't - * remove the key before the others want it removed too. + * List of user claims to this key (struct fscrypt_master_key_user). + * Normally each key will be added by just one user, but it's possible + * that multiple users share a key, and in that case we need to keep + * track of those users so that one user can't remove the key before the + * others want it removed too. * - * This is NULL for v1 policy keys; those can only be added by root. + * Used only for v2 policy keys. v1 policy keys can be added only by + * root, so user tracking doesn't apply to them. * - * Locking: protected by ->mk_sem. (We don't just rely on the keyrings - * subsystem semaphore ->mk_users->sem, as we need support for atomic - * search+insert along with proper synchronization with other fields.) + * Locking: protected by ->mk_sem. */ - struct key *mk_users; + struct list_head mk_users; /* * List of inodes that were unlocked using this key. This allows the @@ -632,12 +645,21 @@ struct fscrypt_master_key { spinlock_t mk_decrypted_inodes_lock; /* - * Per-mode encryption keys for the various types of encryption policies - * that use them. Allocated and derived on-demand. + * A list of 'struct fscrypt_mode_key' for the (hkdf_context, mode_num, + * data_unit_bits, inlinecrypt) combinations that are in use for this + * master key, for hkdf_context in [HKDF_CONTEXT_DIRECT_KEY, + * HKDF_CONTEXT_IV_INO_LBLK_32_KEY, HKDF_CONTEXT_IV_INO_LBLK_64_KEY]. + * + * This is a linked list and not a hash table because in practice + * there's just a single encryption policy per master key, using + * _at most_ 2 nodes in this list. Per-file keys don't use this at all. + * + * This list is append-only until the master key is fully removed, at + * which time the list is cleared. Before then, + * fscrypt_mode_key_setup_mutex synchronizes appends, and searches use + * the RCU read lock together with ->mk_sem held for read. */ - struct fscrypt_prepared_key mk_direct_keys[FSCRYPT_MODE_MAX + 1]; - struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[FSCRYPT_MODE_MAX + 1]; - struct fscrypt_prepared_key mk_iv_ino_lblk_32_keys[FSCRYPT_MODE_MAX + 1]; + struct list_head mk_mode_keys; /* Hash key for inode numbers. Initialized only when needed. */ siphash_key_t mk_ino_hash_key; diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index 37d42d357925e..47324062fee51 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -198,13 +198,7 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, goto fail; } - /* - * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared(). - * I.e., here we publish ->blk_key with a RELEASE barrier so that - * concurrent tasks can ACQUIRE it. Note that this concurrency is only - * possible for per-mode keys, not for per-file keys. - */ - smp_store_release(&prep_key->blk_key, blk_key); + prep_key->blk_key = blk_key; return 0; fail: diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index be8e6e8011f21..38b73e7030733 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -65,36 +65,33 @@ static void fscrypt_free_master_key(struct rcu_head *head) kfree_sensitive(mk); } +static void clear_mk_users(struct fscrypt_master_key *mk); + void fscrypt_put_master_key(struct fscrypt_master_key *mk) { if (!refcount_dec_and_test(&mk->mk_struct_refs)) return; /* - * No structural references left, so free ->mk_users, and also free the + * No structural references left, so clear ->mk_users, and also free the * fscrypt_master_key struct itself after an RCU grace period ensures * that concurrent keyring lookups can no longer find it. */ WARN_ON_ONCE(refcount_read(&mk->mk_active_refs) != 0); - if (mk->mk_users) { - /* Clear the keyring so the quota gets released right away. */ - keyring_clear(mk->mk_users); - key_put(mk->mk_users); - mk->mk_users = NULL; - } + clear_mk_users(mk); call_rcu(&mk->mk_rcu_head, fscrypt_free_master_key); } void fscrypt_put_master_key_activeref(struct super_block *sb, struct fscrypt_master_key *mk) { - size_t i; + struct fscrypt_mode_key *node, *tmp; if (!refcount_dec_and_test(&mk->mk_active_refs)) return; /* * No active references left, so complete the full removal of this * fscrypt_master_key struct by removing it from the keyring and - * destroying any subkeys embedded in it. + * destroying any non-file-scoped subkeys. */ if (WARN_ON_ONCE(!sb->s_master_keys)) @@ -110,13 +107,16 @@ void fscrypt_put_master_key_activeref(struct super_block *sb, WARN_ON_ONCE(mk->mk_present); WARN_ON_ONCE(!list_empty(&mk->mk_decrypted_inodes)); - for (i = 0; i <= FSCRYPT_MODE_MAX; i++) { - fscrypt_destroy_prepared_key( - sb, &mk->mk_direct_keys[i]); - fscrypt_destroy_prepared_key( - sb, &mk->mk_iv_ino_lblk_64_keys[i]); - fscrypt_destroy_prepared_key( - sb, &mk->mk_iv_ino_lblk_32_keys[i]); + /* + * Destroy any non-file-scoped subkeys. Since ->mk_active_refs == 0, + * they're no longer referenced by any inodes. Nor can key setup run + * and use them again. So they're no longer needed. (This implies no + * concurrent readers, so we don't need list_del_rcu() for example.) + */ + list_for_each_entry_safe(node, tmp, &mk->mk_mode_keys, link) { + fscrypt_destroy_prepared_key(sb, &node->key); + list_del(&node->link); + kfree(node); } memzero_explicit(&mk->mk_ino_hash_key, sizeof(mk->mk_ino_hash_key)); @@ -162,8 +162,8 @@ static void fscrypt_user_key_describe(const struct key *key, struct seq_file *m) } /* - * Type of key in ->mk_users. Each key of this type represents a particular - * user who has added a particular master key. + * Type of fscrypt_master_key_user::quota_key. This contains no secret; it + * exists solely to charge a user's key quota. * * Note that the name of this key type really should be something like * ".fscrypt-user" instead of simply ".fscrypt". But the shorter name is chosen @@ -177,30 +177,9 @@ static struct key_type key_type_fscrypt_user = { .describe = fscrypt_user_key_describe, }; -#define FSCRYPT_MK_USERS_DESCRIPTION_SIZE \ - (CONST_STRLEN("fscrypt-") + 2 * FSCRYPT_KEY_IDENTIFIER_SIZE + \ - CONST_STRLEN("-users") + 1) - #define FSCRYPT_MK_USER_DESCRIPTION_SIZE \ (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + CONST_STRLEN(".uid.") + 10 + 1) -static void format_mk_users_keyring_description( - char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE], - const u8 mk_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) -{ - sprintf(description, "fscrypt-%*phN-users", - FSCRYPT_KEY_IDENTIFIER_SIZE, mk_identifier); -} - -static void format_mk_user_description( - char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE], - const u8 mk_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) -{ - - sprintf(description, "%*phN.uid.%u", FSCRYPT_KEY_IDENTIFIER_SIZE, - mk_identifier, __kuid_val(current_fsuid())); -} - /* Create ->s_master_keys if needed. Synchronized by fscrypt_add_key_mutex. */ static int allocate_filesystem_keyring(struct super_block *sb) { @@ -335,91 +314,94 @@ fscrypt_find_master_key(struct super_block *sb, return mk; } -static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk) -{ - char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE]; - struct key *keyring; - - format_mk_users_keyring_description(description, - mk->mk_spec.u.identifier); - keyring = keyring_alloc(description, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, - current_cred(), KEY_POS_SEARCH | - KEY_USR_SEARCH | KEY_USR_READ | KEY_USR_VIEW, - KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); - if (IS_ERR(keyring)) - return PTR_ERR(keyring); - - mk->mk_users = keyring; - return 0; -} - -/* - * Find the current user's "key" in the master key's ->mk_users. - * Returns ERR_PTR(-ENOKEY) if not found. - */ -static struct key *find_master_key_user(struct fscrypt_master_key *mk) +/* Find the current user's claim in ->mk_users. ->mk_sem must be held. */ +static struct fscrypt_master_key_user * +find_master_key_user(struct fscrypt_master_key *mk) { - char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE]; - key_ref_t keyref; - - format_mk_user_description(description, mk->mk_spec.u.identifier); + struct fscrypt_master_key_user *mk_user; + kuid_t uid = current_fsuid(); - /* - * We need to mark the keyring reference as "possessed" so that we - * acquire permission to search it, via the KEY_POS_SEARCH permission. - */ - keyref = keyring_search(make_key_ref(mk->mk_users, true /*possessed*/), - &key_type_fscrypt_user, description, false); - if (IS_ERR(keyref)) { - if (PTR_ERR(keyref) == -EAGAIN || /* not found */ - PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */ - keyref = ERR_PTR(-ENOKEY); - return ERR_CAST(keyref); + list_for_each_entry(mk_user, &mk->mk_users, link) { + if (uid_eq(mk_user->uid, uid)) + return mk_user; } - return key_ref_to_ptr(keyref); + return NULL; } /* - * Give the current user a "key" in ->mk_users. This charges the user's quota + * Give the current user a claim in ->mk_users. This charges the user's quota * and marks the master key as added by the current user, so that it cannot be * removed by another user with the key. Either ->mk_sem must be held for * write, or the master key must be still undergoing initialization. */ static int add_master_key_user(struct fscrypt_master_key *mk) { + kuid_t uid = current_fsuid(); char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE]; - struct key *mk_user; + struct key *quota_key; + struct fscrypt_master_key_user *mk_user; int err; - format_mk_user_description(description, mk->mk_spec.u.identifier); - mk_user = key_alloc(&key_type_fscrypt_user, description, - current_fsuid(), current_gid(), current_cred(), - KEY_POS_SEARCH | KEY_USR_VIEW, 0, NULL); - if (IS_ERR(mk_user)) - return PTR_ERR(mk_user); + snprintf(description, sizeof(description), "%*phN.uid.%u", + FSCRYPT_KEY_IDENTIFIER_SIZE, mk->mk_spec.u.identifier, + __kuid_val(uid)); + quota_key = key_alloc(&key_type_fscrypt_user, description, uid, + current_gid(), current_cred(), + KEY_POS_SEARCH | KEY_USR_VIEW, 0, NULL); + if (IS_ERR(quota_key)) + return PTR_ERR(quota_key); + + err = key_instantiate_and_link(quota_key, NULL, 0, NULL, NULL); + if (err) { + key_put(quota_key); + return err; + } - err = key_instantiate_and_link(mk_user, NULL, 0, mk->mk_users, NULL); - key_put(mk_user); - return err; + mk_user = kzalloc_obj(*mk_user); + if (!mk_user) { + key_put(quota_key); + return -ENOMEM; + } + mk_user->uid = uid; + mk_user->quota_key = quota_key; + list_add(&mk_user->link, &mk->mk_users); + return 0; +} + +static void unlink_and_free_mk_user(struct fscrypt_master_key_user *mk_user) +{ + list_del(&mk_user->link); + key_put(mk_user->quota_key); + kfree(mk_user); } /* - * Remove the current user's "key" from ->mk_users. + * Remove the current user's claim from ->mk_users. * ->mk_sem must be held for write. * - * Returns 0 if removed, -ENOKEY if not found, or another -errno code. + * Returns 0 if removed or -ENOKEY if not found. */ static int remove_master_key_user(struct fscrypt_master_key *mk) { - struct key *mk_user; - int err; + struct fscrypt_master_key_user *mk_user; mk_user = find_master_key_user(mk); - if (IS_ERR(mk_user)) - return PTR_ERR(mk_user); - err = key_unlink(mk->mk_users, mk_user); - key_put(mk_user); - return err; + if (!mk_user) + return -ENOKEY; + unlink_and_free_mk_user(mk_user); + return 0; +} + +/* + * Clear ->mk_users. Either ->mk_sem must be held for write, or 'mk' must have + * no structural references left. + */ +static void clear_mk_users(struct fscrypt_master_key *mk) +{ + struct fscrypt_master_key_user *mk_user, *tmp; + + list_for_each_entry_safe(mk_user, tmp, &mk->mk_users, link) + unlink_and_free_mk_user(mk_user); } /* @@ -442,13 +424,14 @@ static int add_new_master_key(struct super_block *sb, refcount_set(&mk->mk_struct_refs, 1); mk->mk_spec = *mk_spec; + INIT_LIST_HEAD(&mk->mk_users); + INIT_LIST_HEAD(&mk->mk_decrypted_inodes); spin_lock_init(&mk->mk_decrypted_inodes_lock); + INIT_LIST_HEAD(&mk->mk_mode_keys); + if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { - err = allocate_master_key_users_keyring(mk); - if (err) - goto out_put; err = add_master_key_user(mk); if (err) goto out_put; @@ -477,19 +460,13 @@ static int add_existing_master_key(struct fscrypt_master_key *mk, int err; /* - * If the current user is already in ->mk_users, then there's nothing to - * do. Otherwise, we need to add the user to ->mk_users. (Neither is - * applicable for v1 policy keys, which have NULL ->mk_users.) + * For v2 policy keys (FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER): If the current + * user is already in ->mk_users, then there's nothing to do. + * Otherwise, add the user to ->mk_users. */ - if (mk->mk_users) { - struct key *mk_user = find_master_key_user(mk); - - if (mk_user != ERR_PTR(-ENOKEY)) { - if (IS_ERR(mk_user)) - return PTR_ERR(mk_user); - key_put(mk_user); + if (mk->mk_spec.type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + if (find_master_key_user(mk) != NULL) return 0; - } err = add_master_key_user(mk); if (err) return err; @@ -888,7 +865,6 @@ int fscrypt_verify_key_added(struct super_block *sb, { struct fscrypt_key_specifier mk_spec; struct fscrypt_master_key *mk; - struct key *mk_user; int err; mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER; @@ -900,13 +876,10 @@ int fscrypt_verify_key_added(struct super_block *sb, goto out; } down_read(&mk->mk_sem); - mk_user = find_master_key_user(mk); - if (IS_ERR(mk_user)) { - err = PTR_ERR(mk_user); - } else { - key_put(mk_user); + if (find_master_key_user(mk) != NULL) err = 0; - } + else + err = -ENOKEY; up_read(&mk->mk_sem); fscrypt_put_master_key(mk); out: @@ -1098,16 +1071,18 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) down_write(&mk->mk_sem); /* If relevant, remove current user's (or all users) claim to the key */ - if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) { - if (all_users) - err = keyring_clear(mk->mk_users); - else + if (!list_empty(&mk->mk_users)) { + if (all_users) { + clear_mk_users(mk); + err = 0; + } else { err = remove_master_key_user(mk); + } if (err) { up_write(&mk->mk_sem); goto out_put_key; } - if (mk->mk_users->keys.nr_leaves_on_tree != 0) { + if (!list_empty(&mk->mk_users)) { /* * Other users have still added the key too. We removed * the current user's claim to the key, but we still @@ -1193,6 +1168,8 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) struct super_block *sb = file_inode(filp)->i_sb; struct fscrypt_get_key_status_arg arg; struct fscrypt_master_key *mk; + kuid_t uid; + const struct fscrypt_master_key_user *mk_user; int err; if (copy_from_user(&arg, uarg, sizeof(arg))) @@ -1225,19 +1202,13 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) } arg.status = FSCRYPT_KEY_STATUS_PRESENT; - if (mk->mk_users) { - struct key *mk_user; - arg.user_count = mk->mk_users->keys.nr_leaves_on_tree; - mk_user = find_master_key_user(mk); - if (!IS_ERR(mk_user)) { + uid = current_fsuid(); + list_for_each_entry(mk_user, &mk->mk_users, link) { + arg.user_count++; + if (uid_eq(mk_user->uid, uid)) arg.status_flags |= FSCRYPT_KEY_STATUS_FLAG_ADDED_BY_SELF; - key_put(mk_user); - } else if (mk_user != ERR_PTR(-ENOKEY)) { - err = PTR_ERR(mk_user); - goto out_release_key; - } } err = 0; out_release_key: diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index ce327bfdada46..f905f9f94bdd5 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -163,13 +163,7 @@ int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode); if (IS_ERR(tfm)) return PTR_ERR(tfm); - /* - * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared(). - * I.e., here we publish ->tfm with a RELEASE barrier so that - * concurrent tasks can ACQUIRE it. Note that this concurrency is only - * possible for per-mode keys, not for per-file keys. - */ - smp_store_release(&prep_key->tfm, tfm); + prep_key->tfm = tfm; return 0; } @@ -190,9 +184,37 @@ int fscrypt_set_per_file_enc_key(struct fscrypt_inode_info *ci, return fscrypt_prepare_key(&ci->ci_enc_key, raw_key, ci); } +/* + * Find the fscrypt_prepared_key (if any) for a particular (mk, hkdf_context, + * mode_num, data_unit_bits, inlinecrypt) combination. + * + * The caller must hold ->mk_sem for reading and ->mk_present must be true, + * ensuring that ->mk_mode_keys is still append-only. + */ +static struct fscrypt_prepared_key * +fscrypt_find_mode_key(struct fscrypt_master_key *mk, u8 hkdf_context, + u8 mode_num, const struct fscrypt_inode_info *ci) +{ + struct fscrypt_mode_key *node; + + /* + * The RCU read lock here is used only to synchronize with concurrent + * list_add_tail_rcu(). Concurrent deletions are impossible here, so + * returning a pointer to a node without taking any refcount is safe. + */ + guard(rcu)(); + list_for_each_entry_rcu(node, &mk->mk_mode_keys, link) { + if (node->hkdf_context == hkdf_context && + node->mode_num == mode_num && + node->data_unit_bits == ci->ci_data_unit_bits && + fscrypt_is_key_prepared(&node->key, ci)) + return &node->key; + } + return NULL; +} + static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci, struct fscrypt_master_key *mk, - struct fscrypt_prepared_key *keys, u8 hkdf_context, bool include_fs_uuid) { const struct inode *inode = ci->ci_inode; @@ -200,7 +222,8 @@ static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci, struct fscrypt_mode *mode = ci->ci_mode; const u8 mode_num = mode - fscrypt_modes; struct fscrypt_prepared_key *prep_key; - u8 mode_key[FSCRYPT_MAX_RAW_KEY_SIZE]; + struct fscrypt_mode_key *new_node; + u8 raw_mode_key[FSCRYPT_MAX_RAW_KEY_SIZE]; u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; unsigned int hkdf_infolen = 0; bool use_hw_wrapped_key = false; @@ -223,48 +246,56 @@ static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci, use_hw_wrapped_key = true; } - prep_key = &keys[mode_num]; - if (fscrypt_is_key_prepared(prep_key, ci)) { + prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci); + if (prep_key) { ci->ci_enc_key = *prep_key; return 0; } - mutex_lock(&fscrypt_mode_key_setup_mutex); + guard(mutex)(&fscrypt_mode_key_setup_mutex); - if (fscrypt_is_key_prepared(prep_key, ci)) - goto done_unlock; + prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci); + if (prep_key) { + ci->ci_enc_key = *prep_key; + return 0; + } + + new_node = kzalloc_obj(*new_node); + if (!new_node) + return -ENOMEM; + new_node->hkdf_context = hkdf_context; + new_node->mode_num = mode_num; + new_node->data_unit_bits = ci->ci_data_unit_bits; + prep_key = &new_node->key; if (use_hw_wrapped_key) { err = fscrypt_prepare_inline_crypt_key(prep_key, mk->mk_secret.bytes, mk->mk_secret.size, true, ci); - if (err) - goto out_unlock; - goto done_unlock; + } else { + static_assert(sizeof(mode_num) == 1); + static_assert(sizeof(sb->s_uuid) == 16); + static_assert(sizeof(hkdf_info) == 17); + hkdf_info[hkdf_infolen++] = mode_num; + if (include_fs_uuid) { + memcpy(&hkdf_info[hkdf_infolen], &sb->s_uuid, + sizeof(sb->s_uuid)); + hkdf_infolen += sizeof(sb->s_uuid); + } + fscrypt_hkdf_expand(&mk->mk_secret.hkdf, hkdf_context, + hkdf_info, hkdf_infolen, raw_mode_key, + mode->keysize); + err = fscrypt_prepare_key(prep_key, raw_mode_key, ci); + memzero_explicit(raw_mode_key, mode->keysize); } - - BUILD_BUG_ON(sizeof(mode_num) != 1); - BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); - BUILD_BUG_ON(sizeof(hkdf_info) != 17); - hkdf_info[hkdf_infolen++] = mode_num; - if (include_fs_uuid) { - memcpy(&hkdf_info[hkdf_infolen], &sb->s_uuid, - sizeof(sb->s_uuid)); - hkdf_infolen += sizeof(sb->s_uuid); + if (err) { + kfree(new_node); + return err; } - fscrypt_hkdf_expand(&mk->mk_secret.hkdf, hkdf_context, hkdf_info, - hkdf_infolen, mode_key, mode->keysize); - err = fscrypt_prepare_key(prep_key, mode_key, ci); - memzero_explicit(mode_key, mode->keysize); - if (err) - goto out_unlock; -done_unlock: + list_add_tail_rcu(&new_node->link, &mk->mk_mode_keys); ci->ci_enc_key = *prep_key; - err = 0; -out_unlock: - mutex_unlock(&fscrypt_mode_key_setup_mutex); - return err; + return 0; } /* @@ -311,8 +342,8 @@ static int fscrypt_setup_iv_ino_lblk_32_key(struct fscrypt_inode_info *ci, { int err; - err = setup_per_mode_enc_key(ci, mk, mk->mk_iv_ino_lblk_32_keys, - HKDF_CONTEXT_IV_INO_LBLK_32_KEY, true); + err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_IV_INO_LBLK_32_KEY, + true); if (err) return err; @@ -364,8 +395,8 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_inode_info *ci, * encryption key. This ensures that the master key is * consistently used only for HKDF, avoiding key reuse issues. */ - err = setup_per_mode_enc_key(ci, mk, mk->mk_direct_keys, - HKDF_CONTEXT_DIRECT_KEY, false); + err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_DIRECT_KEY, + false); } else if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { /* @@ -374,9 +405,8 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_inode_info *ci, * the IVs. This format is optimized for use with inline * encryption hardware compliant with the UFS standard. */ - err = setup_per_mode_enc_key(ci, mk, mk->mk_iv_ino_lblk_64_keys, - HKDF_CONTEXT_IV_INO_LBLK_64_KEY, - true); + err = setup_per_mode_enc_key( + ci, mk, HKDF_CONTEXT_IV_INO_LBLK_64_KEY, true); } else if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) { err = fscrypt_setup_iv_ino_lblk_32_key(ci, mk); diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index d54bdd8fc4f2e..64826a9b79a55 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -968,10 +968,10 @@ static void midcomms_new_msg_cb(void *data) atomic_inc(&mh->node->send_queue_cnt); spin_lock_bh(&mh->node->send_queue_lock); + /* need to be locked with list_add_tail_rcu() because list is ordered */ + mh->seq = atomic_fetch_inc(&mh->node->seq_send); list_add_tail_rcu(&mh->list, &mh->node->send_queue); spin_unlock_bh(&mh->node->send_queue_lock); - - mh->seq = atomic_fetch_inc(&mh->node->seq_send); } static struct dlm_msg *dlm_midcomms_get_msg_3_2(struct dlm_mhandle *mh, int nodeid, diff --git a/fs/efs/file.c b/fs/efs/file.c index 9e641da6fab27..9153dfe79bbcc 100644 --- a/fs/efs/file.c +++ b/fs/efs/file.c @@ -18,16 +18,9 @@ int efs_get_block(struct inode *inode, sector_t iblock, if (create) return error; - if (iblock >= inode->i_blocks) { -#ifdef DEBUG - /* - * i have no idea why this happens as often as it does - */ - pr_warn("%s(): block %d >= %ld (filesize %ld)\n", - __func__, block, inode->i_blocks, inode->i_size); -#endif + if (iblock >= inode->i_blocks) return 0; - } + phys = efs_map_block(inode, iblock); if (phys) map_bh(bh_result, inode->i_sb, phys); @@ -42,16 +35,8 @@ int efs_bmap(struct inode *inode, efs_block_t block) { } /* are we about to read past the end of a file ? */ - if (!(block < inode->i_blocks)) { -#ifdef DEBUG - /* - * i have no idea why this happens as often as it does - */ - pr_warn("%s(): block %d >= %ld (filesize %ld)\n", - __func__, block, inode->i_blocks, inode->i_size); -#endif + if (!(block < inode->i_blocks)) return 0; - } return efs_map_block(inode, block); } diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index a188c570087ae..e0c47da4f09e0 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -191,8 +191,9 @@ static int erofs_read_inode(struct inode *inode) err = -EFSCORRUPTED; goto err_out; } else { - inode->i_blocks = le32_to_cpu(copied.i_u.blocks_lo) << - (sb->s_blocksize_bits - 9); + inode->i_blocks = (le32_to_cpu(copied.i_u.blocks_lo) | + ((u64)le16_to_cpu(copied.i_nb.blocks_hi) << 32)) << + (sb->s_blocksize_bits - 9); } if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 802add6652fda..579443e6acfeb 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -1048,11 +1048,11 @@ static int __init erofs_module_init(void) static void __exit erofs_module_exit(void) { unregister_filesystem(&erofs_fs_type); + erofs_exit_ishare(); - /* Ensure all RCU free inodes / pclusters are safe to be destroyed. */ + /* ensure all delayed rcu free inodes & pclusters are flushed */ rcu_barrier(); - erofs_exit_ishare(); erofs_exit_sysfs(); z_erofs_exit_subsystem(); erofs_exit_shrinker(); diff --git a/fs/eventpoll.c b/fs/eventpoll.c index a3090b446af10..c35580194ad0e 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -38,48 +38,174 @@ #include #include #include +#include #include /* - * LOCKING: - * There are three level of locking required by epoll : + * fs/eventpoll.c - Efficient event polling ("epoll") kernel implementation. * - * 1) epnested_mutex (mutex) - * 2) ep->mtx (mutex) - * 3) ep->lock (spinlock) * - * The acquire order is the one listed above, from 1 to 3. - * We need a spinlock (ep->lock) because we manipulate objects - * from inside the poll callback, that might be triggered from - * a wake_up() that in turn might be called from IRQ context. - * So we can't sleep inside the poll callback and hence we need - * a spinlock. During the event transfer loop (from kernel to - * user space) we could end up sleeping due a copy_to_user(), so - * we need a lock that will allow us to sleep. This lock is a - * mutex (ep->mtx). It is acquired during the event transfer loop, - * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file(). - * The epnested_mutex is acquired when inserting an epoll fd onto another - * epoll fd. We do this so that we walk the epoll tree and ensure that this - * insertion does not create a cycle of epoll file descriptors, which - * could lead to deadlock. We need a global mutex to prevent two - * simultaneous inserts (A into B and B into A) from racing and - * constructing a cycle without either insert observing that it is - * going to. - * It is necessary to acquire multiple "ep->mtx"es at once in the - * case when one epoll fd is added to another. In this case, we - * always acquire the locks in the order of nesting (i.e. after - * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired - * before e2->mtx). Since we disallow cycles of epoll file - * descriptors, this ensures that the mutexes are well-ordered. In - * order to communicate this nesting to lockdep, when walking a tree - * of epoll file descriptors, we use the current recursion depth as - * the lockdep subkey. - * It is possible to drop the "ep->mtx" and to use the global - * mutex "epnested_mutex" (together with "ep->lock") to have it working, - * but having "ep->mtx" will make the interface more scalable. - * Events that require holding "epnested_mutex" are very rare, while for - * normal operations the epoll private "ep->mtx" will guarantee - * a better scalability. + * Overview + * -------- + * + * Each epoll_create(2) returns an anonymous [eventpoll] file whose + * ->private_data is a struct eventpoll. Each EPOLL_CTL_ADD installs + * a struct epitem linking one (watched file, fd) pair back to that + * eventpoll via the watched file's f_op->poll() wait queue(s). When + * the watched file signals readiness, ep_poll_callback() fires and + * marks the epitem ready. epoll_wait(2) drains the ready list under + * ep->mtx, re-queueing items in level-triggered mode. + * + * epoll instances can watch other epoll instances up to EP_MAX_NESTS + * deep; cycles are forbidden and detected at EPOLL_CTL_ADD time. + * + * + * Locking + * ------- + * + * Three levels, acquired from outer to inner: + * + * epnested_mutex (global; rare; taken only for EPOLL_CTL_ADD + * loop / path checks) + * > ep->mtx (per-eventpoll; sleepable; serializes most ops) + * > ep->lock (per-eventpoll; IRQ-safe spinlock) + * + * file->f_lock (per-file; NOT IRQ-safe; guards f_ep hlist ops; + * nested inside ep->mtx, outside ep->lock) + * + * Rationale: + * - ep->lock is a spinlock because ep_poll_callback() is called from + * wake_up() which may run in hard-IRQ context. All ep->lock + * critical sections use spin_lock_irqsave(). + * - ep->mtx is a sleepable mutex because the event delivery loop + * calls copy_to_user(), and ep_insert() may sleep in + * kmem_cache_alloc() and f_op->poll(). + * - epnested_mutex is global because cycle detection needs a global + * view of the epoll topology; a per-object scheme would let two + * concurrent inserts (A into B, B into A) construct a cycle + * without either observer seeing it. + * - Per-ep ep->mtx is preferred for scalability elsewhere. Events + * that require epnested_mutex are rare. + * + * When EPOLL_CTL_ADD nests one eventpoll inside another we acquire + * ep->mtx on both: outer first, target second. Since cycles are + * forbidden the set of live ep->mtx holds is always a strict chain, + * communicated to lockdep via mutex_lock_nested() subclasses derived + * from the current recursion depth. + * + * + * Field protection + * ---------------- + * + * struct eventpoll: + * mtx - self + * rbr - ep->mtx + * ovflist, rdllist - ep->lock (IRQ-safe) + * wq - ep->lock for queue mutation + * poll_wait - internal waitqueue spinlock + * refs - file->f_lock for adds; ep->mtx for removes; + * RCU for readers (hlist_del_rcu + kfree_rcu(ep)) + * ws - ep->mtx + * gen, loop_check_depth - epnested_mutex + * file, user - immutable after setup + * refcount - atomic (refcount_t) + * napi_* - READ_ONCE / WRITE_ONCE + * + * struct epitem: + * rbn / rcu union - rbn: ep->mtx (while epi is linked in ep->rbr). + * rcu: written only by kfree_rcu(epi) on the free + * path; otherwise untouched by epoll code. + * rdllink, next - ep->lock + * ffd, ep - immutable after ep_insert() + * pwqlist - ep->mtx for writes; POLLFREE clears pwq->whead + * via smp_store_release(), see below + * fllink - file->f_lock for mutation; hlist_del_rcu + + * kfree_rcu(epi) for safe RCU readers + * ws - RCU (rcu_assign_pointer / + * rcu_dereference_check(mtx)) + * event - ep->mtx for writes; lockless read in + * ep_poll_callback pairs with smp_mb() in + * ep_modify() + * + * + * Ready-list state machine + * ------------------------ + * + * Readiness is tracked in two lists under ep->lock: + * + * rdllist - doubly-linked FIFO; the "current" ready list. + * ovflist - singly-linked LIFO; used during a scan to catch + * events that arrive while rdllist is being iterated + * without ep->lock. + * + * Encoded in ep->ovflist: + * EP_UNACTIVE_PTR - no scan active; callback appends to rdllist. + * NULL - scan active, no spill yet. + * pointer to epi - scan active with spilled items (LIFO). + * + * Encoded in epi->ovflist_next: + * EP_UNACTIVE_PTR - epi is not on ovflist. + * otherwise - next epi on ovflist (NULL at tail). + * + * ep_start_scan() flips "not scanning" to "scanning" and splices + * rdllist into a caller-local scan_batch. ep_done_scan() drains ovflist + * back to rdllist (list_add head-insert reverses LIFO to FIFO), + * flips back to "not scanning", and re-splices any items the caller + * left in scan_batch (e.g., level-triggered re-queues). + * + * + * Removal paths + * ------------- + * + * Three paths dispose of epitems and/or eventpolls: + * + * A. ep_remove() - EPOLL_CTL_DEL and ep_insert() + * rollback. Caller holds ep->mtx. + * B. ep_clear_and_put() - close of the epoll fd itself + * (ep_eventpoll_release). + * C. eventpoll_release_file() - close of a watched file, invoked + * from __fput(). + * + * Coordination: + * A and C exclude each other via the watched file's refcount. + * A pins the file with epi_fget() before touching file->f_ep or + * file->f_lock; if the pin fails, __fput() is in flight and C + * will clean this epi up. See the epi_fget() block comment. + * A and B both hold ep->mtx serially. B walks the rbtree with + * rb_next() captured before ep_remove() erases the current node. + * B and C both take ep->mtx; the loser sees fewer entries or an + * empty file->f_ep. + * + * Within every path the internal order is strict: + * ep_unregister_pollwait() - drain pwqlist; synchronizes with any + * in-flight ep_poll_callback via the + * watched wait-queue head's lock. + * ep_remove_file() - hlist_del_rcu of epi->fllink and, + * if last watcher, clear file->f_ep, + * under file->f_lock. + * ep_remove_epi() - rb_erase, rdllist unlink (ep->lock), + * wakeup_source_unregister, + * kfree_rcu(epi). + * + * kfree_rcu(epi) defers the free past RCU readers in + * reverse_path_check_proc(); kfree_rcu(ep) defers past readers in + * ep_get_upwards_depth_proc(). + * + * + * POLLFREE handshake + * ------------------ + * + * When a subsystem tears down a wait-queue head that an epitem is + * registered on (binder, signalfd, ...), it wakes the callback with + * POLLFREE and must RCU-defer the head's free. The store/load pair: + * + * ep_poll_callback() POLLFREE branch: + * smp_store_release(&pwq->whead, NULL) + * + * ep_remove_wait_queue(): + * smp_load_acquire(&pwq->whead) + * + * See those sites for the full argument. */ /* Epoll private bits inside the event mask */ @@ -136,14 +262,16 @@ struct epitem { struct rcu_head rcu; }; - /* List header used to link this structure to the eventpoll ready list */ + /* Link on the owning eventpoll's ready list (ep->rdllist). */ struct list_head rdllink; /* - * Works together "struct eventpoll"->ovflist in keeping the - * single linked chain of items. + * Link on the owning eventpoll's scan-overflow list (ep->ovflist), + * EP_UNACTIVE_PTR when not linked. See epi_on_ovflist() / + * epi_clear_ovflist() and the "Ready-list state machine" section + * in the top-of-file banner. */ - struct epitem *next; + struct epitem *ovflist_next; /* The file descriptor information this item refers to */ struct epoll_filefd ffd; @@ -190,6 +318,9 @@ struct eventpoll { /* Lock which protects rdllist and ovflist */ spinlock_t lock; + /* Protect switching between rdllist and ovflist */ + seqcount_spinlock_t seq; + /* RB tree root used to store monitored fd structs */ struct rb_root_cached rbr; @@ -372,6 +503,43 @@ static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p) return container_of(p, struct eppoll_entry, wait)->base; } +/* + * Ready-list / ovflist state (see "Ready-list state machine" in the + * top-of-file banner for the full state machine). EP_UNACTIVE_PTR is + * the sentinel; these wrappers name each transition and each test so + * call sites do not need to know the sentinel's value. + */ + +/* True iff @ep is between ep_enter_scan() and ep_exit_scan(). */ +static inline bool ep_is_scanning(struct eventpoll *ep) +{ + return READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; +} + +/* Called by ep_start_scan(): divert ep_poll_callback() to ovflist. */ +static inline void ep_enter_scan(struct eventpoll *ep) +{ + WRITE_ONCE(ep->ovflist, NULL); +} + +/* Called by ep_done_scan(): redirect ep_poll_callback() back to rdllist. */ +static inline void ep_exit_scan(struct eventpoll *ep) +{ + WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); +} + +/* True iff @epi is currently linked on its ep's ovflist. */ +static inline bool epi_on_ovflist(const struct epitem *epi) +{ + return epi->ovflist_next != EP_UNACTIVE_PTR; +} + +/* Mark @epi as not on any ovflist (init and post-drain). */ +static inline void epi_clear_ovflist(struct epitem *epi) +{ + epi->ovflist_next = EP_UNACTIVE_PTR; +} + /** * ep_events_available - Checks if ready events might be available. * @@ -382,8 +550,10 @@ static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p) */ static inline int ep_events_available(struct eventpoll *ep) { - return !list_empty_careful(&ep->rdllist) || - READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; + unsigned int seq = read_seqcount_begin(&ep->seq); + + return !list_empty_careful(&ep->rdllist) || ep_is_scanning(ep) || + read_seqcount_retry(&ep->seq, seq); } #ifdef CONFIG_NET_RX_BUSY_POLL @@ -723,7 +893,7 @@ static inline void ep_pm_stay_awake_rcu(struct epitem *epi) * ep->mutex needs to be held because we could be hit by * eventpoll_release_file() and epoll_ctl(). */ -static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist) +static void ep_start_scan(struct eventpoll *ep, struct list_head *scan_batch) { /* * Steal the ready list, and re-init the original one to the @@ -735,13 +905,17 @@ static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist) */ lockdep_assert_irqs_enabled(); spin_lock_irq(&ep->lock); - list_splice_init(&ep->rdllist, txlist); - WRITE_ONCE(ep->ovflist, NULL); + write_seqcount_begin(&ep->seq); + + list_splice_init(&ep->rdllist, scan_batch); + ep_enter_scan(ep); + + write_seqcount_end(&ep->seq); spin_unlock_irq(&ep->lock); } static void ep_done_scan(struct eventpoll *ep, - struct list_head *txlist) + struct list_head *scan_batch) { struct epitem *epi, *nepi; @@ -751,34 +925,35 @@ static void ep_done_scan(struct eventpoll *ep, * other events might have been queued by the poll callback. * We re-insert them inside the main ready-list here. */ - for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; - nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { + for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; ) { + nepi = epi->ovflist_next; + epi_clear_ovflist(epi); /* - * We need to check if the item is already in the list. - * During the "sproc" callback execution time, items are - * queued into ->ovflist but the "txlist" might already - * contain them, and the list_splice() below takes care of them. + * Skip items that the caller already returned via @scan_batch + * -- the list_splice() below takes care of those. */ if (!ep_is_linked(epi)) { /* - * ->ovflist is LIFO, so we have to reverse it in order - * to keep in FIFO. + * ovflist is LIFO; list_add() head-insert here + * reverses the iteration order into FIFO. */ list_add(&epi->rdllink, &ep->rdllist); ep_pm_stay_awake(epi); } } - /* - * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after - * releasing the lock, events will be queued in the normal way inside - * ep->rdllist. - */ - WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); + + write_seqcount_begin(&ep->seq); + + /* Back out of scan mode; callbacks target ep->rdllist again. */ + ep_exit_scan(ep); /* - * Quickly re-inject items left on "txlist". + * Quickly re-inject items left on "scan_batch". */ - list_splice(txlist, &ep->rdllist); + list_splice(scan_batch, &ep->rdllist); + + write_seqcount_end(&ep->seq); + __pm_relax(ep->ws); if (!list_empty(&ep->rdllist)) { @@ -999,7 +1174,7 @@ static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth) { struct eventpoll *ep = file->private_data; - LIST_HEAD(txlist); + LIST_HEAD(scan_batch); struct epitem *epi, *tmp; poll_table pt; __poll_t res = 0; @@ -1014,8 +1189,8 @@ static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int dep * the ready list. */ mutex_lock_nested(&ep->mtx, depth); - ep_start_scan(ep, &txlist); - list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { + ep_start_scan(ep, &scan_batch); + list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) { if (ep_item_poll(epi, &pt, depth + 1)) { res = EPOLLIN | EPOLLRDNORM; break; @@ -1029,7 +1204,7 @@ static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int dep list_del_init(&epi->rdllink); } } - ep_done_scan(ep, &txlist); + ep_done_scan(ep, &scan_batch); mutex_unlock(&ep->mtx); return res; } @@ -1155,11 +1330,12 @@ static int ep_alloc(struct eventpoll **pep) mutex_init(&ep->mtx); spin_lock_init(&ep->lock); + seqcount_spinlock_init(&ep->seq, &ep->lock); init_waitqueue_head(&ep->wq); init_waitqueue_head(&ep->poll_wait); INIT_LIST_HEAD(&ep->rdllist); ep->rbr = RB_ROOT_CACHED; - ep->ovflist = EP_UNACTIVE_PTR; + ep->ovflist = EP_UNACTIVE_PTR; /* not scanning */ ep->user = get_current_user(); refcount_set(&ep->refcount, 1); @@ -1283,9 +1459,9 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v * semantics). All the events that happen during that period of time are * chained in ep->ovflist and requeued later on. */ - if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) { - if (epi->next == EP_UNACTIVE_PTR) { - epi->next = READ_ONCE(ep->ovflist); + if (ep_is_scanning(ep)) { + if (!epi_on_ovflist(epi)) { + epi->ovflist_next = READ_ONCE(ep->ovflist); WRITE_ONCE(ep->ovflist, epi); ep_pm_stay_awake_rcu(epi); } @@ -1526,7 +1702,7 @@ static noinline void ep_destroy_wakeup_source(struct epitem *epi) wakeup_source_unregister(ws); } -static int attach_epitem(struct file *file, struct epitem *epi) +static int ep_attach_file(struct file *file, struct epitem *epi) { struct epitems_head *to_free = NULL; struct hlist_head *head = NULL; @@ -1561,68 +1737,112 @@ static int attach_epitem(struct file *file, struct epitem *epi) } /* - * Must be called with "mtx" held. + * Charge the user's epoll_watches quota, allocate a fresh epitem for + * @tfile/@fd, and initialize its fields. The returned item is not yet + * linked into any data structure; the caller must install it via + * ep_register_epitem() (which takes over on success) or kmem_cache_free() + * it and decrement epoll_watches on its own. + * + * Returns ERR_PTR(-ENOSPC) if the quota is exceeded, ERR_PTR(-ENOMEM) + * if the slab allocation fails. */ -static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, - struct file *tfile, int fd, int full_check) +static struct epitem *ep_alloc_epitem(struct eventpoll *ep, + const struct epoll_event *event, + struct file *tfile, int fd) { - int error, pwake = 0; - __poll_t revents; struct epitem *epi; - struct ep_pqueue epq; - struct eventpoll *tep = NULL; - - if (is_file_epoll(tfile)) - tep = tfile->private_data; - - lockdep_assert_irqs_enabled(); if (unlikely(percpu_counter_compare(&ep->user->epoll_watches, max_user_watches) >= 0)) - return -ENOSPC; + return ERR_PTR(-ENOSPC); percpu_counter_inc(&ep->user->epoll_watches); - if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) { + epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL); + if (unlikely(!epi)) { percpu_counter_dec(&ep->user->epoll_watches); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } - /* Item initialization follow here ... */ INIT_LIST_HEAD(&epi->rdllink); epi->ep = ep; ep_set_ffd(&epi->ffd, tfile, fd); epi->event = *event; - epi->next = EP_UNACTIVE_PTR; + epi_clear_ovflist(epi); + + return epi; +} + +/* + * Install @epi into its target file's f_ep hlist and into @ep's rbtree, + * taking one additional reference on @ep for the lifetime of the item. + * + * If @tep is non-NULL, the target file is itself an eventpoll; we hold + * tep->mtx at subclass 1 across the attach + rbtree insert to serialize + * with the target side. RB tree ops are protected by @ep->mtx, which + * the caller already holds. + * + * On failure the epi is freed and the epoll_watches counter decremented, + * matching ep_alloc_epitem()'s allocation. After this returns + * successfully, ep_insert()'s later error paths use ep_remove() for + * unwind; that cannot drop @ep's refcount to zero because the ep file + * itself still holds the original reference. + */ +static int ep_register_epitem(struct eventpoll *ep, struct epitem *epi, + struct eventpoll *tep, int full_check) +{ + struct file *tfile = epi->ffd.file; + int error; if (tep) mutex_lock_nested(&tep->mtx, 1); - /* Add the current item to the list of active epoll hook for this file */ - if (unlikely(attach_epitem(tfile, epi) < 0)) { + + error = ep_attach_file(tfile, epi); + if (unlikely(error)) { if (tep) mutex_unlock(&tep->mtx); kmem_cache_free(epi_cache, epi); percpu_counter_dec(&ep->user->epoll_watches); - return -ENOMEM; + return error; } if (full_check && !tep) list_file(tfile); - /* - * Add the current item to the RB tree. All RB tree operations are - * protected by "mtx", and ep_insert() is called with "mtx" held. - */ ep_rbtree_insert(ep, epi); + if (tep) mutex_unlock(&tep->mtx); - /* - * ep_remove() calls in the later error paths can't lead to - * ep_free() as the ep file itself still holds an ep reference. - */ ep_get(ep); + return 0; +} + +/* + * Must be called with "mtx" held. + */ +static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, + struct file *tfile, int fd, int full_check) +{ + int error, pwake = 0; + __poll_t revents; + struct epitem *epi; + struct ep_pqueue epq; + struct eventpoll *tep = NULL; - /* now check if we've created too many backpaths */ + if (is_file_epoll(tfile)) + tep = tfile->private_data; + + lockdep_assert_irqs_enabled(); + + epi = ep_alloc_epitem(ep, event, tfile, fd); + if (IS_ERR(epi)) + return PTR_ERR(epi); + + error = ep_register_epitem(ep, epi, tep, full_check); + if (error) + return error; + + /* Reject the insert if the new link would create too many back-paths. */ if (unlikely(full_check && reverse_path_check())) { ep_remove(ep, epi); return -EINVAL; @@ -1649,28 +1869,21 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, */ revents = ep_item_poll(epi, &epq.pt, 1); - /* - * We have to check if something went wrong during the poll wait queue - * install process. Namely an allocation for a wait queue failed due - * high memory pressure. - */ + /* ep_ptable_queue_proc() signals allocation failure by clearing epq.epi. */ if (unlikely(!epq.epi)) { ep_remove(ep, epi); return -ENOMEM; } - /* We have to drop the new item inside our item list to keep track of it */ + /* Drop the new item onto the ready list if it is already ready. */ spin_lock_irq(&ep->lock); - /* record NAPI ID of new item if present */ ep_set_busy_poll_napi_id(epi); - /* If the file is already "ready" we drop it inside the ready list */ if (revents && !ep_is_linked(epi)) { list_add_tail(&epi->rdllink, &ep->rdllist); ep_pm_stay_awake(epi); - /* Notify waiting tasks that events are available */ if (waitqueue_active(&ep->wq)) wake_up(&ep->wq); if (waitqueue_active(&ep->poll_wait)) @@ -1762,11 +1975,87 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi, return 0; } +/* + * Attempt to deliver one event for @epi into @*uevents. + * + * Returns 1 if an event was delivered (with *uevents advanced to the + * next slot), 0 if the re-poll reported no caller-requested events + * (@epi drops out of the ready list; a future callback will re-add + * it), or -EFAULT if copy_to_user() faulted (in which case @epi is + * re-inserted at the head of @scan_batch so ep_done_scan() merges it + * back to rdllist for the next attempt). + * + * PM bookkeeping and level-triggered re-queue are handled here. + * Caller holds ep->mtx and the scan is active. + */ +static int ep_deliver_event(struct eventpoll *ep, struct epitem *epi, + poll_table *pt, + struct epoll_event __user **uevents, + struct list_head *scan_batch) +{ + struct epoll_event __user *next; + struct wakeup_source *ws; + __poll_t revents; + + /* + * Activate ep->ws before deactivating epi->ws to prevent + * triggering auto-suspend here (in case we reactivate epi->ws + * below). Rearranging to delay the deactivation would let + * epi->ws drift out of sync with ep_is_linked(). + */ + ws = ep_wakeup_source(epi); + if (ws) { + if (ws->active) + __pm_stay_awake(ep->ws); + __pm_relax(ws); + } + + list_del_init(&epi->rdllink); + + /* + * Re-poll under ep->mtx so userspace cannot change the item + * out from under us. If no caller-requested events remain, + * @epi stays off the ready list; the poll callback will + * re-queue it when events next appear. + */ + revents = ep_item_poll(epi, pt, 1); + if (!revents) + return 0; + + next = epoll_put_uevent(revents, epi->event.data, *uevents); + if (!next) { + /* + * copy_to_user() faulted: put the item back so + * ep_done_scan() splices it onto rdllist for the next + * attempt. + */ + list_add(&epi->rdllink, scan_batch); + ep_pm_stay_awake(epi); + return -EFAULT; + } + *uevents = next; + + if (epi->event.events & EPOLLONESHOT) { + epi->event.events &= EP_PRIVATE_BITS; + } else if (!(epi->event.events & EPOLLET)) { + /* + * Level-triggered: re-queue so the next epoll_wait() + * rechecks availability. We are the sole writer to + * rdllist here -- epoll_ctl() callers are locked out + * by ep->mtx, and the poll callback queues to ovflist + * during scans. + */ + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake(epi); + } + return 1; +} + static int ep_send_events(struct eventpoll *ep, struct epoll_event __user *events, int maxevents) { struct epitem *epi, *tmp; - LIST_HEAD(txlist); + LIST_HEAD(scan_batch); poll_table pt; int res = 0; @@ -1781,74 +2070,28 @@ static int ep_send_events(struct eventpoll *ep, init_poll_funcptr(&pt, NULL); mutex_lock(&ep->mtx); - ep_start_scan(ep, &txlist); + ep_start_scan(ep, &scan_batch); /* - * We can loop without lock because we are passed a task private list. - * Items cannot vanish during the loop we are holding ep->mtx. + * We can loop without lock because we are passed a task-private + * scan_batch; items cannot vanish while we hold ep->mtx. */ - list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { - struct wakeup_source *ws; - __poll_t revents; + list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) { + int delivered; if (res >= maxevents) break; - /* - * Activate ep->ws before deactivating epi->ws to prevent - * triggering auto-suspend here (in case we reactive epi->ws - * below). - * - * This could be rearranged to delay the deactivation of epi->ws - * instead, but then epi->ws would temporarily be out of sync - * with ep_is_linked(). - */ - ws = ep_wakeup_source(epi); - if (ws) { - if (ws->active) - __pm_stay_awake(ep->ws); - __pm_relax(ws); - } - - list_del_init(&epi->rdllink); - - /* - * If the event mask intersect the caller-requested one, - * deliver the event to userspace. Again, we are holding ep->mtx, - * so no operations coming from userspace can change the item. - */ - revents = ep_item_poll(epi, &pt, 1); - if (!revents) - continue; - - events = epoll_put_uevent(revents, epi->event.data, events); - if (!events) { - list_add(&epi->rdllink, &txlist); - ep_pm_stay_awake(epi); + delivered = ep_deliver_event(ep, epi, &pt, &events, &scan_batch); + if (delivered < 0) { if (!res) - res = -EFAULT; + res = delivered; break; } - res++; - if (epi->event.events & EPOLLONESHOT) - epi->event.events &= EP_PRIVATE_BITS; - else if (!(epi->event.events & EPOLLET)) { - /* - * If this file has been added with Level - * Trigger mode, we need to insert back inside - * the ready list, so that the next call to - * epoll_wait() will check again the events - * availability. At this point, no one can insert - * into ep->rdllist besides us. The epoll_ctl() - * callers are locked out by - * ep_send_events() holding "mtx" and the - * poll callback will queue them in ep->ovflist. - */ - list_add_tail(&epi->rdllink, &ep->rdllist); - ep_pm_stay_awake(epi); - } + res += delivered; } - ep_done_scan(ep, &txlist); + + ep_done_scan(ep, &scan_batch); mutex_unlock(&ep->mtx); return res; diff --git a/fs/exfat/balloc.c b/fs/exfat/balloc.c index 625f2f14d4fe0..e66ebf899778c 100644 --- a/fs/exfat/balloc.c +++ b/fs/exfat/balloc.c @@ -112,7 +112,7 @@ static int exfat_allocate_bitmap(struct super_block *sb, } if (exfat_test_bitmap_range(sb, sbi->map_clu, - EXFAT_B_TO_CLU_ROUND_UP(map_size, sbi)) == false) + exfat_bytes_to_cluster_round_up(sbi, map_size)) == false) goto err_out; return 0; diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c index ac008ccaa97de..b316541a09aed 100644 --- a/fs/exfat/dir.c +++ b/fs/exfat/dir.c @@ -76,7 +76,7 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); struct exfat_inode_info *ei = EXFAT_I(inode); - unsigned int dentry = EXFAT_B_TO_DEN(*cpos) & 0xFFFFFFFF; + unsigned int dentry = exfat_bytes_to_dentries(*cpos) & 0xFFFFFFFF; struct buffer_head *bh; /* check if the given file ID is opened */ @@ -84,13 +84,13 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent return -EPERM; exfat_chain_set(&dir, ei->start_clu, - EXFAT_B_TO_CLU(i_size_read(inode), sbi), ei->flags); + exfat_bytes_to_cluster(sbi, i_size_read(inode)), ei->flags); dentries_per_clu = sbi->dentries_per_clu; - max_dentries = (unsigned int)min_t(u64, MAX_EXFAT_DENTRIES, - (u64)EXFAT_CLU_TO_DEN(sbi->num_clusters, sbi)); + max_dentries = min(MAX_EXFAT_DENTRIES, + exfat_cluster_to_dentries(sbi, sbi->num_clusters)); - clu_offset = EXFAT_DEN_TO_CLU(dentry, sbi); + clu_offset = exfat_dentries_to_cluster(sbi, dentry); exfat_chain_dup(&clu, &dir); if (clu.flags == ALLOC_FAT_CHAIN) { @@ -147,10 +147,10 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent dir_entry->dir = clu; brelse(bh); - ei->hint_bmap.off = EXFAT_DEN_TO_CLU(dentry, sbi); + ei->hint_bmap.off = exfat_dentries_to_cluster(sbi, dentry); ei->hint_bmap.clu = clu.dir; - *cpos = EXFAT_DEN_TO_B(dentry + 1 + num_ext); + *cpos = exfat_dentries_to_bytes(dentry + 1 + num_ext); return 0; } @@ -160,7 +160,7 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent out: dir_entry->namebuf.lfn[0] = '\0'; - *cpos = EXFAT_DEN_TO_B(dentry); + *cpos = exfat_dentries_to_bytes(dentry); return 0; } @@ -465,37 +465,75 @@ static void exfat_free_benign_secondary_clusters(struct inode *inode, return; exfat_chain_set(&dir, start_clu, - EXFAT_B_TO_CLU_ROUND_UP(size, EXFAT_SB(sb)), + exfat_bytes_to_cluster_round_up(EXFAT_SB(sb), size), flags); exfat_free_cluster(inode, &dir); } +/* + * exfat_init_ext_entry - initialize extension entries in a directory entry set + * @es: target entry set + * @num_entries: number of entries excluding benign secondary entries + * @p_uniname: filename to store + * @old_es: optional source entry set with benign secondary entries, or NULL + * @num_extra: number of benign secondary entries to copy from @old_es + * + * Set up the file, stream extension, and filename entries in @es, optionally + * preserving @num_extra benign secondary entries from @old_es. @es and @old_es + * may refer to the same entry set; excess entries are marked as deleted. + */ void exfat_init_ext_entry(struct exfat_entry_set_cache *es, int num_entries, - struct exfat_uni_name *p_uniname) + struct exfat_uni_name *p_uniname, + struct exfat_entry_set_cache *old_es, int num_extra) { - int i; + int i, src_start = 0, old_num; unsigned short *uniname = p_uniname->name; struct exfat_dentry *ep; - es->num_entries = num_entries; + if (WARN_ON(num_extra < 0 || (num_extra && (!old_es || + old_es->num_entries < ES_IDX_FIRST_FILENAME + num_extra)))) + num_extra = 0; + + /* + * Save old entry count and source position before modifying + * es->num_entries, since old_es and es may point to the same + * entry set. + */ + old_num = es->num_entries; + if (old_es && num_extra > 0) + src_start = old_es->num_entries - num_extra; + + es->num_entries = num_entries + num_extra; ep = exfat_get_dentry_cached(es, ES_IDX_FILE); - ep->dentry.file.num_ext = (unsigned char)(num_entries - 1); + ep->dentry.file.num_ext = (unsigned char)(num_entries - 1 + num_extra); ep = exfat_get_dentry_cached(es, ES_IDX_STREAM); ep->dentry.stream.name_len = p_uniname->name_len; ep->dentry.stream.name_hash = cpu_to_le16(p_uniname->name_hash); + if (old_es && num_extra > 0) { + for (i = 0; i < num_extra; i++) + *exfat_get_dentry_cached(es, num_entries + i) = + *exfat_get_dentry_cached(old_es, src_start + i); + } + for (i = ES_IDX_FIRST_FILENAME; i < num_entries; i++) { ep = exfat_get_dentry_cached(es, i); exfat_init_name_entry(ep, uniname); uniname += EXFAT_FILE_NAME_LEN; } + /* Mark excess old entries as deleted (in-place shrink) */ + for (i = num_entries + num_extra; i < old_num; i++) { + ep = exfat_get_dentry_cached(es, i); + exfat_set_entry_type(ep, TYPE_DELETED); + } + exfat_update_dir_chksum(es); } void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, - int order) + int order, bool free_benign) { int i; struct exfat_dentry *ep; @@ -503,7 +541,7 @@ void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, for (i = order; i < es->num_entries; i++) { ep = exfat_get_dentry_cached(es, i); - if (exfat_get_entry_type(ep) & TYPE_BENIGN_SEC) + if (free_benign && (exfat_get_entry_type(ep) & TYPE_BENIGN_SEC)) exfat_free_benign_secondary_clusters(inode, ep); exfat_set_entry_type(ep, TYPE_DELETED); @@ -556,10 +594,11 @@ static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir unsigned int off, clu = 0; struct exfat_sb_info *sbi = EXFAT_SB(sb); - off = EXFAT_DEN_TO_B(entry); + off = exfat_dentries_to_bytes(entry); clu = p_dir->dir; - ret = exfat_cluster_walk(sb, &clu, EXFAT_B_TO_CLU(off, sbi), p_dir->flags); + ret = exfat_cluster_walk(sb, &clu, exfat_bytes_to_cluster(sbi, off), + p_dir->flags); if (ret) return ret; @@ -567,7 +606,7 @@ static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir exfat_fs_error(sb, "unexpected early break in cluster chain (clu : %u, len : %d)", p_dir->dir, - EXFAT_B_TO_CLU(off, sbi)); + exfat_bytes_to_cluster(sbi, off)); return -EIO; } @@ -577,13 +616,13 @@ static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir } /* byte offset in cluster */ - off = EXFAT_CLU_OFFSET(off, sbi); + off = exfat_cluster_offset(sbi, off); /* byte offset in sector */ - *offset = EXFAT_BLK_OFFSET(off, sb); + *offset = exfat_block_offset(sb, off); /* sector offset in cluster */ - *sector = EXFAT_B_TO_BLK(off, sb); + *sector = exfat_bytes_to_block(sb, off); *sector += exfat_cluster_to_sector(sbi, clu); return 0; } @@ -593,7 +632,7 @@ struct exfat_dentry *exfat_get_dentry(struct super_block *sb, { struct exfat_sb_info *sbi = EXFAT_SB(sb); unsigned int sect_per_clus = sbi->sect_per_clus; - unsigned int dentries_per_page = EXFAT_B_TO_DEN(PAGE_SIZE); + unsigned int dentries_per_page = exfat_bytes_to_dentries(PAGE_SIZE); int off; sector_t sec; @@ -672,8 +711,8 @@ struct exfat_dentry *exfat_get_dentry_cached( struct exfat_entry_set_cache *es, int num) { int off = es->start_off + num * DENTRY_SIZE; - struct buffer_head *bh = es->bh[EXFAT_B_TO_BLK(off, es->sb)]; - char *p = bh->b_data + EXFAT_BLK_OFFSET(off, es->sb); + struct buffer_head *bh = es->bh[exfat_bytes_to_block(es->sb, off)]; + char *p = bh->b_data + exfat_block_offset(es->sb, off); return (struct exfat_dentry *)p; } @@ -741,7 +780,7 @@ static int __exfat_get_dentry_set(struct exfat_entry_set_cache *es, es->num_entries = num_entries; - num_bh = EXFAT_B_TO_BLK_ROUND_UP(off + num_entries * DENTRY_SIZE, sb); + num_bh = exfat_bytes_to_block_round_up(sb, off + num_entries * DENTRY_SIZE); if (num_bh > ARRAY_SIZE(es->__bh)) { es->bh = kmalloc_objs(*es->bh, num_bh, GFP_NOFS); if (!es->bh) { @@ -830,7 +869,7 @@ static int exfat_validate_empty_dentry_set(struct exfat_entry_set_cache *es) err_used_follow_unused: off = es->start_off + (i << DENTRY_SIZE_BITS); - bh = es->bh[EXFAT_B_TO_BLK(off, es->sb)]; + bh = es->bh[exfat_bytes_to_block(es->sb, off)]; exfat_fs_error(es->sb, "in sector %lld, dentry %d should be unused, but 0x%x", @@ -839,7 +878,8 @@ static int exfat_validate_empty_dentry_set(struct exfat_entry_set_cache *es) return -EIO; count_skip_entries: - es->num_entries = EXFAT_B_TO_DEN(EXFAT_BLK_TO_B(es->num_bh, es->sb) - es->start_off); + es->num_entries = + exfat_bytes_to_dentries(exfat_block_to_bytes(es->sb, es->num_bh) - es->start_off); for (; i < es->num_entries; i++) { ep = exfat_get_dentry_cached(es, i); if (IS_EXFAT_DELETED(ep->type)) @@ -892,7 +932,7 @@ static inline void exfat_set_empty_hint(struct exfat_inode_info *ei, { if (ei->hint_femp.eidx == EXFAT_HINT_NONE || ei->hint_femp.eidx > dentry) { - int total_entries = EXFAT_B_TO_DEN(i_size_read(&ei->vfs_inode)); + int total_entries = exfat_bytes_to_dentries(i_size_read(&ei->vfs_inode)); if (candi_empty->count == 0) { candi_empty->cur = *clu; @@ -1027,22 +1067,26 @@ int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, continue; } - brelse(bh); if (entry_type == TYPE_EXTEND) { unsigned short entry_uniname[16], unichar; + unsigned int offset; if (step != DIRENT_STEP_NAME || name_len >= MAX_NAME_LENGTH) { + brelse(bh); step = DIRENT_STEP_FILE; continue; } - if (++order == 2) - uniname = p_uniname->name; - else - uniname += EXFAT_FILE_NAME_LEN; - + offset = (++order - 2) * EXFAT_FILE_NAME_LEN; len = exfat_extract_uni_name(ep, entry_uniname); + brelse(bh); + if (offset > MAX_NAME_LENGTH || + len > MAX_NAME_LENGTH - offset) { + step = DIRENT_STEP_FILE; + continue; + } + uniname = p_uniname->name + offset; name_len += len; unichar = *(uniname+len); @@ -1061,6 +1105,7 @@ int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, continue; } + brelse(bh); if (entry_type & (TYPE_CRITICAL_SEC | TYPE_BENIGN_SEC)) { if (step == DIRENT_STEP_SECD) { @@ -1215,7 +1260,7 @@ static int exfat_get_volume_label_dentry(struct super_block *sb, es->bh = es->__bh; es->bh[0] = bh; es->num_bh = 1; - es->start_off = EXFAT_DEN_TO_B(i) % sb->s_blocksize; + es->start_off = exfat_dentries_to_bytes(i) % sb->s_blocksize; return 0; } diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h index 89ef5368277f8..c634540e0de62 100644 --- a/fs/exfat/exfat_fs.h +++ b/fs/exfat/exfat_fs.h @@ -12,6 +12,7 @@ #include #include #include +#include #define EXFAT_ROOT_INO 1 @@ -84,38 +85,6 @@ enum { (min_t(blkcnt_t, (sb)->s_bdi->ra_pages, (sb)->s_bdi->io_pages) \ << (PAGE_SHIFT - (sb)->s_blocksize_bits)) -/* - * helpers for cluster size to byte conversion. - */ -#define EXFAT_CLU_TO_B(b, sbi) ((b) << (sbi)->cluster_size_bits) -#define EXFAT_B_TO_CLU(b, sbi) ((b) >> (sbi)->cluster_size_bits) -#define EXFAT_B_TO_CLU_ROUND_UP(b, sbi) \ - (((b - 1) >> (sbi)->cluster_size_bits) + 1) -#define EXFAT_CLU_OFFSET(off, sbi) ((off) & ((sbi)->cluster_size - 1)) - -/* - * helpers for block size to byte conversion. - */ -#define EXFAT_BLK_TO_B(b, sb) ((b) << (sb)->s_blocksize_bits) -#define EXFAT_B_TO_BLK(b, sb) ((b) >> (sb)->s_blocksize_bits) -#define EXFAT_B_TO_BLK_ROUND_UP(b, sb) \ - (((b - 1) >> (sb)->s_blocksize_bits) + 1) -#define EXFAT_BLK_OFFSET(off, sb) ((off) & ((sb)->s_blocksize - 1)) - -/* - * helpers for block size to dentry size conversion. - */ -#define EXFAT_B_TO_DEN(b) ((b) >> DENTRY_SIZE_BITS) -#define EXFAT_DEN_TO_B(b) ((b) << DENTRY_SIZE_BITS) - -/* - * helpers for cluster size to dentry size conversion. - */ -#define EXFAT_CLU_TO_DEN(clu, sbi) \ - ((clu) << ((sbi)->cluster_size_bits - DENTRY_SIZE_BITS)) -#define EXFAT_DEN_TO_CLU(dentry, sbi) \ - ((dentry) >> ((sbi)->cluster_size_bits - DENTRY_SIZE_BITS)) - /* * helpers for fat entry. */ @@ -149,7 +118,7 @@ enum { * The 608 bytes are in 3 sectors at most (even 512 Byte sector). */ #define DIR_CACHE_SIZE \ - (DIV_ROUND_UP(EXFAT_DEN_TO_B(ES_MAX_ENTRY_NUM), SECTOR_SIZE) + 1) + (DIV_ROUND_UP(ES_MAX_ENTRY_NUM << DENTRY_SIZE_BITS, SECTOR_SIZE) + 1) /* Superblock flags */ #define EXFAT_FLAGS_SHUTDOWN 1 @@ -259,6 +228,7 @@ struct exfat_sb_info { unsigned long long FAT1_start_sector; /* FAT1 start sector */ unsigned long long FAT2_start_sector; /* FAT2 start sector */ unsigned long long data_start_sector; /* data area start sector */ + unsigned long long data_start_bytes; unsigned int num_FAT_sectors; /* num of FAT sectors */ unsigned int root_dir; /* root dir cluster */ unsigned int dentries_per_clu; /* num of dentries per cluster */ @@ -432,6 +402,101 @@ static inline loff_t exfat_ondisk_size(const struct inode *inode) return ((loff_t)inode->i_blocks) << 9; } +static inline loff_t exfat_cluster_to_phys_bytes(struct exfat_sb_info *sbi, + unsigned int clus) +{ + return ((loff_t)(clus - EXFAT_RESERVED_CLUSTERS) << sbi->cluster_size_bits) + + sbi->data_start_bytes; +} + +/* + * helpers for cluster size to byte conversion. + */ +static inline loff_t exfat_cluster_to_bytes(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return (loff_t)nr_clusters << sbi->cluster_size_bits; +} + +static inline blkcnt_t exfat_cluster_to_sectors(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return (blkcnt_t)nr_clusters << (sbi->cluster_size_bits - 9); +} + +static inline u32 exfat_bytes_to_cluster(struct exfat_sb_info *sbi, loff_t size) +{ + return (u32)(size >> sbi->cluster_size_bits); +} + +static inline u32 exfat_bytes_to_cluster_round_up(struct exfat_sb_info *sbi, + loff_t size) +{ + if (size <= 0) + return 0; + return (u32)((size - 1) >> sbi->cluster_size_bits) + 1; +} + +static inline u32 exfat_cluster_offset(struct exfat_sb_info *sbi, loff_t off) +{ + return off & (sbi->cluster_size - 1); +} + +/* + * helpers for block size to byte conversion. + */ +static inline loff_t exfat_block_to_bytes(struct super_block *sb, + sector_t block) +{ + return (loff_t)block << sb->s_blocksize_bits; +} + +static inline sector_t exfat_bytes_to_block(struct super_block *sb, loff_t size) +{ + return (sector_t)(size >> sb->s_blocksize_bits); +} + +static inline sector_t exfat_bytes_to_block_round_up(struct super_block *sb, + loff_t size) +{ + if (size <= 0) + return 0; + return (sector_t)(((size - 1) >> sb->s_blocksize_bits) + 1); +} + +static inline u32 exfat_block_offset(struct super_block *sb, loff_t off) +{ + return (u32)(off & (sb->s_blocksize - 1)); +} + +/* + * helpers for block size to dentry size conversion. + */ +static inline u32 exfat_bytes_to_dentries(loff_t b) +{ + return (u32)(b >> DENTRY_SIZE_BITS); +} + +static inline u32 exfat_dentries_to_bytes(u32 dentry) +{ + return dentry << DENTRY_SIZE_BITS; +} + +/* + * helpers for cluster size to dentry size conversion. + */ +static inline u32 exfat_cluster_to_dentries(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return nr_clusters << (sbi->cluster_size_bits - DENTRY_SIZE_BITS); +} + +static inline u32 exfat_dentries_to_cluster(struct exfat_sb_info *sbi, + u32 dentry) +{ + return dentry >> (sbi->cluster_size_bits - DENTRY_SIZE_BITS); +} + /* super.c */ int exfat_set_volume_dirty(struct super_block *sb); int exfat_clear_volume_dirty(struct super_block *sb); @@ -524,9 +589,10 @@ void exfat_init_dir_entry(struct exfat_entry_set_cache *es, unsigned int type, unsigned int start_clu, unsigned long long size, struct timespec64 *ts); void exfat_init_ext_entry(struct exfat_entry_set_cache *es, int num_entries, - struct exfat_uni_name *p_uniname); + struct exfat_uni_name *p_uniname, + struct exfat_entry_set_cache *old_es, int num_extra); void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, - int order); + int order, bool free_benign); void exfat_update_dir_chksum(struct exfat_entry_set_cache *es); int exfat_calc_num_entries(struct exfat_uni_name *p_uniname); int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c index dce0955e689aa..45b0b754a2e4b 100644 --- a/fs/exfat/fatent.c +++ b/fs/exfat/fatent.c @@ -412,8 +412,8 @@ int exfat_zeroed_cluster(struct inode *dir, unsigned int clu) if (IS_DIRSYNC(dir)) return sync_blockdev_range(sb->s_bdev, - EXFAT_BLK_TO_B(blknr, sb), - EXFAT_BLK_TO_B(last_blknr, sb) - 1); + exfat_block_to_bytes(sb, blknr), + exfat_block_to_bytes(sb, last_blknr) - 1); return 0; } diff --git a/fs/exfat/file.c b/fs/exfat/file.c index 354bdcfe4abcd..29a36a80e29bc 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -33,9 +33,9 @@ static int exfat_cont_expand(struct inode *inode, loff_t size) if (ret) return ret; - num_clusters = EXFAT_B_TO_CLU(exfat_ondisk_size(inode), sbi); + num_clusters = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); /* integer overflow is already checked in inode_newsize_ok(). */ - new_num_clusters = EXFAT_B_TO_CLU_ROUND_UP(size, sbi); + new_num_clusters = exfat_bytes_to_cluster_round_up(sbi, size); if (new_num_clusters == num_clusters) goto out; @@ -200,8 +200,8 @@ int __exfat_truncate(struct inode *inode) exfat_set_volume_dirty(sb); - num_clusters_new = EXFAT_B_TO_CLU_ROUND_UP(i_size_read(inode), sbi); - num_clusters_phys = EXFAT_B_TO_CLU(exfat_ondisk_size(inode), sbi); + num_clusters_new = exfat_bytes_to_cluster_round_up(sbi, i_size_read(inode)); + num_clusters_phys = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); exfat_chain_set(&clu, ei->start_clu, num_clusters_phys, ei->flags); diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c index 1ea4c740fef9e..a10d4f3c66a17 100644 --- a/fs/exfat/inode.c +++ b/fs/exfat/inode.c @@ -124,7 +124,8 @@ void exfat_sync_inode(struct inode *inode) * *clu = (~0), if it's unable to allocate a new cluster */ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, - unsigned int *clu, unsigned int *count, int create) + unsigned int *clu, unsigned int *count, int create, + bool *balloc) { int ret; unsigned int last_clu; @@ -135,7 +136,7 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, unsigned int local_clu_offset = clu_offset; unsigned int num_to_be_allocated = 0, num_clusters; - num_clusters = EXFAT_B_TO_CLU(exfat_ondisk_size(inode), sbi); + num_clusters = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); if (clu_offset >= num_clusters) num_to_be_allocated = clu_offset - num_clusters + 1; @@ -216,7 +217,8 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, *clu = new_clu.dir; - inode->i_blocks += EXFAT_CLU_TO_B(num_to_be_allocated, sbi) >> 9; + inode->i_blocks += + exfat_cluster_to_sectors(sbi, num_to_be_allocated); /* * Move *clu pointer along FAT chains (hole care) because the @@ -228,6 +230,8 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, if (exfat_cluster_walk(sb, clu, num_to_be_allocated - 1, ei->flags)) return -EIO; *count = 1; + if (balloc) + *balloc = true; } /* hint information */ @@ -254,14 +258,14 @@ static int exfat_get_block(struct inode *inode, sector_t iblock, mutex_lock(&sbi->s_lock); i_size = i_size_read(inode); - last_block = EXFAT_B_TO_BLK_ROUND_UP(i_size, sb); + last_block = exfat_bytes_to_block_round_up(sb, i_size); if (iblock >= last_block && !create) goto done; /* Is this block already allocated? */ - count = EXFAT_B_TO_CLU_ROUND_UP(bh_result->b_size, sbi); + count = exfat_bytes_to_cluster_round_up(sbi, bh_result->b_size); err = exfat_map_cluster(inode, iblock >> sbi->sect_per_clus_bits, - &cluster, &count, create); + &cluster, &count, create, NULL); if (err) { if (err != -ENOSPC) exfat_fs_error_ratelimit(sb, @@ -296,9 +300,9 @@ static int exfat_get_block(struct inode *inode, sector_t iblock, * care the last nested block if valid_size is not equal to i_size. */ if (i_size == ei->valid_size || create || !bh_result->b_folio) - valid_blks = EXFAT_B_TO_BLK_ROUND_UP(ei->valid_size, sb); + valid_blks = exfat_bytes_to_block_round_up(sb, ei->valid_size); else - valid_blks = EXFAT_B_TO_BLK(ei->valid_size, sb); + valid_blks = exfat_bytes_to_block(sb, ei->valid_size); /* The range has been fully written, map it */ if (iblock + max_blocks < valid_blks) @@ -313,7 +317,7 @@ static int exfat_get_block(struct inode *inode, sector_t iblock, /* The area has not been written, map and mark as new for create case */ if (create) { set_buffer_new(bh_result); - ei->valid_size = EXFAT_BLK_TO_B(iblock + max_blocks, sb); + ei->valid_size = exfat_block_to_bytes(sb, iblock + max_blocks); mark_inode_dirty(inode); goto done; } @@ -343,7 +347,7 @@ static int exfat_get_block(struct inode *inode, sector_t iblock, goto done; } - pos = EXFAT_BLK_TO_B(iblock, sb); + pos = exfat_block_to_bytes(sb, iblock); size = ei->valid_size - pos; addr = folio_address(bh_result->b_folio) + offset_in_folio(bh_result->b_folio, pos); @@ -374,7 +378,7 @@ static int exfat_get_block(struct inode *inode, sector_t iblock, */ clear_buffer_mapped(bh_result); done: - bh_result->b_size = EXFAT_BLK_TO_B(max_blocks, sb); + bh_result->b_size = exfat_block_to_bytes(sb, max_blocks); if (err < 0) clear_buffer_mapped(bh_result); unlock_ret: diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c index 2c5636634b4a4..3191b88113e3b 100644 --- a/fs/exfat/namei.c +++ b/fs/exfat/namei.c @@ -208,7 +208,7 @@ static int exfat_search_empty_slot(struct super_block *sb, int dentries_per_clu; struct exfat_chain clu; struct exfat_sb_info *sbi = EXFAT_SB(sb); - int total_entries = EXFAT_CLU_TO_DEN(p_dir->size, sbi); + unsigned int total_entries = exfat_cluster_to_dentries(sbi, p_dir->size); dentries_per_clu = sbi->dentries_per_clu; @@ -266,7 +266,7 @@ static int exfat_search_empty_slot(struct super_block *sb, static int exfat_check_max_dentries(struct inode *inode) { - if (EXFAT_B_TO_DEN(i_size_read(inode)) >= MAX_EXFAT_DENTRIES) { + if (exfat_bytes_to_dentries(i_size_read(inode)) >= MAX_EXFAT_DENTRIES) { /* * exFAT spec allows a dir to grow up to 8388608(256MB) * dentries @@ -314,7 +314,8 @@ int exfat_find_empty_entry(struct inode *inode, } exfat_chain_set(p_dir, ei->start_clu, - EXFAT_B_TO_CLU(i_size_read(inode), sbi), ei->flags); + exfat_bytes_to_cluster(sbi, i_size_read(inode)), + ei->flags); while ((dentry = exfat_search_empty_slot(sb, &hint_femp, p_dir, num_entries, es)) < 0) { @@ -375,7 +376,7 @@ int exfat_find_empty_entry(struct inode *inode, hint_femp.cur.size++; p_dir->size++; - size = EXFAT_CLU_TO_B(p_dir->size, sbi); + size = exfat_cluster_to_bytes(sbi, p_dir->size); /* directory inode should be updated in here */ i_size_write(inode, size); @@ -503,7 +504,7 @@ static int exfat_add_entry(struct inode *inode, const char *path, * the first cluster is not determined yet. (0) */ exfat_init_dir_entry(&es, type, start_clu, clu_size, &ts); - exfat_init_ext_entry(&es, num_entries, &uniname); + exfat_init_ext_entry(&es, num_entries, &uniname, NULL, 0); ret = exfat_put_dentry_set(&es, IS_DIRSYNC(inode)); if (ret) @@ -604,7 +605,7 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, return ret; exfat_chain_set(&cdir, ei->start_clu, - EXFAT_B_TO_CLU(i_size_read(dir), sbi), ei->flags); + exfat_bytes_to_cluster(sbi, i_size_read(dir)), ei->flags); /* check the validation of hint_stat and initialize it if required */ if (ei->version != (inode_peek_iversion_raw(dir) & 0xffffffff)) { @@ -681,7 +682,7 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, return -EIO; } - if (unlikely(EXFAT_B_TO_CLU_ROUND_UP(info->size, sbi) > sbi->used_clusters)) { + if (unlikely(exfat_bytes_to_cluster_round_up(sbi, info->size) > sbi->used_clusters)) { exfat_fs_error(sb, "data size is invalid(%lld)", info->size); return -EIO; } @@ -695,7 +696,8 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, if (info->type == TYPE_DIR) { exfat_chain_set(&cdir, info->start_clu, - EXFAT_B_TO_CLU(info->size, sbi), info->flags); + exfat_bytes_to_cluster(sbi, info->size), + info->flags); count = exfat_count_dir_entries(sb, &cdir); if (count < 0) return -EIO; @@ -814,7 +816,7 @@ static int exfat_unlink(struct inode *dir, struct dentry *dentry) exfat_set_volume_dirty(sb); /* update the directory entry */ - exfat_remove_entries(inode, &es, ES_IDX_FILE); + exfat_remove_entries(inode, &es, ES_IDX_FILE, true); err = exfat_put_dentry_set(&es, IS_DIRSYNC(inode)); if (err) @@ -951,7 +953,7 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) } exfat_chain_set(&clu_to_free, ei->start_clu, - EXFAT_B_TO_CLU_ROUND_UP(i_size_read(inode), sbi), ei->flags); + exfat_bytes_to_cluster_round_up(sbi, i_size_read(inode)), ei->flags); err = exfat_check_dir_empty(sb, &clu_to_free); if (err) { @@ -969,7 +971,7 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) exfat_set_volume_dirty(sb); - exfat_remove_entries(inode, &es, ES_IDX_FILE); + exfat_remove_entries(inode, &es, ES_IDX_FILE, true); err = exfat_put_dentry_set(&es, IS_DIRSYNC(dir)); if (err) @@ -996,6 +998,23 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) return err; } +/* + * Count benign secondary entries beyond the filename entries. + * Returns the count, or -EIO if the entry set is inconsistent. + */ +static int exfat_count_extra_entries(struct exfat_entry_set_cache *es) +{ + struct exfat_dentry *stream; + unsigned int name_entries; + int extra; + + stream = exfat_get_dentry_cached(es, ES_IDX_STREAM); + name_entries = EXFAT_FILENAME_ENTRY_NUM(stream->dentry.stream.name_len); + extra = es->num_entries - (ES_IDX_FIRST_FILENAME + name_entries); + + return extra >= 0 ? extra : -EIO; +} + static int exfat_rename_file(struct inode *parent_inode, struct exfat_uni_name *p_uniname, struct exfat_inode_info *ei) { @@ -1004,6 +1023,7 @@ static int exfat_rename_file(struct inode *parent_inode, struct super_block *sb = parent_inode->i_sb; struct exfat_entry_set_cache old_es, new_es; int sync = IS_DIRSYNC(parent_inode); + unsigned int num_extra_entries, num_total_entries; if (unlikely(exfat_forced_shutdown(sb))) return -EIO; @@ -1013,19 +1033,23 @@ static int exfat_rename_file(struct inode *parent_inode, return num_new_entries; ret = exfat_get_dentry_set_by_ei(&old_es, sb, ei); - if (ret) { - ret = -EIO; - return ret; - } + if (ret) + return -EIO; epold = exfat_get_dentry_cached(&old_es, ES_IDX_FILE); - if (old_es.num_entries < num_new_entries) { + ret = exfat_count_extra_entries(&old_es); + if (ret < 0) + goto put_old_es; + num_extra_entries = ret; + num_total_entries = num_new_entries + num_extra_entries; + + if (old_es.num_entries < num_total_entries) { int newentry; struct exfat_chain dir; newentry = exfat_find_empty_entry(parent_inode, &dir, - num_new_entries, &new_es); + num_total_entries, &new_es); if (newentry < 0) { ret = newentry; /* -EIO or -ENOSPC */ goto put_old_es; @@ -1042,13 +1066,23 @@ static int exfat_rename_file(struct inode *parent_inode, epnew = exfat_get_dentry_cached(&new_es, ES_IDX_STREAM); *epnew = *epold; - exfat_init_ext_entry(&new_es, num_new_entries, p_uniname); + exfat_init_ext_entry(&new_es, num_new_entries, p_uniname, + &old_es, num_extra_entries); ret = exfat_put_dentry_set(&new_es, sync); - if (ret) + if (ret) { + /* Best-effort delete to avoid duplicate entries */ + if (!exfat_get_dentry_set(&new_es, sb, + &dir, newentry, + ES_ALL_ENTRIES)) { + exfat_remove_entries(parent_inode, &new_es, + ES_IDX_FILE, false); + exfat_put_dentry_set(&new_es, false); + } goto put_old_es; + } - exfat_remove_entries(parent_inode, &old_es, ES_IDX_FILE); + exfat_remove_entries(parent_inode, &old_es, ES_IDX_FILE, false); ei->dir = dir; ei->entry = newentry; } else { @@ -1057,8 +1091,8 @@ static int exfat_rename_file(struct inode *parent_inode, ei->attr |= EXFAT_ATTR_ARCHIVE; } - exfat_remove_entries(parent_inode, &old_es, ES_IDX_FIRST_FILENAME + 1); - exfat_init_ext_entry(&old_es, num_new_entries, p_uniname); + exfat_init_ext_entry(&old_es, num_new_entries, p_uniname, + &old_es, num_extra_entries); } return exfat_put_dentry_set(&old_es, sync); @@ -1074,6 +1108,7 @@ static int exfat_move_file(struct inode *parent_inode, struct exfat_dentry *epmov, *epnew; struct exfat_entry_set_cache mov_es, new_es; struct exfat_chain newdir; + unsigned int num_extra_entries, num_total_entries; num_new_entries = exfat_calc_num_entries(p_uniname); if (num_new_entries < 0) @@ -1083,8 +1118,14 @@ static int exfat_move_file(struct inode *parent_inode, if (ret) return -EIO; + ret = exfat_count_extra_entries(&mov_es); + if (ret < 0) + goto put_mov_es; + num_extra_entries = ret; + num_total_entries = num_new_entries + num_extra_entries; + newentry = exfat_find_empty_entry(parent_inode, &newdir, - num_new_entries, &new_es); + num_total_entries, &new_es); if (newentry < 0) { ret = newentry; /* -EIO or -ENOSPC */ goto put_mov_es; @@ -1102,21 +1143,31 @@ static int exfat_move_file(struct inode *parent_inode, epnew = exfat_get_dentry_cached(&new_es, ES_IDX_STREAM); *epnew = *epmov; - exfat_init_ext_entry(&new_es, num_new_entries, p_uniname); - exfat_remove_entries(parent_inode, &mov_es, ES_IDX_FILE); + exfat_init_ext_entry(&new_es, num_new_entries, p_uniname, + &mov_es, num_extra_entries); + + exfat_remove_entries(parent_inode, &mov_es, ES_IDX_FILE, false); ei->dir = newdir; ei->entry = newentry; ret = exfat_put_dentry_set(&new_es, IS_DIRSYNC(parent_inode)); - if (ret) + if (ret) { + /* Best-effort delete to avoid duplicate entries */ + if (!exfat_get_dentry_set(&new_es, parent_inode->i_sb, + &newdir, newentry, + ES_ALL_ENTRIES)) { + exfat_remove_entries(parent_inode, &new_es, + ES_IDX_FILE, false); + exfat_put_dentry_set(&new_es, false); + } goto put_mov_es; + } return exfat_put_dentry_set(&mov_es, IS_DIRSYNC(parent_inode)); put_mov_es: exfat_put_dentry_set(&mov_es, false); - return ret; } @@ -1158,8 +1209,8 @@ static int __exfat_rename(struct inode *old_parent_inode, new_clu.dir = new_ei->start_clu; new_clu.size = - EXFAT_B_TO_CLU_ROUND_UP(i_size_read(new_inode), - sbi); + exfat_bytes_to_cluster_round_up(sbi, + i_size_read(new_inode)); new_clu.flags = new_ei->flags; ret = exfat_check_dir_empty(sb, &new_clu); @@ -1190,7 +1241,7 @@ static int __exfat_rename(struct inode *old_parent_inode, goto del_out; } - exfat_remove_entries(new_inode, &es, ES_IDX_FILE); + exfat_remove_entries(new_inode, &es, ES_IDX_FILE, true); ret = exfat_put_dentry_set(&es, IS_DIRSYNC(new_inode)); if (ret) @@ -1203,8 +1254,8 @@ static int __exfat_rename(struct inode *old_parent_inode, struct exfat_chain new_clu_to_free; exfat_chain_set(&new_clu_to_free, new_ei->start_clu, - EXFAT_B_TO_CLU_ROUND_UP(i_size_read(new_inode), - sbi), new_ei->flags); + exfat_bytes_to_cluster_round_up(sbi, i_size_read(new_inode)), + new_ei->flags); if (exfat_free_cluster(new_inode, &new_clu_to_free)) { /* just set I/O error only */ diff --git a/fs/exfat/super.c b/fs/exfat/super.c index 95d87e2d7717f..388db271c6bf5 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -369,7 +369,7 @@ static int exfat_read_root(struct inode *inode, struct exfat_chain *root_clu) ei->hint_stat.clu = sbi->root_dir; ei->hint_femp.eidx = EXFAT_HINT_NONE; - i_size_write(inode, EXFAT_CLU_TO_B(root_clu->size, sbi)); + i_size_write(inode, exfat_cluster_to_bytes(sbi, root_clu->size)); num_subdirs = exfat_count_dir_entries(sb, root_clu); if (num_subdirs < 0) @@ -499,6 +499,7 @@ static int exfat_read_boot_sector(struct super_block *sb) if (p_boot->num_fats == 2) sbi->FAT2_start_sector += sbi->num_FAT_sectors; sbi->data_start_sector = le32_to_cpu(p_boot->clu_offset); + sbi->data_start_bytes = sbi->data_start_sector << p_boot->sect_size_bits; sbi->num_sectors = le64_to_cpu(p_boot->vol_length); /* because the cluster index starts with 2 */ sbi->num_clusters = le32_to_cpu(p_boot->clu_count) + @@ -538,7 +539,7 @@ static int exfat_read_boot_sector(struct super_block *sb) * machines. */ sb->s_maxbytes = min(MAX_LFS_FILESIZE, - EXFAT_CLU_TO_B((loff_t)EXFAT_MAX_NUM_CLUSTER, sbi)); + exfat_cluster_to_bytes(sbi, (loff_t)EXFAT_MAX_NUM_CLUSTER)); /* check logical sector size */ if (exfat_calibrate_blocksize(sb, 1 << p_boot->sect_size_bits)) diff --git a/fs/ext2/file.c b/fs/ext2/file.c index d9b1eb34694a0..781d227aff155 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -267,12 +267,15 @@ static ssize_t ext2_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) endbyte = pos + status - 1; ret2 = filemap_write_and_wait_range(inode->i_mapping, pos, endbyte); - if (!ret2) + if (!ret2) { invalidate_mapping_pages(inode->i_mapping, pos >> PAGE_SHIFT, endbyte >> PAGE_SHIFT); - if (ret > 0) - generic_write_sync(iocb, ret); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + } else { + ret = ret2; + } } out_unlock: diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 94283a991e5c4..6569d1d575a01 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2000,6 +2000,8 @@ EXT4_INODE_BIT_FNS(flag, flags, 0) static inline int ext4_test_inode_state(struct inode *inode, int bit); static inline void ext4_set_inode_state(struct inode *inode, int bit); static inline void ext4_clear_inode_state(struct inode *inode, int bit); +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode); +static inline int ext4_inode_state_wait_bit(int bit); #if (BITS_PER_LONG < 64) EXT4_INODE_BIT_FNS(state, state_flags, 0) @@ -2015,6 +2017,24 @@ static inline void ext4_clear_state_flags(struct ext4_inode_info *ei) /* We depend on the fact that callers will set i_flags */ } #endif + +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode) +{ +#if (BITS_PER_LONG < 64) + return &EXT4_I(inode)->i_state_flags; +#else + return &EXT4_I(inode)->i_flags; +#endif +} + +static inline int ext4_inode_state_wait_bit(int bit) +{ +#if (BITS_PER_LONG < 64) + return bit; +#else + return bit + 32; +#endif +} #else /* Assume that user mode programs are passing in an ext4fs superblock, not * a kernel struct super_block. This will allow us to call the feature-test diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index b3c22636251dd..1775bce9649a9 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -239,6 +239,8 @@ void ext4_fc_del(struct inode *inode) struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_dentry_update *fc_dentry; wait_queue_head_t *wq; + unsigned long *wait_word = ext4_inode_state_wait_word(inode); + int wait_bit = ext4_inode_state_wait_bit(EXT4_STATE_FC_FLUSHING_DATA); int alloc_ctx; if (ext4_fc_disabled(inode->i_sb)) @@ -268,17 +270,9 @@ void ext4_fc_del(struct inode *inode) WARN_ON(ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING) && !ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)); while (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { -#if (BITS_PER_LONG < 64) - DEFINE_WAIT_BIT(wait, &ei->i_state_flags, - EXT4_STATE_FC_FLUSHING_DATA); - wq = bit_waitqueue(&ei->i_state_flags, - EXT4_STATE_FC_FLUSHING_DATA); -#else - DEFINE_WAIT_BIT(wait, &ei->i_flags, - EXT4_STATE_FC_FLUSHING_DATA); - wq = bit_waitqueue(&ei->i_flags, - EXT4_STATE_FC_FLUSHING_DATA); -#endif + DEFINE_WAIT_BIT(wait, wait_word, wait_bit); + + wq = bit_waitqueue(wait_word, wait_bit); prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); if (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { ext4_fc_unlock(inode->i_sb, alloc_ctx); @@ -542,6 +536,8 @@ void ext4_fc_track_inode(handle_t *handle, struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); wait_queue_head_t *wq; + unsigned long *wait_word = ext4_inode_state_wait_word(inode); + int wait_bit = ext4_inode_state_wait_bit(EXT4_STATE_FC_COMMITTING); int ret; if (S_ISDIR(inode->i_mode)) @@ -564,17 +560,9 @@ void ext4_fc_track_inode(handle_t *handle, struct inode *inode) lockdep_assert_not_held(&ei->i_data_sem); while (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) { -#if (BITS_PER_LONG < 64) - DEFINE_WAIT_BIT(wait, &ei->i_state_flags, - EXT4_STATE_FC_COMMITTING); - wq = bit_waitqueue(&ei->i_state_flags, - EXT4_STATE_FC_COMMITTING); -#else - DEFINE_WAIT_BIT(wait, &ei->i_flags, - EXT4_STATE_FC_COMMITTING); - wq = bit_waitqueue(&ei->i_flags, - EXT4_STATE_FC_COMMITTING); -#endif + DEFINE_WAIT_BIT(wait, wait_word, wait_bit); + + wq = bit_waitqueue(wait_word, wait_bit); prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) schedule(); @@ -1034,6 +1022,8 @@ static int ext4_fc_perform_commit(journal_t *journal) int ret = 0; u32 crc = 0; int alloc_ctx; + int flushing_wait_bit = + ext4_inode_state_wait_bit(EXT4_STATE_FC_FLUSHING_DATA); /* * Step 1: Mark all inodes on s_fc_q[MAIN] with @@ -1059,11 +1049,8 @@ static int ext4_fc_perform_commit(journal_t *journal) list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); -#if (BITS_PER_LONG < 64) - wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_FLUSHING_DATA); -#else - wake_up_bit(&iter->i_flags, EXT4_STATE_FC_FLUSHING_DATA); -#endif + wake_up_bit(ext4_inode_state_wait_word(&iter->vfs_inode), + flushing_wait_bit); } /* @@ -1279,6 +1266,8 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) struct ext4_inode_info *ei; struct ext4_fc_dentry_update *fc_dentry; int alloc_ctx; + int committing_wait_bit = + ext4_inode_state_wait_bit(EXT4_STATE_FC_COMMITTING); if (full && sbi->s_fc_bh) sbi->s_fc_bh = NULL; @@ -1315,11 +1304,8 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) * barrier in prepare_to_wait() in ext4_fc_track_inode(). */ smp_mb(); -#if (BITS_PER_LONG < 64) - wake_up_bit(&ei->i_state_flags, EXT4_STATE_FC_COMMITTING); -#else - wake_up_bit(&ei->i_flags, EXT4_STATE_FC_COMMITTING); -#endif + wake_up_bit(ext4_inode_state_wait_word(&ei->vfs_inode), + committing_wait_bit); } while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index c2c2d6ac7f3d1..4fce9ec176f88 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1560,7 +1560,8 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode)) + if (ext4_has_inline_data(inode) && + ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) return ext4_write_inline_data_end(inode, pos, len, copied, folio); diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 1d0c3d4bdf472..c8387e6a2c6e9 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -830,11 +830,17 @@ int ext4_force_shutdown(struct super_block *sb, u32 flags) bdev_thaw(sb->s_bdev); break; case EXT4_GOING_FLAGS_LOGFLUSH: + /* + * Call ext4_force_commit() before setting EXT4_FLAGS_SHUTDOWN. + * This is because in data=ordered mode, journal commit + * triggers data writeback which fails if shutdown is already + * set, causing the journal to be aborted prematurely before + * the commit succeeds. + */ + (void) ext4_force_commit(sb); set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); - if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) { - (void) ext4_force_commit(sb); + if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) jbd2_journal_abort(sbi->s_journal, -ESHUTDOWN); - } break; case EXT4_GOING_FLAGS_NOLOGFLUSH: set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); @@ -1650,6 +1656,9 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (!(fd_file(donor)->f_mode & FMODE_WRITE)) return -EBADF; + if (file_inode(filp)->i_sb != file_inode(fd_file(donor))->i_sb) + return -EXDEV; + err = mnt_want_write_file(filp); if (err) return err; diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 4a47fbd8dd30c..8cadaeb15b2bd 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -3054,7 +3054,7 @@ static struct dentry *ext4_mkdir(struct mnt_idmap *idmap, struct inode *dir, out_retry: if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) goto retry; - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; } /* diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index fa8d81a30fb91..d3253549173e6 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -47,6 +47,7 @@ static inline int f2fs_acl_count(size_t size) static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) { int i, count; + int err = -EINVAL; struct posix_acl *acl; struct f2fs_acl_header *hdr = (struct f2fs_acl_header *)value; struct f2fs_acl_entry *entry = (struct f2fs_acl_entry *)(hdr + 1); @@ -70,8 +71,11 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) for (i = 0; i < count; i++) { - if ((char *)entry > end) + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry_short) > end)) { + err = -EFSCORRUPTED; goto fail; + } acl->a_entries[i].e_tag = le16_to_cpu(entry->e_tag); acl->a_entries[i].e_perm = le16_to_cpu(entry->e_perm); @@ -86,6 +90,11 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) break; case ACL_USER: + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry) > end)) { + err = -EFSCORRUPTED; + goto fail; + } acl->a_entries[i].e_uid = make_kuid(&init_user_ns, le32_to_cpu(entry->e_id)); @@ -93,6 +102,11 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) sizeof(struct f2fs_acl_entry)); break; case ACL_GROUP: + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry) > end)) { + err = -EFSCORRUPTED; + goto fail; + } acl->a_entries[i].e_gid = make_kgid(&init_user_ns, le32_to_cpu(entry->e_id)); @@ -108,7 +122,7 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) return acl; fail: posix_acl_release(acl); - return ERR_PTR(-EINVAL); + return ERR_PTR(err); } static void *f2fs_acl_to_disk(struct f2fs_sb_info *sbi, diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index c00a6b6ebcbd8..064f5b5374237 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -943,6 +943,7 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi) for (i = 0; i < orphan_blocks; i++) { struct folio *folio; struct f2fs_orphan_block *orphan_blk; + unsigned int entry_count; folio = f2fs_get_meta_folio(sbi, start_blk + i); if (IS_ERR(folio)) { @@ -951,7 +952,18 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi) } orphan_blk = folio_address(folio); - for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) { + entry_count = le32_to_cpu(orphan_blk->entry_count); + if (entry_count > F2FS_ORPHANS_PER_BLOCK) { + f2fs_err(sbi, "invalid orphan inode entry count %u", + entry_count); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_ORPHAN); + err = -EFSCORRUPTED; + f2fs_folio_put(folio, true); + goto out; + } + + for (j = 0; j < entry_count; j++) { nid_t ino = le32_to_cpu(orphan_blk->ino[j]); err = recover_orphan_inode(sbi, ino); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 8d4f1e75dee3e..b5825726743f5 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -943,6 +943,35 @@ void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, } } +void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi) +{ + struct bio_entry *be, *tmp; + struct f2fs_bio_info *io; + enum temp_type temp; + + for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { + LIST_HEAD(list); + + io = sbi->write_io[DATA] + temp; + + /* A lockless list_empty() check is safe here: any bios from + * other kworkers that we miss will be submitted by those + * kworkers accordingly. + */ + if (list_empty(&io->bio_list)) + continue; + + f2fs_down_write(&io->bio_list_lock); + list_splice_init(&io->bio_list, &list); + f2fs_up_write(&io->bio_list_lock); + + list_for_each_entry_safe(be, tmp, &list, list) { + f2fs_submit_write_bio(sbi, be->bio, DATA); + del_bio_entry(be); + } + } +} + int f2fs_merge_page_bio(struct f2fs_io_info *fio) { struct bio *bio = *fio->bio; @@ -2495,7 +2524,7 @@ static int f2fs_read_data_large_folio(struct inode *inode, unsigned nrpages; struct f2fs_folio_state *ffs; int ret = 0; - bool folio_in_bio; + bool folio_in_bio = false; if (!IS_IMMUTABLE(inode) || f2fs_compressed_file(inode)) { if (folio) @@ -2611,18 +2640,17 @@ static int f2fs_read_data_large_folio(struct inode *inode, } trace_f2fs_read_folio(folio, DATA); err_out: - if (!folio_in_bio) { + if (!folio_in_bio) folio_end_read(folio, !ret); - if (ret) - return ret; - } + if (ret) + goto out; if (rac) { folio = readahead_folio(rac); goto next_folio; } out: f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); - if (ret) { + if (ret && folio_in_bio) { /* Wait bios and clear uptodate. */ folio_lock(folio); folio_clear_uptodate(folio); @@ -3822,13 +3850,14 @@ static int __reserve_data_block(struct inode *inode, pgoff_t index, static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, struct folio *folio, loff_t pos, unsigned int len, - block_t *blk_addr, bool *node_changed, bool *use_cow) + block_t *blk_addr, bool *node_changed) { struct inode *inode = folio->mapping->host; struct inode *cow_inode = F2FS_I(inode)->cow_inode; pgoff_t index = folio->index; int err = 0; block_t ori_blk_addr = NULL_ADDR; + bool cow_has_reserved_block = false; /* If pos is beyond the end of file, reserve a new block in COW inode */ if ((pos & PAGE_MASK) >= i_size_read(inode)) @@ -3836,12 +3865,14 @@ static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, /* Look for the block in COW inode first */ err = __find_data_block(cow_inode, index, blk_addr); - if (err) { + if (err) return err; - } else if (*blk_addr != NULL_ADDR) { - *use_cow = true; + + if (__is_valid_data_blkaddr(*blk_addr)) return 0; - } + + if (*blk_addr == NEW_ADDR) + cow_has_reserved_block = true; if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE)) goto reserve_block; @@ -3853,10 +3884,13 @@ static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, reserve_block: /* Finally, we should reserve a new block in COW inode for the update */ - err = __reserve_data_block(cow_inode, index, blk_addr, node_changed); - if (err) - return err; - inc_atomic_write_cnt(inode); + if (!cow_has_reserved_block) { + err = __reserve_data_block(cow_inode, index, blk_addr, + node_changed); + if (err) + return err; + inc_atomic_write_cnt(inode); + } if (ori_blk_addr != NULL_ADDR) *blk_addr = ori_blk_addr; @@ -3873,7 +3907,6 @@ static int f2fs_write_begin(const struct kiocb *iocb, struct folio *folio; pgoff_t index = pos >> PAGE_SHIFT; bool need_balance = false; - bool use_cow = false; block_t blkaddr = NULL_ADDR; int err = 0; @@ -3936,7 +3969,7 @@ static int f2fs_write_begin(const struct kiocb *iocb, if (f2fs_is_atomic_file(inode)) err = prepare_atomic_write_begin(sbi, folio, pos, len, - &blkaddr, &need_balance, &use_cow); + &blkaddr, &need_balance); else err = prepare_write_begin(sbi, folio, pos, len, &blkaddr, &need_balance); @@ -3976,8 +4009,15 @@ static int f2fs_write_begin(const struct kiocb *iocb, err = -EFSCORRUPTED; goto put_folio; } - f2fs_submit_page_read(use_cow ? F2FS_I(inode)->cow_inode : - inode, + /* + * Although the block may be stored in the COW inode, the folio + * belongs to @inode and its data was encrypted (or not) using + * @inode's context (see f2fs_encrypt_one_page()). Read with + * @inode so the post-read decryption decision matches the + * folio's owner; otherwise an unencrypted @inode whose COW inode + * is encrypted hits a NULL ->i_crypt_info on decryption. + */ + f2fs_submit_page_read(inode, NULL, /* can't write to fsverity files */ folio, blkaddr, 0, true); diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index d2e006420f040..61f6b97143663 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -119,10 +119,9 @@ static bool __may_extent_tree(struct inode *inode, enum extent_type type) if (!__init_may_extent_tree(inode, type)) return false; - if (is_inode_flag_set(inode, FI_NO_EXTENT)) - return false; - if (type == EX_READ) { + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return false; if (is_inode_flag_set(inode, FI_COMPRESSED_FILE) && !f2fs_sb_has_readonly(F2FS_I_SB(inode))) return false; @@ -645,14 +644,10 @@ static unsigned int __destroy_extent_node(struct inode *inode, while (atomic_read(&et->node_cnt)) { write_lock(&et->lock); - if (!is_inode_flag_set(inode, FI_NO_EXTENT)) - set_inode_flag(inode, FI_NO_EXTENT); node_cnt += __free_extent_tree(sbi, et, nr_shrink); write_unlock(&et->lock); } - f2fs_bug_on(sbi, atomic_read(&et->node_cnt)); - return node_cnt; } @@ -691,12 +686,12 @@ static void __update_extent_tree_range(struct inode *inode, write_lock(&et->lock); - if (is_inode_flag_set(inode, FI_NO_EXTENT)) { - write_unlock(&et->lock); - return; - } - if (type == EX_READ) { + if (is_inode_flag_set(inode, FI_NO_EXTENT)) { + write_unlock(&et->lock); + return; + } + prev = et->largest; dei.len = 0; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 91f506e7c9cfb..935dd37430323 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1583,6 +1583,7 @@ enum node_type { NODE_TYPE_INODE, NODE_TYPE_XATTR, NODE_TYPE_NON_INODE, + NODE_TYPE_NON_IXNODE, /* non inode and xnode */ }; /* a threshold of maximum elapsed time in critical region to print tracepoint */ @@ -4153,6 +4154,7 @@ void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi, struct folio *folio, enum page_type type); void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, struct bio **bio, struct folio *folio); +void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi); void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi); int f2fs_submit_page_bio(struct f2fs_io_info *fio); int f2fs_merge_page_bio(struct f2fs_io_info *fio); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index fb12c5c9affda..ade4fde2ef0eb 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1098,6 +1098,8 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, return -EPERM; if ((attr->ia_valid & ATTR_SIZE)) { + if (mapping_large_folio_support(inode->i_mapping)) + return -EOPNOTSUPP; if (!f2fs_is_compress_backend_ready(inode) || IS_DEVICE_ALIASING(inode)) return -EOPNOTSUPP; @@ -1914,8 +1916,15 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, if (f2fs_is_pinned_file(inode)) { block_t sec_blks = CAP_BLKS_PER_SEC(sbi); - block_t sec_len = roundup(map.m_len, sec_blks); + block_t sec_len; + if (map.m_lblk % sec_blks) { + map.m_lblk = rounddown(map.m_lblk, sec_blks); + map.m_len = pg_end - map.m_lblk; + if (off_end) + map.m_len++; + } + sec_len = roundup(map.m_len, sec_blks); map.m_len = sec_blks; next_alloc: f2fs_down_write(&sbi->pin_sem); diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index ba93010924c06..11228349b0035 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1220,8 +1220,8 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, static int ra_data_block(struct inode *inode, pgoff_t index) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct address_space *mapping = f2fs_is_cow_file(inode) ? - F2FS_I(inode)->atomic_inode->i_mapping : inode->i_mapping; + struct address_space *mapping = inode->i_mapping; + struct inode *atomic_inode = NULL; struct dnode_of_data dn; struct folio *folio, *efolio; struct f2fs_io_info fio = { @@ -1236,9 +1236,22 @@ static int ra_data_block(struct inode *inode, pgoff_t index) }; int err = 0; + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (f2fs_is_cow_file(inode)) { + atomic_inode = igrab(F2FS_I(inode)->atomic_inode); + if (!atomic_inode) { + f2fs_up_read(&F2FS_I(inode)->i_sem); + return -EBUSY; + } + mapping = atomic_inode->i_mapping; + } + f2fs_up_read(&F2FS_I(inode)->i_sem); + folio = f2fs_grab_cache_folio(mapping, index, true); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out_iput; + } if (f2fs_lookup_read_extent_cache_block(inode, index, &dn.data_blkaddr)) { @@ -1299,11 +1312,16 @@ static int ra_data_block(struct inode *inode, pgoff_t index) f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE); + if (atomic_inode) + iput(atomic_inode); return 0; put_encrypted_page: f2fs_put_page(fio.encrypted_page, true); put_folio: f2fs_folio_put(folio, true); +out_iput: + if (atomic_inode) + iput(atomic_inode); return err; } @@ -1314,8 +1332,8 @@ static int ra_data_block(struct inode *inode, pgoff_t index) static int move_data_block(struct inode *inode, block_t bidx, int gc_type, unsigned int segno, int off) { - struct address_space *mapping = f2fs_is_cow_file(inode) ? - F2FS_I(inode)->atomic_inode->i_mapping : inode->i_mapping; + struct address_space *mapping = inode->i_mapping; + struct inode *atomic_inode = NULL; struct f2fs_io_info fio = { .sbi = F2FS_I_SB(inode), .ino = inode->i_ino, @@ -1337,10 +1355,23 @@ static int move_data_block(struct inode *inode, block_t bidx, (fio.sbi->gc_mode != GC_URGENT_HIGH) ? CURSEG_ALL_DATA_ATGC : CURSEG_COLD_DATA; + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (f2fs_is_cow_file(inode)) { + atomic_inode = igrab(F2FS_I(inode)->atomic_inode); + if (!atomic_inode) { + f2fs_up_read(&F2FS_I(inode)->i_sem); + return -EBUSY; + } + mapping = atomic_inode->i_mapping; + } + f2fs_up_read(&F2FS_I(inode)->i_sem); + /* do not read out */ folio = f2fs_grab_cache_folio(mapping, bidx, false); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out_iput; + } if (!check_valid_map(F2FS_I_SB(inode), segno, off)) { err = -ENOENT; @@ -1468,11 +1499,10 @@ static int move_data_block(struct inode *inode, block_t bidx, put_out: f2fs_put_dnode(&dn); out: - if (!folio_test_uptodate(folio)) - __folio_set_dropbehind(folio); - folio_unlock(folio); - folio_end_dropbehind(folio); - folio_put(folio); + f2fs_folio_put(folio, true); +out_iput: + if (atomic_inode) + iput(atomic_inode); return err; } diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index c6dcda447882f..5f0cf81e49f08 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -324,9 +324,9 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio) } if (f2fs_sb_has_flexible_inline_xattr(sbi) && - f2fs_has_inline_xattr(inode) && - (fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE || - fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE)) { + (fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE || + (f2fs_has_inline_xattr(inode) && + fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE))) { f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %lu", __func__, inode->i_ino, fi->i_inline_xattr_size, MIN_INLINE_XATTR_SIZE, MAX_INLINE_XATTR_SIZE); @@ -561,8 +561,13 @@ static int do_read_inode(struct inode *inode) static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino) { - return ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi) || - ino == F2FS_COMPRESS_INO(sbi); + if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi)) + return true; +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (test_opt(sbi, COMPRESS_CACHE) && ino == F2FS_COMPRESS_INO(sbi)) + return true; +#endif + return false; } struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) @@ -858,10 +863,15 @@ void f2fs_evict_inode(struct inode *inode) f2fs_abort_atomic_write(inode, true); if (fi->cow_inode && f2fs_is_cow_file(fi->cow_inode)) { - clear_inode_flag(fi->cow_inode, FI_COW_FILE); - F2FS_I(fi->cow_inode)->atomic_inode = NULL; - iput(fi->cow_inode); + struct inode *cow_inode = fi->cow_inode; + + f2fs_down_write(&F2FS_I(cow_inode)->i_sem); + clear_inode_flag(cow_inode, FI_COW_FILE); + F2FS_I(cow_inode)->atomic_inode = NULL; fi->cow_inode = NULL; + f2fs_up_write(&F2FS_I(cow_inode)->i_sem); + + iput(cow_inode); } trace_f2fs_evict_inode(inode); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 4e5bd9e4cfc32..01d860e7e169e 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1541,6 +1541,10 @@ int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, if (is_inode) goto out_err; break; + case NODE_TYPE_NON_IXNODE: + if (is_inode || is_xnode) + goto out_err; + break; default: break; } @@ -1634,7 +1638,7 @@ static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start) struct f2fs_sb_info *sbi = F2FS_F_SB(parent); nid_t nid = get_nid(parent, start, false); - return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_REGULAR); + return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE); } static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) @@ -1875,7 +1879,7 @@ int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, } if (!__write_node_folio(node_folio, false, false, NULL, - &wbc, false, FS_GC_NODE_IO, NULL)) + &wbc, false, io_type, NULL)) err = -EAGAIN; goto release_folio; out_folio: diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 788f8b0502492..c0276a3018560 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -444,6 +444,13 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) if (has_enough_free_secs(sbi, 0, 0)) return; + /* + * Submit all cached OPU/IPU DATA bios before triggering + * foreground GC to avoid potential deadlocks. + */ + f2fs_submit_merged_write(sbi, DATA); + f2fs_submit_all_merged_ipu_writes(sbi); + if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && sbi->gc_thread->f2fs_gc_task) { DEFINE_WAIT(wait); @@ -462,6 +469,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) .should_migrate_blocks = false, .err_gc_skipped = false, .nr_free_secs = 1 }; + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); stat_inc_gc_call_count(sbi, FOREGROUND); f2fs_gc(sbi, &gc_control); diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 610d5810074dc..1b8952b3a4478 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -581,8 +581,6 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) size_t prefix_len; size_t size; - prefix = f2fs_xattr_prefix(entry->e_name_index, dentry); - if ((void *)(entry) + sizeof(__u32) > last_base_addr || (void *)XATTR_NEXT_ENTRY(entry) > last_base_addr) { f2fs_err(F2FS_I_SB(inode), "list inode (%llu) has corrupted xattr", @@ -590,9 +588,11 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_XATTR); - break; + error = -EFSCORRUPTED; + goto cleanup; } + prefix = f2fs_xattr_prefix(entry->e_name_index, dentry); if (!prefix) continue; diff --git a/fs/fcntl.c b/fs/fcntl.c index beab8080badf6..92d643a141964 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -929,11 +929,11 @@ void send_sigio(struct fown_struct *fown, int fd, int band) send_sigio_to_task(p, fown, fd, band, type); rcu_read_unlock(); } else { - read_lock(&tasklist_lock); + rcu_read_lock(); do_each_pid_task(pid, type, p) { send_sigio_to_task(p, fown, fd, band, type); } while_each_pid_task(pid, type, p); - read_unlock(&tasklist_lock); + rcu_read_unlock(); } out_unlock_fown: read_unlock_irqrestore(&fown->lock, flags); @@ -975,11 +975,11 @@ int send_sigurg(struct file *file) send_sigurg_to_task(p, fown, type); rcu_read_unlock(); } else { - read_lock(&tasklist_lock); + rcu_read_lock(); do_each_pid_task(pid, type, p) { send_sigurg_to_task(p, fown, type); } while_each_pid_task(pid, type, p); - read_unlock(&tasklist_lock); + rcu_read_unlock(); } out_unlock_fown: read_unlock_irqrestore(&fown->lock, flags); diff --git a/fs/fhandle.c b/fs/fhandle.c index 1ca7eb3a6cb51..f8829231e3d75 100644 --- a/fs/fhandle.c +++ b/fs/fhandle.c @@ -295,7 +295,7 @@ static bool capable_wrt_mount(struct mount *mount) */ guard(rcu)(); mnt_ns = READ_ONCE(mount->mnt_ns); - return ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN); + return mnt_ns && ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN); } static inline int may_decode_fh(struct handle_to_path_ctx *ctx, diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index a65694cbfe689..325a30cc35bfb 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -660,12 +660,19 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) atomic_inc(&isw_nr_in_flight); - /* find and pin the new wb */ + /* + * Paired with synchronize_rcu() in cgroup_writeback_umount(): + * holding rcu_read_lock across inode_prepare_wbs_switch() + * (covering the SB_ACTIVE check and the inode grab) and + * wb_queue_isw() ensures synchronize_rcu() cannot return until + * the work is queued, so the subsequent flush_workqueue() will + * wait for the switch. + */ rcu_read_lock(); + /* find and pin the new wb */ memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys); if (memcg_css && !css_tryget(memcg_css)) memcg_css = NULL; - rcu_read_unlock(); if (!memcg_css) goto out_free; @@ -681,9 +688,11 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) trace_inode_switch_wbs_queue(inode->i_wb, new_wb, 1); wb_queue_isw(new_wb, isw); + rcu_read_unlock(); return; out_free: + rcu_read_unlock(); atomic_dec(&isw_nr_in_flight); if (new_wb) wb_put(new_wb); @@ -741,6 +750,14 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */ nr = 0; + /* + * Paired with synchronize_rcu() in cgroup_writeback_umount(). + * Holding rcu_read_lock across the SB_ACTIVE check, the inode grab + * and wb_queue_isw() ensures synchronize_rcu() cannot return until + * the work is queued, so the subsequent flush_workqueue() will wait + * for the switch. + */ + rcu_read_lock(); spin_lock(&wb->list_lock); /* * In addition to the inodes that have completed writeback, also switch @@ -758,6 +775,7 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) /* no attached inodes? bail out */ if (nr == 0) { + rcu_read_unlock(); atomic_dec(&isw_nr_in_flight); wb_put(new_wb); kfree(isw); @@ -766,6 +784,7 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) trace_inode_switch_wbs_queue(wb, new_wb, nr); wb_queue_isw(new_wb, isw); + rcu_read_unlock(); return restart; } @@ -1222,10 +1241,13 @@ void cgroup_writeback_umount(struct super_block *sb) if (atomic_read(&isw_nr_in_flight)) { /* - * Use rcu_barrier() to wait for all pending callbacks to - * ensure that all in-flight wb switches are in the workqueue. + * Paired with rcu_read_lock() in inode_switch_wbs() and + * cleanup_offline_cgwb(). synchronize_rcu() waits for any + * in-flight switcher that already passed the SB_ACTIVE check + * to finish queueing its work, so flush_workqueue() below + * will then drain it. */ - rcu_barrier(); + synchronize_rcu(); flush_workqueue(isw_wq); } } diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 174333633471b..3d38828af41fc 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -391,7 +391,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, rc = -ENOMEM; cdev = cdev_alloc(); if (!cdev) - goto err_unlock; + goto err_dev; cdev->owner = THIS_MODULE; cdev->ops = &cuse_frontend_fops; @@ -417,6 +417,8 @@ static void cuse_process_init_reply(struct fuse_mount *fm, err_cdev: cdev_del(cdev); +err_dev: + device_del(dev); err_unlock: mutex_unlock(&cuse_lock); put_device(dev); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index c105aaf9ff5d7..5aea3f4d30306 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -148,6 +148,7 @@ static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags) static void fuse_request_free(struct fuse_req *req) { + WARN_ON(!list_empty(&req->intr_entry)); kmem_cache_free(fuse_req_cachep, req); } @@ -447,6 +448,29 @@ static void flush_bg_queue(struct fuse_conn *fc) } } +void fuse_request_bg_finish(struct fuse_conn *fc, struct fuse_req *req) +{ + lockdep_assert_held(&fc->bg_lock); + + clear_bit(FR_BACKGROUND, &req->flags); + if (fc->num_background == fc->max_background) { + fc->blocked = 0; + wake_up(&fc->blocked_waitq); + } else if (!fc->blocked) { + /* + * Wake up next waiter, if any. It's okay to use + * waitqueue_active(), as we've already synced up + * fc->blocked with waiters with the wake_up() call + * above. + */ + if (waitqueue_active(&fc->blocked_waitq)) + wake_up(&fc->blocked_waitq); + } + + fc->num_background--; + fc->active_background--; +} + /* * This function is called when a request is finished. Either a reply * has arrived or it was aborted (and not yet sent) or some error @@ -479,23 +503,7 @@ void fuse_request_end(struct fuse_req *req) WARN_ON(test_bit(FR_SENT, &req->flags)); if (test_bit(FR_BACKGROUND, &req->flags)) { spin_lock(&fc->bg_lock); - clear_bit(FR_BACKGROUND, &req->flags); - if (fc->num_background == fc->max_background) { - fc->blocked = 0; - wake_up(&fc->blocked_waitq); - } else if (!fc->blocked) { - /* - * Wake up next waiter, if any. It's okay to use - * waitqueue_active(), as we've already synced up - * fc->blocked with waiters with the wake_up() call - * above. - */ - if (waitqueue_active(&fc->blocked_waitq)) - wake_up(&fc->blocked_waitq); - } - - fc->num_background--; - fc->active_background--; + fuse_request_bg_finish(fc, req); flush_bg_queue(fc); spin_unlock(&fc->bg_lock); } else { @@ -1037,6 +1045,10 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop if (WARN_ON(folio_test_mlocked(oldfolio))) goto out_fallback_unlock; + err = lock_request(cs->req); + if (err) + goto out_fallback_unlock; + replace_page_cache_folio(oldfolio, newfolio); folio_get(newfolio); @@ -1050,20 +1062,7 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop */ pipe_buf_release(cs->pipe, buf); - err = 0; - spin_lock(&cs->req->waitq.lock); - if (test_bit(FR_ABORTED, &cs->req->flags)) - err = -ENOENT; - else - *foliop = newfolio; - spin_unlock(&cs->req->waitq.lock); - - if (err) { - folio_unlock(newfolio); - folio_put(newfolio); - goto out_put_old; - } - + *foliop = newfolio; folio_unlock(oldfolio); /* Drop ref for ap->pages[] array */ folio_put(oldfolio); @@ -1115,7 +1114,7 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio, cs->nr_segs++; cs->len = 0; - return 0; + return lock_request(cs->req); } /* @@ -1562,9 +1561,15 @@ struct fuse_dev *fuse_get_dev(struct file *file) struct fuse_dev *fud = fuse_file_to_fud(file); int err; - err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_fc_get(fud) != NULL); - if (err) - return ERR_PTR(err); + if (unlikely(!fuse_dev_fc_get(fud))) { + /* only block waiting for mount if sync init was requested */ + if (!fud->sync_init) + return ERR_PTR(-EPERM); + + err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_fc_get(fud) != NULL); + if (err) + return ERR_PTR(err); + } return fud; } @@ -2037,6 +2042,14 @@ static void fuse_resend(struct fuse_conn *fc) fuse_dev_end_requests(&to_queue); return; } + /* + * Remove interrupt entries for resent requests to prevent stale + * intr_entry on fiq->interrupts after the request is re-queued. + */ + list_for_each_entry(req, &to_queue, list) { + if (test_bit(FR_INTERRUPTED, &req->flags)) + list_del_init(&req->intr_entry); + } /* iq and pq requests are both oldest to newest */ list_splice(&to_queue, &fiq->pending); fuse_dev_wake_and_unlock(fiq); @@ -2081,7 +2094,7 @@ static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size, if (err) return err; - if (size - sizeof(outarg) != outarg.count * sizeof(u64)) + if (size - sizeof(outarg) != array_size(outarg.count, sizeof(u64))) return -EINVAL; for (; outarg.count; outarg.count -= num) { diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 7b9822e8837bc..4fde6336cf5ce 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -90,6 +90,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; spin_lock(&fc->bg_lock); + fuse_request_bg_finish(fc, req); fuse_uring_flush_bg(queue); spin_unlock(&fc->bg_lock); } @@ -130,10 +131,9 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring) if (!queue) continue; - queue->stopped = true; - WARN_ON_ONCE(ring->fc->max_background != UINT_MAX); spin_lock(&queue->lock); + queue->stopped = true; spin_lock(&fc->bg_lock); fuse_uring_flush_bg(queue); spin_unlock(&fc->bg_lock); @@ -466,6 +466,7 @@ static void fuse_uring_async_stop_queues(struct work_struct *work) FUSE_URING_TEARDOWN_INTERVAL); } else { wake_up_all(&ring->stop_waitq); + fuse_conn_put(ring->fc); } } @@ -477,6 +478,7 @@ void fuse_uring_stop_queues(struct fuse_ring *ring) fuse_uring_teardown_all_queues(ring); if (atomic_read(&ring->queue_refs) > 0) { + fuse_conn_get(ring->fc); ring->teardown_time = jiffies; INIT_DELAYED_WORK(&ring->async_teardown_work, fuse_uring_async_stop_queues); @@ -507,8 +509,7 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd, queue = ent->queue; spin_lock(&queue->lock); if (ent->state == FRRS_AVAILABLE) { - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); + list_del_init(&ent->list); need_cmd_done = true; ent->cmd = NULL; } @@ -517,6 +518,9 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd, if (need_cmd_done) { /* no queue lock to avoid lock order issues */ io_uring_cmd_done(cmd, -ENOTCONN, issue_flags); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); } } @@ -702,14 +706,37 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, int err; err = fuse_uring_copy_to_ring(ent, req); - if (!err) + if (!err) { set_bit(FR_SENT, &req->flags); - else + } else { + /* + * Copying the request failed. Remove the entry from the + * ent_w_req_queue list and terminate the request + */ + spin_lock(&ent->queue->lock); + list_del_init(&ent->list); + ent->state = FRRS_INVALID; + spin_unlock(&ent->queue->lock); + fuse_uring_req_end(ent, req, err); + } return err; } +/* Used to find the request on SQE commit */ +static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_pqueue *fpq = &queue->fpq; + unsigned int hash; + struct fuse_req *req = ent->fuse_req; + + req->ring_entry = ent; + hash = fuse_req_hash(req->in.h.unique); + list_move_tail(&req->list, &fpq->processing[hash]); +} + /* * Write data to the ring buffer and send the request to userspace, * userspace will read it @@ -732,6 +759,7 @@ static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent, ent->cmd = NULL; ent->state = FRRS_USERSPACE; list_move_tail(&ent->list, &queue->ent_in_userspace); + fuse_uring_add_to_pq(ent); spin_unlock(&queue->lock); io_uring_cmd_done(cmd, 0, issue_flags); @@ -749,19 +777,6 @@ static void fuse_uring_ent_avail(struct fuse_ring_ent *ent, ent->state = FRRS_AVAILABLE; } -/* Used to find the request on SQE commit */ -static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent, - struct fuse_req *req) -{ - struct fuse_ring_queue *queue = ent->queue; - struct fuse_pqueue *fpq = &queue->fpq; - unsigned int hash; - - req->ring_entry = ent; - hash = fuse_req_hash(req->in.h.unique); - list_move_tail(&req->list, &fpq->processing[hash]); -} - /* * Assign a fuse queue entry to the given entry */ @@ -779,10 +794,13 @@ static void fuse_uring_add_req_to_ring_ent(struct fuse_ring_ent *ent, } clear_bit(FR_PENDING, &req->flags); + + /* Until fuse_uring_add_to_pq() the req is not attached to any list */ + list_del_init(&req->list); + ent->fuse_req = req; ent->state = FRRS_FUSE_REQ; list_move_tail(&ent->list, &queue->ent_w_req_queue); - fuse_uring_add_to_pq(ent, req); } /* Fetch the next fuse request if available */ @@ -813,14 +831,11 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, { struct fuse_ring *ring = ent->queue->ring; struct fuse_conn *fc = ring->fc; - ssize_t err = 0; + ssize_t err = -EFAULT; - err = copy_from_user(&req->out.h, &ent->headers->in_out, - sizeof(req->out.h)); - if (err) { - req->out.h.error = -EFAULT; + if (copy_from_user(&req->out.h, &ent->headers->in_out, + sizeof(req->out.h))) goto out; - } err = fuse_uring_out_header_has_err(&req->out.h, req, fc); if (err) { @@ -898,10 +913,15 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, return err; fpq = &queue->fpq; - if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped)) + if (!READ_ONCE(fc->connected)) return err; spin_lock(&queue->lock); + if (unlikely(queue->stopped)) { + spin_unlock(&queue->lock); + return err; + } + /* Find a request based on the unique ID of the fuse request * This should get revised, as it needs a hash calculation and list * search. And full struct fuse_pqueue is needed (memory overhead). @@ -991,12 +1011,12 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent, fuse_uring_ent_avail(ent, queue); spin_unlock(&queue->lock); - if (!ring->ready) { + if (!READ_ONCE(ring->ready)) { bool ready = is_ring_ready(ring, queue->qid); if (ready) { WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); - WRITE_ONCE(ring->ready, true); + smp_store_release(&ring->ready, true); wake_up_all(&fc->blocked_waitq); } } @@ -1201,6 +1221,7 @@ static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, ent->state = FRRS_USERSPACE; list_move_tail(&ent->list, &queue->ent_in_userspace); ent->cmd = NULL; + fuse_uring_add_to_pq(ent); spin_unlock(&queue->lock); io_uring_cmd_done(cmd, ret, issue_flags); @@ -1225,11 +1246,21 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) fuse_uring_next_fuse_req(ent, queue, issue_flags); return; } + fuse_uring_send(ent, cmd, err, issue_flags); } else { err = -ECANCELED; - } - fuse_uring_send(ent, cmd, err, issue_flags); + spin_lock(&queue->lock); + list_del_init(&ent->list); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, err, issue_flags); + + fuse_uring_req_end(ent, ent->fuse_req, err); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); + } } static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 51a563922ce14..054ad4fdd7491 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -152,10 +152,10 @@ static inline void fuse_uring_abort(struct fuse_conn *fc) if (ring == NULL) return; - if (atomic_read(&ring->queue_refs) > 0) { - fuse_uring_abort_end_requests(ring); + fuse_uring_abort_end_requests(ring); + + if (atomic_read(&ring->queue_refs) > 0) fuse_uring_stop_queues(ring); - } } static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc) @@ -169,7 +169,9 @@ static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc) static inline bool fuse_uring_ready(struct fuse_conn *fc) { - return fc->ring && fc->ring->ready; + struct fuse_ring *ring = READ_ONCE(fc->ring); + + return ring && smp_load_acquire(&ring->ready); } #else /* CONFIG_FUSE_IO_URING */ diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index b658b6baf72fe..97751bacb79ce 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1587,40 +1587,34 @@ int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, { int err = -ENOTDIR; struct inode *parent; - struct dentry *dir = NULL; - struct dentry *entry = NULL; + struct dentry *dir; + struct dentry *entry; parent = fuse_ilookup(fc, parent_nodeid, NULL); if (!parent) return -ENOENT; + inode_lock_nested(parent, I_MUTEX_PARENT); if (!S_ISDIR(parent->i_mode)) - goto put_parent; + goto unlock; err = -ENOENT; dir = d_find_alias(parent); if (!dir) - goto put_parent; - while (!entry) { - struct dentry *child = try_lookup_noperm(name, dir); - if (!child || IS_ERR(child)) - goto put_parent; - entry = start_removing_dentry(dir, child); - dput(child); - if (IS_ERR(entry)) - goto put_parent; - if (!d_same_name(entry, dir, name)) { - end_removing(entry); - entry = NULL; - } - } + goto unlock; + + name->hash = full_name_hash(dir, name->name, name->len); + entry = d_lookup(dir, name); + dput(dir); + if (!entry) + goto unlock; fuse_dir_changed(parent); if (!(flags & FUSE_EXPIRE_ONLY)) d_invalidate(entry); fuse_invalidate_entry_cache(entry); - if (child_nodeid != 0) { + if (child_nodeid != 0 && d_really_is_positive(entry)) { inode_lock(d_inode(entry)); if (get_node_id(d_inode(entry)) != child_nodeid) { err = -ENOENT; @@ -1648,10 +1642,10 @@ int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, } else { err = 0; } + dput(entry); - end_removing(entry); - put_parent: - dput(dir); + unlock: + inode_unlock(parent); iput(parent); return err; } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f94f3dc082c6b..b31d0dffaf4c3 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -380,8 +380,14 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * aio and closes the fd before the aio completes. Since aio takes its * own ref to the file, the IO completion has to drop the ref, which is * how the fuse server can end up closing its clients' files. + * + * Exception is virtio-fs, which is not affected by the above (server is + * on host, cannot close open files in guest). Virtio-fs needs sync + * release, because the num_waiting mechanism to wait for all requests + * before commencing with fs shutdown doesn't work if submounts are + * used. */ - fuse_file_put(ff, false); + fuse_file_put(ff, ff->fm->fc->auto_submounts); } void fuse_release_common(struct file *file, bool isdir) diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 910f883cd090f..1853940fe80a5 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -77,6 +77,7 @@ unsigned int fuse_req_hash(u64 unique); struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique); void fuse_dev_end_requests(struct list_head *head); +void fuse_request_bg_finish(struct fuse_conn *fc, struct fuse_req *req); void fuse_copy_init(struct fuse_copy_state *cs, bool write, struct iov_iter *iter); diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c new file mode 100644 index 0000000000000..7cd63502e124b --- /dev/null +++ b/fs/fuse/notify.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "fuse_i.h" +#include + +static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_poll_wakeup_out outarg; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + fuse_copy_finish(cs); + return fuse_notify_poll_wakeup(fc, &outarg); +} + +static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_inval_inode_out outarg; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + fuse_copy_finish(cs); + + down_read(&fc->killsb); + err = fuse_reverse_inval_inode(fc, outarg.ino, + outarg.off, outarg.len); + up_read(&fc->killsb); + return err; +} + +static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_inval_entry_out outarg; + int err; + char *buf; + struct qstr name; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (outarg.namelen > fc->name_max) + return -ENAMETOOLONG; + + err = -EINVAL; + if (size != sizeof(outarg) + outarg.namelen + 1) + return -EINVAL; + + buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + name.name = buf; + name.len = outarg.namelen; + err = fuse_copy_one(cs, buf, outarg.namelen + 1); + if (err) + goto err; + fuse_copy_finish(cs); + buf[outarg.namelen] = 0; + + down_read(&fc->killsb); + err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags); + up_read(&fc->killsb); +err: + kfree(buf); + return err; +} + +static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_delete_out outarg; + int err; + char *buf; + struct qstr name; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (outarg.namelen > fc->name_max) + return -ENAMETOOLONG; + + if (size != sizeof(outarg) + outarg.namelen + 1) + return -EINVAL; + + buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + name.name = buf; + name.len = outarg.namelen; + err = fuse_copy_one(cs, buf, outarg.namelen + 1); + if (err) + goto err; + fuse_copy_finish(cs); + buf[outarg.namelen] = 0; + + down_read(&fc->killsb); + err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0); + up_read(&fc->killsb); +err: + kfree(buf); + return err; +} + +static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_store_out outarg; + struct inode *inode; + struct address_space *mapping; + u64 nodeid; + int err; + unsigned int num; + loff_t file_size; + loff_t pos; + loff_t end; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (size - sizeof(outarg) != outarg.size) + return -EINVAL; + + if (outarg.offset >= MAX_LFS_FILESIZE) + return -EINVAL; + + nodeid = outarg.nodeid; + pos = outarg.offset; + num = min(outarg.size, MAX_LFS_FILESIZE - pos); + + down_read(&fc->killsb); + + err = -ENOENT; + inode = fuse_ilookup(fc, nodeid, NULL); + if (!inode) + goto out_up_killsb; + + mapping = inode->i_mapping; + file_size = i_size_read(inode); + end = pos + num; + if (end > file_size) { + file_size = end; + fuse_write_update_attr(inode, file_size, num); + } + + while (num) { + struct folio *folio; + unsigned int folio_offset; + unsigned int nr_bytes; + pgoff_t index = pos >> PAGE_SHIFT; + + folio = filemap_grab_folio(mapping, index); + err = PTR_ERR(folio); + if (IS_ERR(folio)) + goto out_iput; + + folio_offset = offset_in_folio(folio, pos); + nr_bytes = min(num, folio_size(folio) - folio_offset); + + err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0); + if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && + (nr_bytes == folio_size(folio) || file_size == end)) { + folio_zero_segment(folio, nr_bytes, folio_size(folio)); + folio_mark_uptodate(folio); + } + folio_unlock(folio); + folio_put(folio); + + if (err) + goto out_iput; + + pos += nr_bytes; + num -= nr_bytes; + } + + err = 0; + +out_iput: + iput(inode); +out_up_killsb: + up_read(&fc->killsb); + return err; +} + +struct fuse_retrieve_args { + struct fuse_args_pages ap; + struct fuse_notify_retrieve_in inarg; +}; + +static void fuse_retrieve_end(struct fuse_args *args, int error) +{ + struct fuse_retrieve_args *ra = + container_of(args, typeof(*ra), ap.args); + + release_pages(ra->ap.folios, ra->ap.num_folios); + kfree(ra); +} + +static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, + struct fuse_notify_retrieve_out *outarg) +{ + int err; + struct address_space *mapping = inode->i_mapping; + loff_t file_size; + unsigned int num; + unsigned int offset; + size_t total_len = 0; + unsigned int num_pages; + struct fuse_conn *fc = fm->fc; + struct fuse_retrieve_args *ra; + size_t args_size = sizeof(*ra); + struct fuse_args_pages *ap; + struct fuse_args *args; + loff_t pos = outarg->offset; + + offset = offset_in_page(pos); + file_size = i_size_read(inode); + + num = min(outarg->size, fc->max_write); + if (pos > file_size) + num = 0; + else if (num > file_size - pos) + num = file_size - pos; + + num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE); + num_pages = min(num_pages, fc->max_pages); + num = min(num, num_pages << PAGE_SHIFT); + + args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); + + ra = kzalloc(args_size, GFP_KERNEL); + if (!ra) + return -ENOMEM; + + ap = &ra->ap; + ap->folios = (void *) (ra + 1); + ap->descs = (void *) (ap->folios + num_pages); + + args = &ap->args; + args->nodeid = outarg->nodeid; + args->opcode = FUSE_NOTIFY_REPLY; + args->in_numargs = 3; + args->in_pages = true; + args->end = fuse_retrieve_end; + + while (num && ap->num_folios < num_pages) { + struct folio *folio; + unsigned int folio_offset; + unsigned int nr_bytes; + pgoff_t index = pos >> PAGE_SHIFT; + + folio = filemap_get_folio(mapping, index); + if (IS_ERR(folio)) + break; + + folio_offset = offset_in_folio(folio, pos); + nr_bytes = min(folio_size(folio) - folio_offset, num); + + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].offset = folio_offset; + ap->descs[ap->num_folios].length = nr_bytes; + ap->num_folios++; + + pos += nr_bytes; + num -= nr_bytes; + total_len += nr_bytes; + } + ra->inarg.offset = outarg->offset; + ra->inarg.size = total_len; + fuse_set_zero_arg0(args); + args->in_args[1].size = sizeof(ra->inarg); + args->in_args[1].value = &ra->inarg; + args->in_args[2].size = total_len; + + err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); + if (err) + fuse_retrieve_end(args, err); + + return err; +} + +static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_retrieve_out outarg; + struct fuse_mount *fm; + struct inode *inode; + u64 nodeid; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + fuse_copy_finish(cs); + + if (outarg.offset >= MAX_LFS_FILESIZE) + return -EINVAL; + + down_read(&fc->killsb); + err = -ENOENT; + nodeid = outarg.nodeid; + + inode = fuse_ilookup(fc, nodeid, &fm); + if (inode) { + err = fuse_retrieve(fm, inode, &outarg); + iput(inode); + } + up_read(&fc->killsb); + + return err; +} + +static int fuse_notify_resend(struct fuse_conn *fc) +{ + fuse_chan_resend(fc->chan); + return 0; +} + +/* + * Increments the fuse connection epoch. This will result of dentries from + * previous epochs to be invalidated. Additionally, if inval_wq is set, a work + * queue is scheduled to trigger the invalidation. + */ +static int fuse_notify_inc_epoch(struct fuse_conn *fc) +{ + atomic_inc(&fc->epoch); + if (inval_wq) + schedule_work(&fc->epoch_work); + + return 0; +} + +static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_prune_out outarg; + const unsigned int batch = 512; + u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL); + unsigned int num, i; + int err; + + if (!nodeids) + return -ENOMEM; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (size - sizeof(outarg) != array_size(outarg.count, sizeof(u64))) + return -EINVAL; + + for (; outarg.count; outarg.count -= num) { + num = min(batch, outarg.count); + err = fuse_copy_one(cs, nodeids, num * sizeof(u64)); + if (err) + return err; + + scoped_guard(rwsem_read, &fc->killsb) { + for (i = 0; i < num; i++) + fuse_try_prune_one_inode(fc, nodeids[i]); + } + } + return 0; +} + +int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, + unsigned int size, struct fuse_copy_state *cs) +{ + switch (code) { + case FUSE_NOTIFY_POLL: + return fuse_notify_poll(fc, size, cs); + + case FUSE_NOTIFY_INVAL_INODE: + return fuse_notify_inval_inode(fc, size, cs); + + case FUSE_NOTIFY_INVAL_ENTRY: + return fuse_notify_inval_entry(fc, size, cs); + + case FUSE_NOTIFY_STORE: + return fuse_notify_store(fc, size, cs); + + case FUSE_NOTIFY_RETRIEVE: + return fuse_notify_retrieve(fc, size, cs); + + case FUSE_NOTIFY_DELETE: + return fuse_notify_delete(fc, size, cs); + + case FUSE_NOTIFY_RESEND: + return fuse_notify_resend(fc); + + case FUSE_NOTIFY_INC_EPOCH: + return fuse_notify_inc_epoch(fc); + + case FUSE_NOTIFY_PRUNE: + return fuse_notify_prune(fc, size, cs); + + default: + return -EINVAL; + } +} diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index db5ae8ec10305..c38139225a2e3 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -12,6 +12,7 @@ #include #include #include +#include static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) { @@ -335,6 +336,43 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, return 0; } +static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize) +{ + unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE); + struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + + if (!pages) + return NULL; + + nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages); + if (!nr_alloc) + goto free_array; + + if (nr_alloc < nr_pages) { + nr_pages = nr_alloc; + *bufsize = (size_t) nr_pages << PAGE_SHIFT; + } + + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); + if (!ap->folios) + goto release_pages; + + for (i = 0; i < nr_pages; i++) { + ap->folios[i] = page_folio(pages[i]); + ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE); + } + ap->num_folios = nr_pages; + ap->args.out_pages = true; + + return pages; + +release_pages: + release_pages(pages, nr_pages); +free_array: + kfree(pages); + return NULL; +} + static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) { int plus; @@ -343,18 +381,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_conn *fc = fm->fc; struct fuse_io_args ia = {}; - struct fuse_args *args = &ia.ap.args; + struct fuse_args_pages *ap = &ia.ap; void *buf; size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT); u64 attr_version = 0, evict_ctr = 0; bool locked; + struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize); - buf = kvmalloc(bufsize, GFP_KERNEL); - if (!buf) + if (!pages) return -ENOMEM; - args->out_args[0].value = buf; - plus = fuse_use_readdirplus(inode, ctx); if (plus) { attr_version = fuse_get_attr_version(fm->fc); @@ -364,24 +400,37 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR); } locked = fuse_lock_inode(inode); - res = fuse_simple_request(fm, args); + res = fuse_simple_request(fm, &ap->args); fuse_unlock_inode(inode, locked); - if (res >= 0) { - if (!res) { - struct fuse_file *ff = file->private_data; - - if (ff->open_flags & FOPEN_CACHE_DIR) - fuse_readdir_cache_end(file, ctx->pos); - } else if (plus) { - res = parse_dirplusfile(buf, res, file, ctx, attr_version, - evict_ctr); - } else { + if (res < 0) + goto out; + + if (!res) { + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_readdir_cache_end(file, ctx->pos); + goto out; + } + + buf = vm_map_ram(pages, ap->num_folios, -1); + if (!buf) { + res = -ENOMEM; + } else { + if (plus) + res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr); + else res = parse_dirfile(buf, res, file, ctx); - } + + vm_unmap_ram(buf, ap->num_folios); } +out: + kfree(ap->folios); + release_pages(pages, ap->num_folios); + kfree(pages); - kvfree(buf); fuse_invalidate_atime(inode); + return res; } diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index a2ea121331f18..4d854556b5299 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -643,6 +643,7 @@ static void gfs2_put_super(struct super_block *sb) gfs2_delete_debugfs_file(sdp); gfs2_sys_fs_del(sdp); + rcu_barrier(); free_sbd(sdp); } diff --git a/fs/hfs/bnode.c b/fs/hfs/bnode.c index 13d58c51fc46b..25cef62fbba6d 100644 --- a/fs/hfs/bnode.c +++ b/fs/hfs/bnode.c @@ -41,7 +41,7 @@ u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) node_size = node->tree->node_size; - if ((off + len) > node_size) { + if ((u64)off + len > node_size) { u32 new_len = node_size - off; pr_err("requested length has been corrected: " @@ -64,6 +64,8 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) u32 bytes_read; u32 bytes_to_read; + memset(buf, 0, len); + if (!is_bnode_offset_valid(node, off)) return; diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index 89b33a9d46d5c..1cbba73450389 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -204,7 +204,7 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t pr_err("cannot create new inode: next CNID exceeds limit\n"); goto out_discard; } - inode->i_ino = (u32)next_id; + inode->i_ino = (u32)next_id - 1; inode->i_mode = mode; inode->i_uid = current_fsuid(); inode->i_gid = current_fsgid(); diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index f8b5a8ae58ff5..14d1af2c7ba93 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -25,6 +25,8 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) struct page **pagep; u32 l; + memset(buf, 0, len); + if (!is_bnode_offset_valid(node, off)) return; diff --git a/fs/hfsplus/btree.c b/fs/hfsplus/btree.c index 761c74ccd6531..394542a47e600 100644 --- a/fs/hfsplus/btree.c +++ b/fs/hfsplus/btree.c @@ -365,6 +365,8 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id) } size = tree->node_size; + if (size < HFSPLUS_NODE_MINSZ || size > HFSPLUS_NODE_MXSZ) + goto fail_page; if (!is_power_of_2(size)) goto fail_page; if (!tree->node_count) diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index 3545b8dbf11c5..0e4268de9e60e 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -600,7 +600,7 @@ u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) node_size = node->tree->node_size; - if ((off + len) > node_size) { + if ((u64)off + len > node_size) { u32 new_len = node_size - off; pr_err("requested length has been corrected: " diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c index 452a1f9becb2d..21a1c196c71f2 100644 --- a/fs/hfsplus/xattr.c +++ b/fs/hfsplus/xattr.c @@ -317,7 +317,6 @@ static int hfsplus_create_attributes_file(struct super_block *sb) next_node++; } - hfsplus_mark_inode_dirty(HFSPLUS_ATTR_TREE_I(sb), HFSPLUS_I_ATTR_DIRTY); hfsplus_mark_inode_dirty(attr_file, HFSPLUS_I_ATTR_DIRTY); sbi->attr_tree = hfs_btree_open(sb, HFSPLUS_ATTR_CNID); diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c index 4504f4633f175..0f31e35567b4b 100644 --- a/fs/iomap/bio.c +++ b/fs/iomap/bio.c @@ -78,15 +78,23 @@ u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend) return __iomap_read_end_io(&ioend->io_bio, ioend->io_error); } -static void iomap_bio_submit_read(const struct iomap_iter *iter, - struct iomap_read_folio_ctx *ctx) +void iomap_bio_submit_read_endio(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io) { struct bio *bio = ctx->read_ctx; + bio->bi_end_io = end_io; if (iter->iomap.flags & IOMAP_F_INTEGRITY) fs_bio_integrity_alloc(bio); submit_bio(bio); } +EXPORT_SYMBOL_GPL(iomap_bio_submit_read_endio); + +static void iomap_bio_submit_read(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + return iomap_bio_submit_read_endio(iter, ctx, iomap_read_end_io); +} static struct bio_set *iomap_read_bio_set(struct iomap_read_folio_ctx *ctx) { @@ -127,7 +135,6 @@ static void iomap_read_alloc_bio(const struct iomap_iter *iter, if (ctx->rac) bio->bi_opf |= REQ_RAHEAD; bio->bi_iter.bi_sector = iomap_sector(iomap, iter->pos); - bio->bi_end_io = iomap_read_end_io; bio_add_folio_nofail(bio, folio, plen, offset_in_folio(folio, iter->pos)); ctx->read_ctx = bio; diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index d55b936e69862..5fa9a2c7e30eb 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -850,7 +850,7 @@ static int __iomap_write_begin(const struct iomap_iter *iter, if (status < 0) fserror_report_io(iter->inode, FSERR_BUFFERED_READ, pos, - len, status, GFP_NOFS); + plen, status, GFP_NOFS); if (status) return status; } diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index b36ee619cdcdd..9fedf367c3ebe 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -369,7 +369,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, */ if ((op & REQ_ATOMIC) && WARN_ON_ONCE(ret != iomap_length(iter))) { ret = -EINVAL; - goto out_put_bio; + goto out_bio_release_pages; } if (iter->iomap.flags & IOMAP_F_INTEGRITY) { @@ -393,6 +393,11 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, iomap_dio_submit_bio(iter, dio, bio, pos); return ret; +out_bio_release_pages: + if (dio->flags & IOMAP_DIO_BOUNCE) + bio_iov_iter_unbounce(bio, true, false); + else + bio_release_pages(bio, false); out_put_bio: bio_put(bio); return ret; diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index acf3cf98b23a9..2d5611f6cc57d 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -297,8 +297,12 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, * appending writes. */ ioend->io_size += map_len; - if (ioend->io_offset + ioend->io_size > end_pos) - ioend->io_size = end_pos - ioend->io_offset; + if (ioend->io_offset + ioend->io_size > end_pos) { + if (ioend->io_offset >= end_pos) + ioend->io_size = 0; + else + ioend->io_size = end_pos - ioend->io_offset; + } wbc_account_cgroup_owner(wpc->wbc, folio, map_len); return map_len; diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c index 1232fab59a4e6..2628f31bd3a5d 100644 --- a/fs/isofs/rock.c +++ b/fs/isofs/rock.c @@ -466,6 +466,9 @@ parse_rock_ridge_inode_internal(struct iso_directory_record *de, inode->i_size = symlink_len; while (slen > 1) { rootflag = 0; + /* keep the component within the SL record */ + if (slp->len + 2 > slen) + goto eio; switch (slp->flags & ~1) { case 0: inode->i_size += @@ -621,6 +624,14 @@ static char *get_symlink_chunk(char *rpnt, struct rock_ridge *rr, char *plimit) slp = &rr->u.SL.link; while (slen > 1) { rootflag = 0; + /* + * A component is slp->len + 2 bytes (a two-byte header plus + * len bytes of text). If it does not fit in the bytes left in + * the SL record the record is malformed: fail like the plimit + * checks below so readlink() returns -EIO, not a truncated path. + */ + if (slp->len + 2 > slen) + return NULL; switch (slp->flags & ~1) { case 0: if (slp->len > plimit - rpnt) diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 4f397fcdb13c5..e3b2e38e1a1b7 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -2263,6 +2263,8 @@ jbd2_journal_initialize_fast_commit(journal_t *journal) unsigned long long num_fc_blks; num_fc_blks = jbd2_journal_get_num_fc_blks(sb); + if (num_fc_blks > journal->j_last) + return -EFSCORRUPTED; if (journal->j_last - num_fc_blks < JBD2_MIN_JOURNAL_BLOCKS) return -ENOSPC; diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index 4f9ade82b08ab..97d9d227b66d9 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -597,19 +597,19 @@ void kernfs_put(struct kernfs_node *kn) */ parent = kernfs_parent(kn); - WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS, - "kernfs_put: %s/%s: released with incorrect active_ref %d\n", - parent ? rcu_dereference(parent->name) : "", - rcu_dereference(kn->name), atomic_read(&kn->active)); + if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS) { + guard(rcu)(); + WARN_ONCE(1, + "kernfs_put: %s/%s: released with incorrect active_ref %d\n", + parent ? rcu_dereference(parent->name) : "", + rcu_dereference(kn->name), atomic_read(&kn->active)); + } if (kernfs_type(kn) == KERNFS_LINK) kernfs_put(kn->symlink.target_kn); - if (kn->iattr && kn->iattr->xattrs) { - simple_xattrs_free(kn->iattr->xattrs, NULL); - kfree(kn->iattr->xattrs); - kn->iattr->xattrs = NULL; - } + if (kn->iattr) + simple_xattrs_free(&root->xa_cache, &kn->iattr->xattrs, NULL); spin_lock(&root->kernfs_idr_lock); idr_remove(&root->ino_idr, (u32)kernfs_ino(kn)); @@ -624,6 +624,7 @@ void kernfs_put(struct kernfs_node *kn) } else { /* just released the root kn, free @root too */ idr_destroy(&root->ino_idr); + simple_xattr_cache_cleanup(&root->xa_cache); kfree_rcu(root, rcu); } } @@ -700,6 +701,9 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, } if (parent) { + kernfs_get(parent); + rcu_assign_pointer(kn->__parent, parent); + ret = security_kernfs_init_security(parent, kn); if (ret) goto err_out4; @@ -708,11 +712,10 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, return kn; err_out4: + RCU_INIT_POINTER(kn->__parent, NULL); + kernfs_put(parent); if (kn->iattr) { - if (kn->iattr->xattrs) { - simple_xattrs_free(kn->iattr->xattrs, NULL); - kfree(kn->iattr->xattrs); - } + simple_xattrs_free(&root->xa_cache, &kn->iattr->xattrs, NULL); kmem_cache_free(kernfs_iattrs_cache, kn->iattr); } err_out3: @@ -747,10 +750,6 @@ struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, kn = __kernfs_new_node(kernfs_root(parent), parent, name, mode, uid, gid, flags); - if (kn) { - kernfs_get(parent); - rcu_assign_pointer(kn->__parent, parent); - } return kn; } diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 1163aa7697384..8e0e90c933720 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -40,22 +40,15 @@ struct kernfs_open_node { static DEFINE_SPINLOCK(kernfs_notify_lock); static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL; +/* Compatibility wrappers - use the common hashed node lock */ static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn) { - int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS); - - return &kernfs_locks->open_file_mutex[idx]; + return kernfs_node_lock_ptr(kn); } static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn) { - struct mutex *lock; - - lock = kernfs_open_file_mutex_ptr(kn); - - mutex_lock(lock); - - return lock; + return kernfs_node_lock(kn); } /** diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index 38b28aa7cd023..2cb20294aaf58 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -37,6 +37,7 @@ static struct kernfs_iattrs *__kernfs_iattrs(struct kernfs_node *kn, bool alloc) if (!ret) return NULL; + INIT_LIST_HEAD_RCU(&ret->xattrs); /* assign default attributes */ ret->ia_uid = GLOBAL_ROOT_UID; ret->ia_gid = GLOBAL_ROOT_GID; @@ -144,8 +145,7 @@ ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size) if (!attrs) return -ENOMEM; - return simple_xattr_list(d_inode(dentry), READ_ONCE(attrs->xattrs), - buf, size); + return simple_xattr_list(d_inode(dentry), &attrs->xattrs, buf, size); } static inline void set_default_inode_attr(struct inode *inode, umode_t mode) @@ -297,34 +297,35 @@ int kernfs_xattr_get(struct kernfs_node *kn, const char *name, void *value, size_t size) { struct kernfs_iattrs *attrs = kernfs_iattrs_noalloc(kn); - struct simple_xattrs *xattrs; + struct simple_xattr_cache *cache = &kernfs_root(kn)->xa_cache; if (!attrs) return -ENODATA; - xattrs = READ_ONCE(attrs->xattrs); - if (!xattrs) - return -ENODATA; - - return simple_xattr_get(xattrs, name, value, size); + return simple_xattr_get(cache, &attrs->xattrs, name, value, size); } int kernfs_xattr_set(struct kernfs_node *kn, const char *name, const void *value, size_t size, int flags) { struct simple_xattr *old_xattr; - struct simple_xattrs *xattrs; struct kernfs_iattrs *attrs; + struct simple_xattr_cache *cache = &kernfs_root(kn)->xa_cache; attrs = kernfs_iattrs(kn); if (!attrs) return -ENOMEM; - xattrs = simple_xattrs_lazy_alloc(&attrs->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); + /* + * Protect xattr modifications with the hashed per-node mutex. + * Multiple superblocks (with different namespaces) can share the same + * kernfs_node, so inode locking alone is insufficient. The hashed mutex + * ensures serialization of concurrent xattr operations on the same node, + * including the lazy allocation of the xattrs structure itself. + */ + CLASS(kernfs_node_lock, lock)(kn); - old_xattr = simple_xattr_set(xattrs, name, value, size, flags); + old_xattr = simple_xattr_set(cache, &attrs->xattrs, name, value, size, flags); if (IS_ERR(old_xattr)) return PTR_ERR(old_xattr); @@ -362,7 +363,6 @@ static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler, { const char *full_name = xattr_full_name(handler, suffix); struct kernfs_node *kn = inode->i_private; - struct simple_xattrs *xattrs; struct kernfs_iattrs *attrs; if (!(kernfs_root(kn)->flags & KERNFS_ROOT_SUPPORT_USER_XATTR)) @@ -372,11 +372,11 @@ static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler, if (!attrs) return -ENOMEM; - xattrs = simple_xattrs_lazy_alloc(&attrs->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); + /* See comment in kernfs_xattr_set() about locking. */ + CLASS(kernfs_node_lock, lock)(kn); - return simple_xattr_set_limited(xattrs, &attrs->xattr_limits, + return simple_xattr_set_limited(&kernfs_root(kn)->xa_cache, + &attrs->xattrs, &attrs->xattr_limits, full_name, value, size, flags); } diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index 8d8912f50b054..aa784b540b361 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -26,7 +26,7 @@ struct kernfs_iattrs { struct timespec64 ia_mtime; struct timespec64 ia_ctime; - struct simple_xattrs *xattrs; + struct list_head xattrs; struct simple_xattr_limits xattr_limits; }; @@ -54,6 +54,8 @@ struct kernfs_root { rwlock_t kernfs_rename_lock; struct rcu_head rcu; + + struct simple_xattr_cache xa_cache; }; /* +1 to avoid triggering overflow warning when negating it */ @@ -211,4 +213,24 @@ extern const struct inode_operations kernfs_symlink_iops; * kernfs locks */ extern struct kernfs_global_locks *kernfs_locks; + +/* Hashed mutex helpers - protect per-node data structures */ +static inline struct mutex *kernfs_node_lock_ptr(struct kernfs_node *kn) +{ + int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS); + + return &kernfs_locks->node_mutex[idx]; +} + +static inline struct mutex *kernfs_node_lock(struct kernfs_node *kn) +{ + struct mutex *lock = kernfs_node_lock_ptr(kn); + + mutex_lock(lock); + return lock; +} + +DEFINE_CLASS(kernfs_node_lock, struct mutex *, + mutex_unlock(_T), kernfs_node_lock(kn), struct kernfs_node *kn) + #endif /* __KERNFS_INTERNAL_H */ diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index 6e3217b6e4811..f183a96778b9a 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -446,7 +446,7 @@ static void __init kernfs_mutex_init(void) int count; for (count = 0; count < NR_KERNFS_LOCKS; count++) - mutex_init(&kernfs_locks->open_file_mutex[count]); + mutex_init(&kernfs_locks->node_mutex[count]); } static void __init kernfs_lock_init(void) diff --git a/fs/libfs.c b/fs/libfs.c index 1bbea5e7bae37..e8226b9e1bc8c 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -736,6 +736,7 @@ struct pseudo_fs_context *init_pseudo(struct fs_context *fc, fc->fs_private = ctx; fc->ops = &pseudo_fs_context_ops; fc->sb_flags |= SB_NOUSER; + fc->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; fc->global = true; } return ctx; diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h index 1db6cb3525425..9aa6acb43f9a4 100644 --- a/fs/lockd/lockd.h +++ b/fs/lockd/lockd.h @@ -52,6 +52,14 @@ */ #define LOCKD_DFLT_TIMEO 10 +/* + * Number of leading bytes of nfs_fh.data that file_hash() + * digests when bucketing nlm_files[]. Sized for historical + * NFSv2 handles; nfs_fh.data must be initialized at least + * this far before lookup, regardless of fh.size. + */ +#define LOCKD_FH_HASH_SIZE 32 + /* error codes new to NLMv4 */ #define nlm4_deadlock cpu_to_be32(NLM_DEADLCK) #define nlm4_rofs cpu_to_be32(NLM_ROFS) diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index 41cab858de578..f3ba2615ae77d 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -157,6 +157,9 @@ nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host, return nlm_lck_denied_nolocks; lock->fh.size = xdr_lock->fh.len; memcpy(lock->fh.data, xdr_lock->fh.data, xdr_lock->fh.len); + if (xdr_lock->fh.len < LOCKD_FH_HASH_SIZE) + memset(lock->fh.data + xdr_lock->fh.len, 0, + LOCKD_FH_HASH_SIZE - xdr_lock->fh.len); lock->oh.len = xdr_lock->oh.len; lock->oh.data = xdr_lock->oh.data; @@ -513,12 +516,12 @@ nlm4svc_proc_unlock(struct svc_rqst *rqstp) * nlm4_res NLMPROC4_GRANTED(nlm4_testargs) = 5; * * Permissible procedure status codes: - * %NLM4_GRANTED: The requested lock was granted. - * %NLM4_DENIED: The server could not allocate the resources - * needed to process the request. - * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is - * re-establishing existing locks, and is not - * yet ready to accept normal service requests. + * %NLM4_GRANTED: The granted lock was accepted. + * %NLM4_DENIED: The procedure failed, possibly due to + * internal resource constraints. + * %NLM4_DENIED_GRACE_PERIOD: The client host recently restarted and + * its NLM is re-establishing existing locks, + * so it is not yet ready to accept callbacks. */ static __be32 nlm4svc_proc_granted(struct svc_rqst *rqstp) @@ -669,6 +672,8 @@ __nlm4svc_proc_lock_msg(struct svc_rqst *rqstp, struct nlm_res *resp) resp->status = nlmsvc_lock(rqstp, file, host, &argp->lock, argp->xdrgen.block, &resp->cookie, argp->xdrgen.reclaim); + if (resp->status == nlm__int__deadlock) + resp->status = nlm4_deadlock; nlmsvc_release_lockowner(&argp->lock); out: @@ -697,7 +702,7 @@ static __be32 nlm4svc_proc_lock_msg(struct svc_rqst *rqstp) struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp; struct nlm_host *host; - host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, true); + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); if (!host) return rpc_system_err; diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index c0a3487719e28..110e186802b6c 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -49,7 +49,7 @@ static inline __be32 cast_status(__be32 status) status = nlm_lck_denied_nolocks; break; default: - if (be32_to_cpu(status) >= 30000) + if (be32_to_cpu(status) > be32_to_cpu(nlm__int__drop_reply)) pr_warn_once("lockd: unhandled internal status %u\n", be32_to_cpu(status)); break; diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c index 9da9d6e0b42e2..c7945282d479f 100644 --- a/fs/lockd/svcsubs.c +++ b/fs/lockd/svcsubs.c @@ -17,7 +17,6 @@ #include #include #include -#include #include "lockd.h" #include "share.h" @@ -67,7 +66,7 @@ static inline unsigned int file_hash(struct nfs_fh *f) { unsigned int tmp=0; int i; - for (i=0; idata[i]; return tmp & (FILE_NRHASH - 1); } @@ -150,6 +149,8 @@ nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result, mutex_lock(&file->f_mutex); nfserr = nlm_do_fopen(rqstp, file, mode); mutex_unlock(&file->f_mutex); + if (nfserr) + goto out_unlock; goto found; } nlm_debug_print_fh("creating file for", &lock->fh); @@ -166,7 +167,7 @@ nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result, nfserr = nlm_do_fopen(rqstp, file, mode); if (nfserr) - goto out_unlock; + goto out_free; hlist_add_head(&file->f_list, &nlm_files[hash]); diff --git a/fs/minix/minix.h b/fs/minix/minix.h index f2025c9b58252..9e52d4302f0d2 100644 --- a/fs/minix/minix.h +++ b/fs/minix/minix.h @@ -97,7 +97,7 @@ static inline struct minix_inode_info *minix_i(struct inode *inode) static inline unsigned minix_blocks_needed(unsigned bits, unsigned blocksize) { - return DIV_ROUND_UP(bits, blocksize * 8); + return DIV_ROUND_UP_POW2(bits, blocksize * 8); } #if defined(CONFIG_MINIX_FS_NATIVE_ENDIAN) && \ diff --git a/fs/namei.c b/fs/namei.c index 4787244ca4a75..0da20b387e96e 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -4735,6 +4735,10 @@ int vfs_tmpfile(struct mnt_idmap *idmap, int error; int open_flag = file->f_flags; + /* A tmpfile is I_LINKABLE, so guard its owner like may_o_create(). */ + if (!fsuidgid_has_mapping(dir->i_sb, idmap)) + return -EOVERFLOW; + /* we want directory to be writable */ error = inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC); if (error) diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index f59a70f3a086b..2b42758e01ec9 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -98,7 +98,12 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) goto abandon; } - list_for_each_continue(next, &stream->subrequests) { + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); if (subreq->start + subreq->transferred != start + len || test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index c03c7cc45e471..d0d884731dc5d 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -106,7 +106,7 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, _enter("R=%x", wreq->debug_id); ictx = netfs_inode(wreq->inode); - if (is_cacheable && netfs_is_cache_enabled(ictx)) + if (is_cacheable) fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx)); if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE) < 0) goto nomem; @@ -588,8 +588,10 @@ int netfs_writepages(struct address_space *mapping, } error = netfs_write_folio(wreq, wbc, folio); - if (error < 0) - break; + if (error == -ENOMEM) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + } } while ((folio = writeback_iter(mapping, wbc, folio, &error))); netfs_end_issue_write(wreq); @@ -602,7 +604,14 @@ int netfs_writepages(struct address_space *mapping, return error; couldnt_start: - netfs_kill_dirty_pages(mapping, wbc, folio); + if (error == -ENOMEM) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + folio = writeback_iter(mapping, wbc, folio, &error); + WARN_ON_ONCE(folio != NULL); + } else { + netfs_kill_dirty_pages(mapping, wbc, folio); + } out: mutex_unlock(&ictx->wb_lock); _leave(" = %d", error); @@ -628,6 +637,7 @@ struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len } wreq->io_streams[0].avail = true; + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); trace_netfs_write(wreq, netfs_write_trace_writethrough); return wreq; } diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index 32735abfa03f0..058bc7a166a59 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -72,7 +72,12 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) return; - list_for_each_continue(next, &stream->subrequests) { + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); if (subreq->start + subreq->transferred != start + len || test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index 4ea9221ded426..10f2354ba3048 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -257,6 +257,7 @@ static u32 initiate_file_draining(struct nfs_client *clp, struct pnfs_layout_hdr *lo; u32 rv = NFS4ERR_NOMATCHING_LAYOUT; LIST_HEAD(free_me_list); + bool return_range = false; ino = nfs_layout_find_inode(clp, &args->cbl_fh, &args->cbl_stateid); if (IS_ERR(ino)) { @@ -301,13 +302,13 @@ static u32 initiate_file_draining(struct nfs_client *clp, /* Embrace your forgetfulness! */ rv = NFS4ERR_NOMATCHING_LAYOUT; - if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) { - NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, - &args->cbl_range); - } + return_range = true; } unlock: spin_unlock(&ino->i_lock); + if (return_range && NFS_SERVER(ino)->pnfs_curr_ld->return_range) + NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, + &args->cbl_range); pnfs_free_lseg_list(&free_me_list); /* Free all lsegs that are attached to commit buckets */ nfs_commit_inode(ino, 0); diff --git a/fs/nfs/client.c b/fs/nfs/client.c index be02bb227741d..0781b15e7e05c 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -1074,6 +1074,7 @@ struct nfs_server *nfs_alloc_server(void) server->io_stats = nfs_alloc_iostats(); if (!server->io_stats) { + ida_free(&s_sysfs_ids, server->s_sysfs_id); kfree(server); return NULL; } diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 122fb3f14ffb8..9546d2195c25f 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -173,6 +173,7 @@ int nfs4_check_delegation(struct inode *inode, fmode_t type) static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_stateid *stateid) { struct inode *inode = state->inode; + struct nfs_inode *nfsi = NFS_I(inode); struct file_lock *fl; struct file_lock_context *flctx = locks_inode_context(inode); struct list_head *list; @@ -182,6 +183,9 @@ static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_state goto out; list = &flctx->flc_posix; + + /* Guard against reclaim and new lock/unlock calls */ + down_write(&nfsi->rwsem); spin_lock(&flctx->flc_lock); restart: for_each_file_lock(fl, list) { @@ -189,8 +193,10 @@ static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_state continue; spin_unlock(&flctx->flc_lock); status = nfs4_lock_delegation_recall(fl, state, stateid); - if (status < 0) + if (status < 0) { + up_write(&nfsi->rwsem); goto out; + } spin_lock(&flctx->flc_lock); } if (list == &flctx->flc_posix) { @@ -198,6 +204,7 @@ static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_state goto restart; } spin_unlock(&flctx->flc_lock); + up_write(&nfsi->rwsem); out: return status; } diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index e9ce1883288c5..cb7ddaefdfb22 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -3344,6 +3344,8 @@ static int nfs_open_permission_mask(int openflags) mask |= MAY_READ; if ((openflags & O_ACCMODE) != O_RDONLY) mask |= MAY_WRITE; + if (openflags & O_TRUNC) + mask |= MAY_WRITE; } return mask; diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index e85380e3b11d7..70f2cbd46960f 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -778,6 +778,8 @@ filelayout_alloc_lseg(struct pnfs_layout_hdr *layoutid, static bool filelayout_lseg_is_striped(const struct nfs4_filelayout_segment *flseg) { + if (flseg->dsaddr) + return flseg->dsaddr->stripe_count > 1; return flseg->num_fh > 1; } diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 8b1559171fe39..2e7f857d5a8cb 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -551,6 +551,10 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, if (!p) goto out_err_free; fh_count = be32_to_cpup(p); + if (fh_count == 0) { + rc = -EINVAL; + goto out_err_free; + } dss_info->fh_versions = kzalloc_objs(struct nfs_fh, fh_count, gfp_flags); @@ -632,6 +636,9 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, if (!p) goto out_sort_mirrors; fls->flags = be32_to_cpup(p); + if (fls->flags & FF_FLAGS_NO_IO_THRU_MDS) + set_bit(NFS4_FF_HDR_NO_IO_THRU_MDS, + &FF_LAYOUT_FROM_HDR(lh)->flags); p = xdr_inline_decode(&stream, 4); if (!p) @@ -1181,6 +1188,16 @@ ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio, 0, NFS4_MAX_UINT64, IOMODE_RW, NFS_I(pgio->pg_inode)->layout, pgio->pg_lseg); + if (NFS_I(pgio->pg_inode)->layout && + ff_layout_hdr_no_fallback_to_mds(NFS_I(pgio->pg_inode)->layout)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: no current lseg but the server's + * policy forbids MDS fallback. Surface -EAGAIN so writeback + * retries rather than silently issuing the WRITE via MDS. + */ + pgio->pg_error = -EAGAIN; + goto out; + } /* no lseg means that pnfs is not in use, so no mirroring here */ nfs_pageio_reset_write_mds(pgio); out: @@ -2200,6 +2217,14 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) out_failed: if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; + if (ff_layout_no_fallback_to_mds(lseg)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, + * never fall through to MDS I/O. + */ + pnfs_error_mark_layout_for_return(hdr->inode, lseg); + return PNFS_TRY_AGAIN; + } trace_pnfs_mds_fallback_read_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, IOMODE_READ, NFS_I(hdr->inode)->layout, lseg); @@ -2285,6 +2310,14 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) out_failed: if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; + if (ff_layout_no_fallback_to_mds(lseg)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, + * never fall through to MDS I/O. + */ + pnfs_error_mark_layout_for_return(hdr->inode, lseg); + return PNFS_TRY_AGAIN; + } trace_pnfs_mds_fallback_write_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, IOMODE_RW, NFS_I(hdr->inode)->layout, lseg); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h index 17a008c8e97ce..a5bd00f69e824 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.h +++ b/fs/nfs/flexfilelayout/flexfilelayout.h @@ -112,12 +112,16 @@ struct nfs4_ff_layout_segment { struct nfs4_ff_layout_mirror *mirror_array[] __counted_by(mirror_array_cnt); }; +/* nfs4_flexfile_layout::flags bit indices */ +#define NFS4_FF_HDR_NO_IO_THRU_MDS 0 /* any lseg has had FF_FLAGS_NO_IO_THRU_MDS */ + struct nfs4_flexfile_layout { struct pnfs_layout_hdr generic_hdr; struct pnfs_ds_commit_info commit_info; struct list_head mirrors; struct list_head error_list; /* nfs4_ff_layout_ds_err */ ktime_t last_report_time; /* Layoutstat report times */ + unsigned long flags; }; struct nfs4_flexfile_layoutreturn_args { @@ -184,6 +188,18 @@ ff_layout_no_fallback_to_mds(struct pnfs_layout_segment *lseg) return FF_LAYOUT_LSEG(lseg)->flags & FF_FLAGS_NO_IO_THRU_MDS; } +/* + * Sticky hdr-level mirror of FF_FLAGS_NO_IO_THRU_MDS so callers that have + * no current lseg (e.g. between LAYOUTRETURN and the next LAYOUTGET) can + * still honor the no-MDS-fallback policy. + */ +static inline bool +ff_layout_hdr_no_fallback_to_mds(struct pnfs_layout_hdr *lo) +{ + return test_bit(NFS4_FF_HDR_NO_IO_THRU_MDS, + &FF_LAYOUT_FROM_HDR(lo)->flags); +} + static inline bool ff_layout_no_read_on_rw(struct pnfs_layout_segment *lseg) { diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index c105882edd16e..1967de7d1dff9 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -1769,7 +1769,9 @@ struct file_system_type nfs_fs_type = { .init_fs_context = nfs_init_fs_context, .parameters = nfs_fs_parameters, .kill_sb = nfs_kill_super, - .fs_flags = FS_RENAME_DOES_D_MOVE|FS_BINARY_MOUNTDATA, + .fs_flags = FS_RENAME_DOES_D_MOVE | + FS_BINARY_MOUNTDATA | + FS_USERNS_DELEGATABLE, }; MODULE_ALIAS_FS("nfs"); EXPORT_SYMBOL_GPL(nfs_fs_type); @@ -1781,7 +1783,9 @@ struct file_system_type nfs4_fs_type = { .init_fs_context = nfs_init_fs_context, .parameters = nfs_fs_parameters, .kill_sb = nfs_kill_super, - .fs_flags = FS_RENAME_DOES_D_MOVE|FS_BINARY_MOUNTDATA, + .fs_flags = FS_RENAME_DOES_D_MOVE | + FS_BINARY_MOUNTDATA | + FS_USERNS_DELEGATABLE, }; MODULE_ALIAS_FS("nfs4"); MODULE_ALIAS("nfs4"); diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 18d46b0e71ddc..1d5d62f88ddef 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -847,17 +847,19 @@ void nfs_super_set_maxbytes(struct super_block *sb, __u64 maxfilesize) } /* - * Record the page as unstable (an extra writeback period) and mark its - * inode as dirty. + * Record the request's range as unstable (an extra writeback period) and + * mark its inode as dirty. */ -static inline void nfs_folio_mark_unstable(struct folio *folio, +static inline void nfs_folio_mark_unstable(struct nfs_page *req, struct nfs_commit_info *cinfo) { + struct folio *folio = nfs_page_to_folio(req); + if (folio && !cinfo->dreq) { struct inode *inode = folio->mapping->host; - long nr = folio_nr_pages(folio); + long nr = DIV_ROUND_UP(req->wb_bytes, PAGE_SIZE); - /* This page is really still in write-back - just that the + /* This range is really still in write-back - just that the * writeback is happening on the server now. */ node_stat_mod_folio(folio, NR_WRITEBACK, nr); diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index a9b8d482d2894..4db27f4eb01e9 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -5302,10 +5302,9 @@ static struct dentry *nfs4_proc_mkdir(struct inode *dir, struct dentry *dentry, do { alias = _nfs4_proc_mkdir(dir, dentry, sattr, label, &err); trace_nfs4_mkdir(dir, &dentry->d_name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, &exception); if (err) - alias = ERR_PTR(nfs4_handle_exception(NFS_SERVER(dir), - err, - &exception)); + alias = ERR_PTR(err); } while (exception.retry); nfs4_label_release_security(label); @@ -7085,7 +7084,6 @@ static void nfs4_locku_done(struct rpc_task *task, void *data) switch (task->tk_status) { case 0: renew_lease(calldata->server, calldata->timestamp); - locks_lock_inode_wait(calldata->lsp->ls_state->inode, &calldata->fl); if (nfs4_update_lock_stateid(calldata->lsp, &calldata->res.stateid)) break; @@ -7353,11 +7351,6 @@ static void nfs4_lock_done(struct rpc_task *task, void *calldata) case 0: renew_lease(NFS_SERVER(d_inode(data->ctx->dentry)), data->timestamp); - if (data->arg.new_lock && !data->cancelled) { - data->fl.c.flc_flags &= ~(FL_SLEEP | FL_ACCESS); - if (locks_lock_inode_wait(lsp->ls_state->inode, &data->fl) < 0) - goto out_restart; - } if (data->arg.new_lock_owner != 0) { nfs_confirm_seqid(&lsp->ls_seqid, 0); nfs4_stateid_copy(&lsp->ls_stateid, &data->res.stateid); @@ -7468,11 +7461,10 @@ static int _nfs4_do_setlk(struct nfs4_state *state, int cmd, struct file_lock *f msg.rpc_argp = &data->arg; msg.rpc_resp = &data->res; task_setup_data.callback_data = data; - if (recovery_type > NFS_LOCK_NEW) { - if (recovery_type == NFS_LOCK_RECLAIM) - data->arg.reclaim = NFS_LOCK_RECLAIM; - } else - data->arg.new_lock = 1; + + if (recovery_type == NFS_LOCK_RECLAIM) + data->arg.reclaim = NFS_LOCK_RECLAIM; + task = rpc_run_task(&task_setup_data); if (IS_ERR(task)) return PTR_ERR(task); @@ -7582,6 +7574,13 @@ static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock up_read(&nfsi->rwsem); mutex_unlock(&sp->so_delegreturn_mutex); status = _nfs4_do_setlk(state, cmd, request, NFS_LOCK_NEW); + if (status) + goto out; + + down_read(&nfsi->rwsem); + request->c.flc_flags &= ~(FL_SLEEP | FL_ACCESS); + status = locks_lock_inode_wait(state->inode, request); + up_read(&nfsi->rwsem); out: request->c.flc_flags = flags; return status; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 743467e9ba202..b7dcf58f21c39 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1463,8 +1463,6 @@ _pnfs_return_layout(struct inode *ino) pnfs_clear_layoutcommit(ino, &tmp_list); pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); - if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) - NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); /* Don't send a LAYOUTRETURN if list was initially empty */ if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || @@ -1476,6 +1474,8 @@ _pnfs_return_layout(struct inode *ino) send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); spin_unlock(&ino->i_lock); + if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) + NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); if (send) status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 0); @@ -2229,11 +2229,11 @@ pnfs_update_layout(struct inode *ino, dprintk("%s wait for layoutreturn\n", __func__); lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); if (!IS_ERR(lseg)) { - pnfs_put_layout_hdr(lo); dprintk("%s retrying\n", __func__); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); + pnfs_put_layout_hdr(lo); goto lookup_again; } trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 12632a706da88..648c95b78eeab 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -1075,14 +1075,14 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags) /* r_netid */ nlen = xdr_stream_decode_string_dup(xdr, &netid, XDR_MAX_NETOBJ, gfp_flags); - if (unlikely(nlen < 0)) + if (unlikely(nlen <= 0)) goto out_err; /* r_addr: ip/ip6addr with port in dec octets - see RFC 5665 */ /* port is ".ABC.DEF", 8 chars max */ rlen = xdr_stream_decode_string_dup(xdr, &buf, INET6_ADDRSTRLEN + IPV6_SCOPE_ID_LEN + 8, gfp_flags); - if (unlikely(rlen < 0)) + if (unlikely(rlen <= 0)) goto out_free_netid; /* replace port '.' with '-' */ @@ -1199,7 +1199,7 @@ pnfs_layout_mark_request_commit(struct nfs_page *req, nfs_request_add_commit_list_locked(req, list, cinfo); mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); - nfs_folio_mark_unstable(nfs_page_to_folio(req), cinfo); + nfs_folio_mark_unstable(req, cinfo); return; out_resched: mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); diff --git a/fs/nfs/read.c b/fs/nfs/read.c index e1fe78d7b8d0f..2b70bd2b934b9 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -132,10 +132,32 @@ static void nfs_readpage_release(struct nfs_page *req, int error) static void nfs_page_group_set_uptodate(struct nfs_page *req) { - if (nfs_page_group_sync_on_bit(req, PG_UPTODATE)) + bool uptodate = false; + + nfs_page_group_lock(req); + if (!test_bit(PG_READ_FAILED, &req->wb_head->wb_flags) && + nfs_page_group_sync_on_bit_locked(req, PG_UPTODATE)) + uptodate = true; + nfs_page_group_unlock(req); + + if (uptodate) folio_mark_uptodate(nfs_page_to_folio(req)); } +static void nfs_page_group_mark_read_failed(struct nfs_page *req) +{ + struct nfs_page *tmp; + + nfs_page_group_lock(req); + set_bit(PG_READ_FAILED, &req->wb_head->wb_flags); + tmp = req; + do { + clear_bit(PG_UPTODATE, &tmp->wb_flags); + tmp = tmp->wb_this_page; + } while (tmp != req); + nfs_page_group_unlock(req); +} + static void nfs_read_completion(struct nfs_pgio_header *hdr) { unsigned long bytes = 0; @@ -172,6 +194,7 @@ static void nfs_read_completion(struct nfs_pgio_header *hdr) if (bytes <= hdr->good_bytes) nfs_page_group_set_uptodate(req); else { + nfs_page_group_mark_read_failed(req); error = hdr->error; xchg(&nfs_req_openctx(req)->error, error); } diff --git a/fs/nfs/write.c b/fs/nfs/write.c index d7c399763ad94..f7a5fb8140c47 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -807,7 +807,7 @@ nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo) mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo); mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); - nfs_folio_mark_unstable(nfs_page_to_folio(req), cinfo); + nfs_folio_mark_unstable(req, cinfo); } EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); @@ -866,10 +866,12 @@ nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, nfs_request_add_commit_list(req, cinfo); } -static void nfs_folio_clear_commit(struct folio *folio) +static void nfs_folio_clear_commit(struct nfs_page *req) { + struct folio *folio = nfs_page_to_folio(req); + if (folio) { - long nr = folio_nr_pages(folio); + long nr = DIV_ROUND_UP(req->wb_bytes, PAGE_SIZE); node_stat_mod_folio(folio, NR_WRITEBACK, -nr); bdi_wb_stat_mod(folio->mapping->host, WB_WRITEBACK, -nr); @@ -889,7 +891,7 @@ static void nfs_clear_request_commit(struct nfs_commit_info *cinfo, nfs_request_remove_commit_list(req, cinfo); } mutex_unlock(&NFS_I(inode)->commit_mutex); - nfs_folio_clear_commit(nfs_page_to_folio(req)); + nfs_folio_clear_commit(req); } } @@ -1741,7 +1743,7 @@ void nfs_retry_commit(struct list_head *page_list, req = nfs_list_entry(page_list->next); nfs_list_remove_request(req); nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx); - nfs_folio_clear_commit(nfs_page_to_folio(req)); + nfs_folio_clear_commit(req); nfs_unlock_and_release_request(req); } } @@ -1813,7 +1815,7 @@ static void nfs_commit_release_pages(struct nfs_commit_data *data) req = nfs_list_entry(data->pages.next); nfs_list_remove_request(req); folio = nfs_page_to_folio(req); - nfs_folio_clear_commit(folio); + nfs_folio_clear_commit(req); dprintk("NFS: commit (%s/%llu %d@%lld)", nfs_req_openctx(req)->dentry->d_sb->s_id, diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 665153f1720e0..0baa58d1dbfcd 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -36,30 +36,19 @@ * second map contains a reference to the entry in the first map. */ -static struct workqueue_struct *nfsd_export_wq; - #define EXPKEY_HASHBITS 8 #define EXPKEY_HASHMAX (1 << EXPKEY_HASHBITS) #define EXPKEY_HASHMASK (EXPKEY_HASHMAX -1) -static void expkey_release(struct work_struct *work) +static void expkey_put(struct kref *ref) { - struct svc_expkey *key = container_of(to_rcu_work(work), - struct svc_expkey, ek_rwork); + struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); if (test_bit(CACHE_VALID, &key->h.flags) && !test_bit(CACHE_NEGATIVE, &key->h.flags)) path_put(&key->ek_path); auth_domain_put(key->ek_client); - kfree(key); -} - -static void expkey_put(struct kref *ref) -{ - struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); - - INIT_RCU_WORK(&key->ek_rwork, expkey_release); - queue_rcu_work(nfsd_export_wq, &key->ek_rwork); + kfree_rcu(key, ek_rcu); } static int expkey_upcall(struct cache_detail *cd, struct cache_head *h) @@ -364,13 +353,11 @@ static void export_stats_destroy(struct export_stats *stats) EXP_STATS_COUNTERS_NUM); } -static void svc_export_release(struct work_struct *work) +static void svc_export_release(struct rcu_head *rcu_head) { - struct svc_export *exp = container_of(to_rcu_work(work), - struct svc_export, ex_rwork); + struct svc_export *exp = container_of(rcu_head, struct svc_export, + ex_rcu); - path_put(&exp->ex_path); - auth_domain_put(exp->ex_client); nfsd4_fslocs_free(&exp->ex_fslocs); export_stats_destroy(exp->ex_stats); kfree(exp->ex_stats); @@ -382,8 +369,9 @@ static void svc_export_put(struct kref *ref) { struct svc_export *exp = container_of(ref, struct svc_export, h.ref); - INIT_RCU_WORK(&exp->ex_rwork, svc_export_release); - queue_rcu_work(nfsd_export_wq, &exp->ex_rwork); + path_put(&exp->ex_path); + auth_domain_put(exp->ex_client); + call_rcu(&exp->ex_rcu, svc_export_release); } static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h) @@ -1492,36 +1480,6 @@ const struct seq_operations nfs_exports_op = { .show = e_show, }; -/** - * nfsd_export_wq_init - allocate the export release workqueue - * - * Called once at module load. The workqueue runs deferred svc_export and - * svc_expkey release work scheduled by queue_rcu_work() in the cache put - * callbacks. - * - * Return values: - * %0: workqueue allocated - * %-ENOMEM: allocation failed - */ -int nfsd_export_wq_init(void) -{ - nfsd_export_wq = alloc_workqueue("nfsd_export", WQ_UNBOUND, 0); - if (!nfsd_export_wq) - return -ENOMEM; - return 0; -} - -/** - * nfsd_export_wq_shutdown - drain and free the export release workqueue - * - * Called once at module unload. Per-namespace teardown in - * nfsd_export_shutdown() has already drained all deferred work. - */ -void nfsd_export_wq_shutdown(void) -{ - destroy_workqueue(nfsd_export_wq); -} - /* * Initialize the exports module. */ @@ -1583,9 +1541,6 @@ nfsd_export_shutdown(struct net *net) cache_unregister_net(nn->svc_expkey_cache, net); cache_unregister_net(nn->svc_export_cache, net); - /* Drain deferred export and expkey release work. */ - rcu_barrier(); - flush_workqueue(nfsd_export_wq); cache_destroy_net(nn->svc_expkey_cache, net); cache_destroy_net(nn->svc_export_cache, net); svcauth_unix_purge(net); diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index b053993745749..d2b09cd761453 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -76,7 +75,7 @@ struct svc_export { u32 ex_layout_types; struct nfsd4_deviceid_map *ex_devid_map; struct cache_detail *cd; - struct rcu_work ex_rwork; + struct rcu_head ex_rcu; unsigned long ex_xprtsec_modes; struct export_stats *ex_stats; }; @@ -93,7 +92,7 @@ struct svc_expkey { u32 ek_fsid[6]; struct path ek_path; - struct rcu_work ek_rwork; + struct rcu_head ek_rcu; }; #define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC)) @@ -111,8 +110,6 @@ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, /* * Function declarations */ -int nfsd_export_wq_init(void); -void nfsd_export_wq_shutdown(void); int nfsd_export_init(struct net *); void nfsd_export_shutdown(struct net *); void nfsd_export_flush(struct net *); diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 0ac538c761800..76305b86c1a95 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -131,10 +131,7 @@ static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp) resp->status = fh_getattr(fh, &resp->stat); out: - /* argp->acl_{access,default} may have been allocated in - nfssvc_decode_setaclargs. */ - posix_acl_release(argp->acl_access); - posix_acl_release(argp->acl_default); + /* argp->acl_{access,default} are released in nfsaclsvc_release_setacl. */ return rpc_success; out_drop_lock: @@ -310,6 +307,16 @@ static void nfsaclsvc_release_access(struct svc_rqst *rqstp) fh_put(&resp->fh); } +static void nfsaclsvc_release_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(argp->acl_access); + posix_acl_release(argp->acl_default); +} + #define ST 1 /* status*/ #define AT 21 /* attributes */ #define pAT (1+AT) /* post attributes - conditional */ @@ -343,7 +350,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_func = nfsacld_proc_setacl, .pc_decode = nfsaclsvc_decode_setaclargs, .pc_encode = nfssvc_encode_attrstatres, - .pc_release = nfssvc_release_attrstat, + .pc_release = nfsaclsvc_release_setacl, .pc_argsize = sizeof(struct nfsd3_setaclargs), .pc_argzero = sizeof(struct nfsd3_setaclargs), .pc_ressize = sizeof(struct nfsd_attrstat), diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index 7b5433bd30197..e87731380be85 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -118,10 +118,7 @@ static __be32 nfsd3_proc_setacl(struct svc_rqst *rqstp) out_errno: resp->status = nfserrno(error); out: - /* argp->acl_{access,default} may have been allocated in - nfs3svc_decode_setaclargs. */ - posix_acl_release(argp->acl_access); - posix_acl_release(argp->acl_default); + /* argp->acl_{access,default} are released in nfs3svc_release_setacl. */ return rpc_success; } @@ -223,6 +220,16 @@ static void nfs3svc_release_getacl(struct svc_rqst *rqstp) posix_acl_release(resp->acl_default); } +static void nfs3svc_release_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(argp->acl_access); + posix_acl_release(argp->acl_default); +} + #define ST 1 /* status*/ #define AT 21 /* attributes */ #define pAT (1+AT) /* post attributes - conditional */ @@ -256,7 +263,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_func = nfsd3_proc_setacl, .pc_decode = nfs3svc_decode_setaclargs, .pc_encode = nfs3svc_encode_setaclres, - .pc_release = nfs3svc_release_fhandle, + .pc_release = nfs3svc_release_setacl, .pc_argsize = sizeof(struct nfsd3_setaclargs), .pc_argzero = sizeof(struct nfsd3_setaclargs), .pc_ressize = sizeof(struct nfsd3_attrstat), diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index 69e41105efdd5..8e2929cffa7c6 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -264,10 +264,12 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, ls->ls_file = find_any_file(fp); BUG_ON(!ls->ls_file); + ls->ls_fenced = false; + ls->ls_fence_delay = 0; + INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker); + if (nfsd4_layout_setlease(ls)) { - nfsd_file_put(ls->ls_file); - put_nfs4_file(fp); - kmem_cache_free(nfs4_layout_stateid_cache, ls); + nfs4_put_stid(stp); return NULL; } @@ -280,10 +282,6 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, list_add(&ls->ls_perfile, &fp->fi_lo_states); spin_unlock(&fp->fi_lock); - ls->ls_fenced = false; - ls->ls_fence_delay = 0; - INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker); - trace_nfsd_layoutstate_alloc(&ls->ls_stid.sc_stateid); return ls; } diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index ab39ec8854405..8561540ab2db9 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -253,7 +253,7 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, .na_iattr = iap, .na_seclabel = &open->op_label, }; - struct dentry *parent, *child; + struct dentry *parent, *child = ERR_PTR(-EINVAL); __u32 v_mtime, v_atime; struct inode *inode; __be32 status; @@ -277,10 +277,14 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, if (open->op_acl) { if (open->op_dpacl || open->op_pacl) { status = nfserr_inval; - goto out_write; + goto out; + } + if (is_create_with_attrs(open)) { + status = nfsd4_acl_to_attr(NF4REG, open->op_acl, + &attrs); + if (status) + goto out; } - if (is_create_with_attrs(open)) - nfsd4_acl_to_attr(NF4REG, open->op_acl, &attrs); } else if (is_create_with_attrs(open)) { /* The dpacl and pacl will get released by nfsd_attrs_free(). */ attrs.na_dpacl = open->op_dpacl; @@ -293,7 +297,7 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, &QSTR_LEN(open->op_fname, open->op_fnamelen)); if (IS_ERR(child)) { status = nfserrno(PTR_ERR(child)); - goto out_write; + goto out; } if (d_really_is_negative(child)) { @@ -407,7 +411,6 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, out: end_creating(child); nfsd_attrs_free(&attrs); -out_write: fh_drop_write(fhp); return status; } @@ -837,7 +840,7 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out_aftermask; if (create->cr_acl) { - if (create->cr_dpacl || create->cr_pacl) { + if (attrs.na_dpacl || attrs.na_pacl) { status = nfserr_inval; goto out_aftermask; } @@ -1470,7 +1473,7 @@ void nfsd4_async_copy_reaper(struct nfsd_net *nn) list_for_each_safe(pos, next, &clp->async_copies) { copy = list_entry(pos, struct nfsd4_copy, copies); if (test_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags)) { - if (--copy->cp_ttl) { + if (!--copy->cp_ttl) { list_del_init(©->copies); list_add(©->copies, &reaplist); } diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index b338473d6e52d..6ea25a52d2f4e 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -718,7 +718,8 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, return PTR_ERR(namecopy); name.data = namecopy; name.len = namelen; - get_user(princhashlen, &ci->cc_princhash.cp_len); + if (get_user(princhashlen, &ci->cc_princhash.cp_len)) + return -EFAULT; if (princhashlen > 0) { princhashcopy = memdup_user( &ci->cc_princhash.cp_data, diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 6837b63d98645..523db702464c1 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1376,7 +1376,8 @@ static void destroy_delegation(struct nfs4_delegation *dp) * stateid or it's called from a laundromat thread (nfsd4_landromat()) that * determined that this specific state has expired and needs to be revoked * (both mark state with the appropriate stid sc_status mode). It is also - * assumed that a reference was taken on the @dp state. + * assumed that a reference was taken on the @dp state. This function + * consumes that reference. * * If this function finds that the @dp state is SC_STATUS_FREED it means * that a FREE_STATEID operation for this stateid has been processed and @@ -1839,6 +1840,10 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) mutex_unlock(&stp->st_mutex); break; case SC_TYPE_DELEG: + /* Extra reference guards against concurrent + * FREE_STATEID; revoke_delegation() consumes + * it, otherwise release it directly. + */ refcount_inc(&stid->sc_count); dp = delegstateid(stid); spin_lock(&nn->deleg_lock); @@ -1848,6 +1853,8 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) spin_unlock(&nn->deleg_lock); if (dp) revoke_delegation(dp); + else + nfs4_put_stid(stid); break; case SC_TYPE_LAYOUT: ls = layoutstateid(stid); @@ -5049,6 +5056,7 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) { struct nfs4_client *cl = s->sc_client; LIST_HEAD(reaplist); + struct nfs4_layout_stateid *ls; struct nfs4_ol_stateid *stp; struct nfs4_delegation *dp; bool unhashed; @@ -5074,6 +5082,12 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) spin_unlock(&cl->cl_lock); nfs4_put_stid(s); break; + case SC_TYPE_LAYOUT: + ls = layoutstateid(s); + list_del_init(&ls->ls_perclnt); + spin_unlock(&cl->cl_lock); + nfs4_put_stid(s); + break; default: spin_unlock(&cl->cl_lock); } @@ -5169,6 +5183,7 @@ find_or_alloc_open_stateowner(unsigned int strhashval, struct nfsd4_open *open, /* Replace unconfirmed owners without checking for replay. */ release_openowner(oo); oo = NULL; + goto retry; } if (oo) { if (new) diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 2a0946c630e1d..2da83053fa106 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -2008,10 +2008,11 @@ static __be32 nfsd4_decode_secinfo_no_name(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) { struct nfsd4_secinfo_no_name *sin = &u->secinfo_no_name; + + sin->sin_exp = NULL; if (xdr_stream_decode_u32(argp->xdr, &sin->sin_style) < 0) return nfserr_bad_xdr; - sin->sin_exp = NULL; return nfs_ok; } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 04e3954d54bd9..7cb8f95067453 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -2320,12 +2320,9 @@ static int __init init_nfsd(void) if (retval) goto out_free_pnfs; nfsd_lockd_init(); /* lockd->nfsd callbacks */ - retval = nfsd_export_wq_init(); - if (retval) - goto out_free_lockd; retval = register_pernet_subsys(&nfsd_net_ops); if (retval < 0) - goto out_free_export_wq; + goto out_free_lockd; retval = register_cld_notifier(); if (retval) goto out_free_subsys; @@ -2354,8 +2351,6 @@ static int __init init_nfsd(void) unregister_cld_notifier(); out_free_subsys: unregister_pernet_subsys(&nfsd_net_ops); -out_free_export_wq: - nfsd_export_wq_shutdown(); out_free_lockd: nfsd_lockd_shutdown(); nfsd_drc_slab_free(); @@ -2376,7 +2371,6 @@ static void __exit exit_nfsd(void) nfsd4_destroy_laundry_wq(); unregister_cld_notifier(); unregister_pernet_subsys(&nfsd_net_ops); - nfsd_export_wq_shutdown(); nfsd_drc_slab_free(); nfsd_lockd_shutdown(); nfsd4_free_slabs(); diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index eafdf7b7890fd..2c557a9989616 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1508,8 +1508,10 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, nfsd_stats_io_write_add(nn, exp, *cnt); fsnotify_modify(file); host_err = filemap_check_wb_err(file->f_mapping, since); - if (host_err < 0) + if (host_err < 0) { + commit_reset_write_verifier(nn, rqstp, host_err); goto out_nfserr; + } if (stable && fhp->fh_use_wgather) { host_err = wait_for_concurrent_writes(file); @@ -1689,6 +1691,8 @@ nfsd_commit(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, nfsd_copy_write_verifier(verf, nn); err2 = filemap_check_wb_err(nf->nf_file->f_mapping, since); + if (err2 < 0) + commit_reset_write_verifier(nn, rqstp, err2); err = nfserrno(err2); break; case -EINVAL: diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c index 40ac679ec56e4..e2fe95de3d71c 100644 --- a/fs/nilfs2/namei.c +++ b/fs/nilfs2/namei.c @@ -258,7 +258,7 @@ static struct dentry *nilfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, else nilfs_transaction_abort(dir->i_sb); - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; out_fail: drop_nlink(inode); diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 1491a4d4b1e19..9332f5ac60833 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -2512,12 +2512,33 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, struct nilfs_sc_info *sci = nilfs->ns_writer; struct nilfs_transaction_info ti; int err; + size_t i, nfreesegs = argv[4].v_nmembs; + __u64 *segnumv = kbufs[4]; if (unlikely(!sci)) return -EROFS; nilfs_transaction_lock(sb, &ti, 1); + /* + * Validate segment numbers under ns_segctor_sem (held for write + * by nilfs_transaction_lock above) so the check is serialized + * against nilfs_ioctl_resize(), which can modify ns_nsegments. + * Rejecting bad input here, before any segment-cleaning work + * begins, avoids the per-element diagnostic path inside + * nilfs_sufile_updatev() that would otherwise run under this + * same lock and stall concurrent readers. + */ + for (i = 0; i < nfreesegs; i++) { + if (segnumv[i] >= nilfs->ns_nsegments) { + nilfs_err(sb, + "Segment number %llu to be freed is out of range", + (unsigned long long)segnumv[i]); + err = -EINVAL; + goto bail_unlock; + } + } + err = nilfs_mdt_save_to_shadow_map(nilfs->ns_dat); if (unlikely(err)) goto out_unlock; @@ -2558,6 +2579,7 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, sci->sc_freesegs = NULL; sci->sc_nfreesegs = 0; nilfs_mdt_clear_shadow_map(nilfs->ns_dat); + bail_unlock: nilfs_transaction_unlock(sb); return err; } diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index 7aa5ef8606cdd..893a504cb80c8 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -1070,8 +1070,6 @@ nilfs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_time_gran = 1; sb->s_max_links = NILFS_LINK_MAX; - sb->s_bdi = bdi_get(sb->s_bdev->bd_disk->bdi); - err = load_nilfs(nilfs, sb); if (err) goto failed_nilfs; diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index 1fbf832ad1654..173de4cbee0f7 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c @@ -38,11 +38,9 @@ static void ntfs_iomap_read_end_io(struct bio *bio) } static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter, - struct iomap_read_folio_ctx *ctx) + struct iomap_read_folio_ctx *ctx) { - struct bio *bio = ctx->read_ctx; - bio->bi_end_io = ntfs_iomap_read_end_io; - submit_bio(bio); + iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io); } static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { @@ -251,6 +249,8 @@ static int ntfs_writepages(struct address_space *mapping, .wbc = wbc, .ops = &ntfs_writeback_ops, }; + bool need_iput = false; + int ret; if (NVolShutdown(ni->vol)) return -EIO; @@ -267,7 +267,20 @@ static int ntfs_writepages(struct address_space *mapping, return -EOPNOTSUPP; } - return iomap_writepages(&wpc); + /* + * Prevent eviction in writeback to avoid deadlock in + * ntfs_drop_big_inode(). + */ + if ((ni->type == AT_DATA || ni->type == AT_INDEX_ALLOCATION) && + igrab(inode)) + need_iput = true; + + ret = iomap_writepages(&wpc); + + if (need_iput) + iput(inode); + + return ret; } static int ntfs_swap_activate(struct swap_info_struct *sis, diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c index 421c6cdcbb530..0f1d0b54cfb52 100644 --- a/fs/ntfs/attrib.c +++ b/fs/ntfs/attrib.c @@ -174,7 +174,10 @@ int ntfs_map_runlist_nolock(struct ntfs_inode *ni, s64 vcn, struct ntfs_attr_sea err = -EIO; goto err_out; } - WARN_ON(!ctx->attr->non_resident); + if (unlikely(!ctx->attr->non_resident)) { + err = -EIO; + goto err_out; + } } a = ctx->attr; /* @@ -588,6 +591,8 @@ static u32 ntfs_resident_attr_min_value_length(const __le32 type) sizeof(__le16) * 1; case AT_VOLUME_INFORMATION: return sizeof(struct volume_information); + case AT_INDEX_ROOT: + return sizeof(struct index_root); case AT_EA_INFORMATION: return sizeof(struct ea_information); default: @@ -595,6 +600,154 @@ static u32 ntfs_resident_attr_min_value_length(const __le32 type) } } +static bool ntfs_attr_type_is_resident_only(const __le32 type) +{ + switch (type) { + case AT_STANDARD_INFORMATION: + case AT_FILE_NAME: + case AT_OBJECT_ID: + case AT_VOLUME_NAME: + case AT_VOLUME_INFORMATION: + case AT_INDEX_ROOT: + case AT_EA_INFORMATION: + return true; + default: + return false; + } +} + +static bool ntfs_file_name_attr_value_is_valid(const u8 *value, const u32 value_length) +{ + const struct file_name_attr *fn; + u32 file_name_size; + + fn = (const struct file_name_attr *)value; + file_name_size = fn->file_name_length * sizeof(__le16); + + return file_name_size <= + value_length - offsetof(struct file_name_attr, file_name); +} + +static bool ntfs_volume_name_attr_value_is_valid(const u32 value_length) +{ + if (value_length & 1) + return false; + + return value_length <= NTFS_MAX_LABEL_LEN * sizeof(__le16); +} + +static bool ntfs_index_root_attr_value_is_valid(const u8 *value, const u32 value_length) +{ + const struct index_root *ir; + u32 index_size; + u32 entries_offset; + u32 index_length; + u32 allocated_size; + + ir = (const struct index_root *)value; + index_size = value_length - offsetof(struct index_root, index); + entries_offset = le32_to_cpu(ir->index.entries_offset); + index_length = le32_to_cpu(ir->index.index_length); + allocated_size = le32_to_cpu(ir->index.allocated_size); + + if ((entries_offset | index_length | allocated_size) & 7 || + entries_offset < sizeof(struct index_header) || + entries_offset > index_length || + index_length > allocated_size || + allocated_size > index_size || + index_length - entries_offset < sizeof(struct index_entry_header)) + return false; + + return true; +} + +struct ntfs_resident_attr_value { + const u8 *data; + u32 len; +}; + +static bool ntfs_resident_attr_value_get(const struct attr_record *a, + struct ntfs_resident_attr_value *value) +{ + u32 attr_len; + u16 value_offset; + + attr_len = le32_to_cpu(a->length); + if (attr_len < offsetof(struct attr_record, data.resident.reserved) + + sizeof(a->data.resident.reserved)) + return false; + + value->len = le32_to_cpu(a->data.resident.value_length); + value_offset = le16_to_cpu(a->data.resident.value_offset); + + if (value->len > attr_len || value_offset > attr_len - value->len) + return false; + + value->data = (const u8 *)a + value_offset; + return true; +} + +static bool ntfs_non_resident_attr_value_is_valid(const struct attr_record *a) +{ + u32 attr_len; + u32 min_len; + u16 mp_offset; + + attr_len = le32_to_cpu(a->length); + min_len = offsetof(struct attr_record, data.non_resident.initialized_size) + + sizeof(a->data.non_resident.initialized_size); + if (attr_len < min_len) + return false; + + mp_offset = le16_to_cpu(a->data.non_resident.mapping_pairs_offset); + return mp_offset >= min_len && mp_offset <= attr_len; +} + +static bool ntfs_attr_value_is_valid(struct ntfs_volume *vol, + const struct attr_record *a, + const u64 mft_no) +{ + struct ntfs_resident_attr_value value; + u32 min_len; + + if (a->non_resident) { + if (ntfs_attr_type_is_resident_only(a->type)) + goto corrupt; + if (!ntfs_non_resident_attr_value_is_valid(a)) + goto corrupt; + return true; + } + + if (!ntfs_resident_attr_value_get(a, &value)) + goto corrupt; + + min_len = ntfs_resident_attr_min_value_length(a->type); + if (min_len && value.len < min_len) + goto corrupt; + + switch (a->type) { + case AT_FILE_NAME: + if (!ntfs_file_name_attr_value_is_valid(value.data, value.len)) + goto corrupt; + break; + case AT_VOLUME_NAME: + if (!ntfs_volume_name_attr_value_is_valid(value.len)) + goto corrupt; + break; + case AT_INDEX_ROOT: + if (!ntfs_index_root_attr_value_is_valid(value.data, value.len)) + goto corrupt; + break; + } + return true; + +corrupt: + ntfs_error(vol->sb, + "Corrupt %#x attribute in MFT record %llu\n", + le32_to_cpu(a->type), mft_no); + return false; +} + /* * ntfs_attr_find - find (next) attribute in mft record * @type: attribute type to find @@ -705,8 +858,11 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name, } } - if (type == AT_UNUSED) + if (type == AT_UNUSED) { + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; return 0; + } if (a->type != type) continue; /* @@ -747,37 +903,8 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name, } } - /* Validate attribute's value offset/length */ - if (!a->non_resident) { - u32 min_len; - u32 value_length = le32_to_cpu(a->data.resident.value_length); - u16 value_offset = le16_to_cpu(a->data.resident.value_offset); - - if (value_length > le32_to_cpu(a->length) || - value_offset > le32_to_cpu(a->length) - value_length) - break; - - min_len = ntfs_resident_attr_min_value_length(a->type); - if (min_len && value_length < min_len) { - ntfs_error(vol->sb, - "Too small %#x resident attribute value in MFT record %lld\n", - le32_to_cpu(a->type), (long long)ctx->ntfs_ino->mft_no); - break; - } - } else { - u32 min_len; - u16 mp_offset; - - min_len = offsetof(struct attr_record, data.non_resident.initialized_size) + - sizeof(a->data.non_resident.initialized_size); - if (le32_to_cpu(a->length) < min_len) - break; - - mp_offset = le16_to_cpu(a->data.non_resident.mapping_pairs_offset); - if (mp_offset < min_len || - mp_offset > le32_to_cpu(a->length)) - break; - } + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; /* * The names match or @name not present and attribute is @@ -843,11 +970,71 @@ char *ntfs_attr_name_get(const struct ntfs_volume *vol, const __le16 *uname, return NULL; } +/* + * ntfs_attr_list_entry_is_valid - sanity check one $ATTRIBUTE_LIST entry + * @ale: the attribute-list entry to check + * @al_end: end of the attribute-list buffer @ale lives in + * + * Verify that @ale is a well-formed attr_list_entry wholly contained in + * [.., @al_end): its fixed header must lie in range before any field is + * dereferenced, its length must be a multiple of 8 that covers the fixed + * header plus the name, the name must lie within the buffer, the entry must + * be in use and carry a live MFT reference. Return true if valid. + */ +bool ntfs_attr_list_entry_is_valid(const struct attr_list_entry *ale, + const u8 *al_end) +{ + const u8 *al = (const u8 *)ale; + u16 ale_len; + + /* The fixed header must be in bounds before it is parsed. */ + if (al + offsetof(struct attr_list_entry, name) > al_end) + return false; + ale_len = le16_to_cpu(ale->length); + /* On-disk entries are 8-byte aligned (see struct attr_list_entry). */ + if (ale_len & 7) + return false; + if (ale->name_offset != sizeof(struct attr_list_entry)) + return false; + if ((u32)ale->name_offset + + (u32)ale->name_length * sizeof(__le16) > ale_len || + al + ale_len > al_end) + return false; + if (ale->type == AT_UNUSED) + return false; + if (MSEQNO_LE(ale->mft_reference) == 0) + return false; + return true; +} + +/* + * ntfs_attr_list_is_valid - sanity check an in-memory $ATTRIBUTE_LIST + * @al_start: start of the attribute list buffer + * @size: length of the attribute list in bytes + * + * Verify that [@al_start, @al_start + @size) is a sequence of valid + * attr_list_entry records (see ntfs_attr_list_entry_is_valid()) that tile the + * buffer exactly. Return true if valid, false otherwise. + */ +bool ntfs_attr_list_is_valid(const u8 *al_start, s64 size) +{ + const u8 *al = al_start; + const u8 *al_end = al_start + size; + + while (al < al_end) { + const struct attr_list_entry *ale = + (const struct attr_list_entry *)al; + + if (!ntfs_attr_list_entry_is_valid(ale, al_end)) + return false; + al += le16_to_cpu(ale->length); + } + return al == al_end; +} + int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size) { struct inode *attr_vi = NULL; - u8 *al; - struct attr_list_entry *ale; if (!al_start || size <= 0) return -EINVAL; @@ -869,19 +1056,7 @@ int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size } iput(attr_vi); - for (al = al_start; al < al_start + size; al += le16_to_cpu(ale->length)) { - ale = (struct attr_list_entry *)al; - if (ale->name_offset != sizeof(struct attr_list_entry)) - break; - if (le16_to_cpu(ale->length) <= ale->name_offset + ale->name_length || - al + le16_to_cpu(ale->length) > al_start + size) - break; - if (ale->type == AT_UNUSED) - break; - if (MSEQNO_LE(ale->mft_reference) == 0) - break; - } - if (al != al_start + size) { + if (!ntfs_attr_list_is_valid(al_start, size)) { ntfs_error(base_ni->vol->sb, "Corrupt attribute list, mft = %llu", base_ni->mft_no); return -EIO; @@ -1137,9 +1312,8 @@ static int ntfs_external_attr_find(const __le32 type, * we have reached the right one or the search has failed. */ if (lowest_vcn && (u8 *)next_al_entry >= al_start && - (u8 *)next_al_entry + 6 < al_end && - (u8 *)next_al_entry + le16_to_cpu( - next_al_entry->length) <= al_end && + ntfs_attr_list_entry_is_valid(next_al_entry, + al_end) && le64_to_cpu(next_al_entry->lowest_vcn) <= lowest_vcn && next_al_entry->type == al_entry->type && @@ -1252,22 +1426,8 @@ static int ntfs_external_attr_find(const __le32 type, ctx->attr = a; - if (a->non_resident) { - u32 min_len; - u16 mp_offset; - - min_len = offsetof(struct attr_record, - data.non_resident.initialized_size) + - sizeof(a->data.non_resident.initialized_size); - - if (le32_to_cpu(a->length) < min_len) - break; - - mp_offset = - le16_to_cpu(a->data.non_resident.mapping_pairs_offset); - if (mp_offset < min_len || mp_offset > attr_len) - break; - } + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; /* * If no @val specified or @val specified and it matches, we @@ -1279,19 +1439,6 @@ static int ntfs_external_attr_find(const __le32 type, u32 value_length = le32_to_cpu(a->data.resident.value_length); u16 value_offset = le16_to_cpu(a->data.resident.value_offset); - if (attr_len < offsetof(struct attr_record, data.resident.reserved) + - sizeof(a->data.resident.reserved)) - break; - if (value_length > attr_len || value_offset > attr_len - value_length) - break; - - value_length = ntfs_resident_attr_min_value_length(a->type); - if (value_length && le32_to_cpu(a->data.resident.value_length) < - value_length) { - pr_err("Too small resident attribute value in MFT record %lld, type %#x\n", - (long long)ctx->ntfs_ino->mft_no, a->type); - break; - } if (value_length == val_len && !memcmp((u8 *)a + value_offset, val, val_len)) { attr_found: @@ -4536,10 +4683,12 @@ static int ntfs_resident_attr_resize(struct ntfs_inode *attr_ni, const s64 newsi while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) { struct inode *tvi; struct attr_record *a; + u32 value_len; a = ctx->attr; if (a->non_resident || a->type == AT_ATTRIBUTE_LIST) continue; + value_len = le32_to_cpu(a->data.resident.value_length); if (ntfs_attr_can_be_non_resident(vol, a->type)) continue; @@ -4551,6 +4700,8 @@ static int ntfs_resident_attr_resize(struct ntfs_inode *attr_ni, const s64 newsi if (le32_to_cpu(a->length) <= (sizeof(struct attr_record) - sizeof(s64)) + ((a->name_length * sizeof(__le16) + 7) & ~7) + 8) continue; + if (a->type == AT_DATA && !value_len) + continue; if (a->type == AT_DATA) tvi = ntfs_iget(sb, base_ni->mft_no); @@ -4563,8 +4714,7 @@ static int ntfs_resident_attr_resize(struct ntfs_inode *attr_ni, const s64 newsi continue; } - if (ntfs_attr_make_non_resident(NTFS_I(tvi), - le32_to_cpu(ctx->attr->data.resident.value_length))) { + if (ntfs_attr_make_non_resident(NTFS_I(tvi), value_len)) { iput(tvi); continue; } @@ -5177,6 +5327,7 @@ int ntfs_non_resident_attr_insert_range(struct ntfs_inode *ni, s64 start_vcn, s6 ret = ntfs_attr_map_whole_runlist(ni); if (ret) { up_write(&ni->runlist.lock); + kfree(hole_rl); return ret; } @@ -5388,6 +5539,7 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo s64 old_data_size; s64 vcn_start, vcn_end, vcn_uninit, vcn, try_alloc_cnt; s64 lcn, alloc_cnt; + s64 rl_lcn, rl_length, rl_vcn; int err = 0; struct runlist_element *rl; bool balloc; @@ -5467,19 +5619,23 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo while (vcn < vcn_uninit) { down_read(&ni->runlist.lock); rl = ntfs_attr_find_vcn_nolock(ni, vcn, NULL); - up_read(&ni->runlist.lock); if (IS_ERR(rl)) { + up_read(&ni->runlist.lock); err = PTR_ERR(rl); goto out; } + rl_lcn = rl->lcn; + rl_length = rl->length; + rl_vcn = rl->vcn; + up_read(&ni->runlist.lock); - if (rl->lcn > 0) { - vcn += rl->length - (vcn - rl->vcn); - } else if (rl->lcn == LCN_DELALLOC || rl->lcn == LCN_HOLE) { - try_alloc_cnt = min(rl->length - (vcn - rl->vcn), + if (rl_lcn > 0) { + vcn += rl_length - (vcn - rl_vcn); + } else if (rl_lcn == LCN_DELALLOC || rl_lcn == LCN_HOLE) { + try_alloc_cnt = min(rl_length - (vcn - rl_vcn), vcn_uninit - vcn); - if (rl->lcn == LCN_DELALLOC) { + if (rl_lcn == LCN_DELALLOC) { vcn += try_alloc_cnt; continue; } @@ -5494,11 +5650,14 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo if (err) goto out; - err = ntfs_dio_zero_range(VFS_I(ni), - lcn << vol->cluster_size_bits, - alloc_cnt << vol->cluster_size_bits); - if (err > 0) - goto out; + if (balloc) { + err = ntfs_dio_zero_range(VFS_I(ni), + lcn << vol->cluster_size_bits, + alloc_cnt << + vol->cluster_size_bits); + if (err > 0) + goto out; + } if (signal_pending(current)) goto out; diff --git a/fs/ntfs/attrib.h b/fs/ntfs/attrib.h index f7acc7986b090..e2224fbfaabe9 100644 --- a/fs/ntfs/attrib.h +++ b/fs/ntfs/attrib.h @@ -71,6 +71,10 @@ int ntfs_attr_lookup(const __le32 type, const __le16 *name, const u32 name_len, const u32 ic, const s64 lowest_vcn, const u8 *val, const u32 val_len, struct ntfs_attr_search_ctx *ctx); +bool ntfs_attr_list_entry_is_valid(const struct attr_list_entry *ale, + const u8 *al_end); +bool ntfs_attr_list_is_valid(const u8 *al_start, s64 size); + int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size); diff --git a/fs/ntfs/attrlist.c b/fs/ntfs/attrlist.c index c2594d4c83b06..be3086d343381 100644 --- a/fs/ntfs/attrlist.c +++ b/fs/ntfs/attrlist.c @@ -57,6 +57,15 @@ int ntfs_attrlist_update(struct ntfs_inode *base_ni) struct ntfs_inode *attr_ni; int err; + /* + * generic_shutdown_super() clears SB_ACTIVE before evicting cached + * inodes. Do not look up the attribute-list inode after SB_ACTIVE has + * been cleared; it may already be I_FREEING, and waiting on it can + * self-deadlock. + */ + if (!(VFS_I(base_ni)->i_sb->s_flags & SB_ACTIVE)) + return -EIO; + attr_vi = ntfs_attr_iget(VFS_I(base_ni), AT_ATTRIBUTE_LIST, AT_UNNAMED, 0); if (IS_ERR(attr_vi)) { err = PTR_ERR(attr_vi); @@ -118,6 +127,7 @@ int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr) int entry_len, entry_offset, err; struct mft_record *ni_mrec; u8 *old_al; + __le64 lowest_vcn; if (!ni || !attr) { ntfs_debug("Invalid arguments.\n"); @@ -158,17 +168,21 @@ int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr) ntfs_error(ni->vol->sb, "Failed to get search context"); goto err_out; } + if (attr->non_resident) + lowest_vcn = attr->data.non_resident.lowest_vcn; + else + lowest_vcn = 0; err = ntfs_attr_lookup(attr->type, (attr->name_length) ? (__le16 *) ((u8 *)attr + le16_to_cpu(attr->name_offset)) : AT_UNNAMED, attr->name_length, CASE_SENSITIVE, - (attr->non_resident) ? le64_to_cpu(attr->data.non_resident.lowest_vcn) : - 0, (attr->non_resident) ? NULL : ((u8 *)attr + + le64_to_cpu(lowest_vcn), + (attr->non_resident) ? NULL : ((u8 *)attr + le16_to_cpu(attr->data.resident.value_offset)), (attr->non_resident) ? 0 : le32_to_cpu(attr->data.resident.value_length), ctx); if (!err) { /* Found some extent, check it to be before new extent. */ - if (ctx->al_entry->lowest_vcn == attr->data.non_resident.lowest_vcn) { + if (ctx->al_entry->lowest_vcn == lowest_vcn) { err = -EEXIST; ntfs_debug("Such attribute already present in the attribute list.\n"); ntfs_attr_put_search_ctx(ctx); diff --git a/fs/ntfs/dir.c b/fs/ntfs/dir.c index 20f5c7074bdd1..6fa9ae3377cb8 100644 --- a/fs/ntfs/dir.c +++ b/fs/ntfs/dir.c @@ -23,6 +23,13 @@ __le16 I30[5] = { cpu_to_le16('$'), cpu_to_le16('I'), cpu_to_le16('3'), cpu_to_le16('0'), 0 }; +static inline u64 ntfs_check_mref(u64 mref) +{ + if (IS_ERR_MREF(mref)) + return ERR_MREF(-EIO); + return mref; +} + /* * ntfs_lookup_inode_by_name - find an inode in a directory given its name * @dir_ni: ntfs inode of the directory in which to search for the name @@ -135,10 +142,6 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, /* Key length should not be zero if it is not last entry. */ if (!ie->key_length) goto dir_err_out; - /* Check the consistency of an index entry */ - if (ntfs_index_entry_inconsistent(NULL, vol, ie, COLLATION_FILE_NAME, - dir_ni->mft_no)) - goto dir_err_out; /* * We perform a case sensitive comparison and if that matches * we are done and return the mft reference of the inode (i.e. @@ -182,7 +185,7 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, mref = le64_to_cpu(ie->data.dir.indexed_file); ntfs_attr_put_search_ctx(ctx); unmap_mft_record(dir_ni); - return mref; + return ntfs_check_mref(mref); } /* * For a case insensitive mount, we also perform a case @@ -277,7 +280,7 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, if (name) { ntfs_attr_put_search_ctx(ctx); unmap_mft_record(dir_ni); - return name->mref; + return ntfs_check_mref(name->mref); } ntfs_debug("Entry not found."); err = -ENOENT; @@ -342,43 +345,20 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, dir_ni->mft_no); goto unm_err_out; } - /* Catch multi sector transfer fixup errors. */ - if (unlikely(!ntfs_is_indx_record(ia->magic))) { - ntfs_error(sb, - "Directory index record with vcn 0x%llx is corrupt. Corrupt inode 0x%llx. Run chkdsk.", - vcn, dir_ni->mft_no); - goto unm_err_out; - } - if (le64_to_cpu(ia->index_block_vcn) != vcn) { - ntfs_error(sb, - "Actual VCN (0x%llx) of index buffer is different from expected VCN (0x%llx). Directory inode 0x%llx is corrupt or driver bug.", - le64_to_cpu(ia->index_block_vcn), - vcn, dir_ni->mft_no); - goto unm_err_out; - } - if (le32_to_cpu(ia->index.allocated_size) + 0x18 != - dir_ni->itype.index.block_size) { - ntfs_error(sb, - "Index buffer (VCN 0x%llx) of directory inode 0x%llx has a size (%u) differing from the directory specified size (%u). Directory inode is corrupt or driver bug.", - vcn, dir_ni->mft_no, - le32_to_cpu(ia->index.allocated_size) + 0x18, - dir_ni->itype.index.block_size); - goto unm_err_out; - } index_end = (u8 *)ia + dir_ni->itype.index.block_size; if (index_end > kaddr + PAGE_SIZE) { ntfs_error(sb, - "Index buffer (VCN 0x%llx) of directory inode 0x%llx crosses page boundary. Impossible! Cannot access! This is probably a bug in the driver.", - vcn, dir_ni->mft_no); + "Index buffer (VCN 0x%llx) of directory inode 0x%llx crosses page boundary. Impossible! Cannot access! This is probably a bug in the driver.", + vcn, dir_ni->mft_no); goto unm_err_out; } - index_end = (u8 *)&ia->index + le32_to_cpu(ia->index.index_length); - if (index_end > (u8 *)ia + dir_ni->itype.index.block_size) { - ntfs_error(sb, - "Size of index buffer (VCN 0x%llx) of directory inode 0x%llx exceeds maximum size.", - vcn, dir_ni->mft_no); + err = ntfs_index_block_inconsistent(vol, ia, + dir_ni->itype.index.block_size, + vcn, COLLATION_FILE_NAME, + dir_ni->mft_no); + if (err) goto unm_err_out; - } + index_end = (u8 *)&ia->index + le32_to_cpu(ia->index.index_length); /* The first index entry. */ ie = (struct index_entry *)((u8 *)&ia->index + le32_to_cpu(ia->index.entries_offset)); @@ -388,15 +368,6 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, * reach the last entry. */ for (;; ie = (struct index_entry *)((u8 *)ie + le16_to_cpu(ie->length))) { - /* Bounds checks. */ - if ((u8 *)ie < (u8 *)ia || - (u8 *)ie + sizeof(struct index_entry_header) > index_end || - (u8 *)ie + sizeof(struct index_entry_header) + le16_to_cpu(ie->key_length) > - index_end || (u8 *)ie + le16_to_cpu(ie->length) > index_end) { - ntfs_error(sb, "Index entry out of bounds in directory inode 0x%llx.", - dir_ni->mft_no); - goto unm_err_out; - } /* * The last entry cannot contain a name. It can however contain * a pointer to a child node in the B+tree so we just break out. @@ -406,10 +377,6 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, /* Key length should not be zero if it is not last entry. */ if (!ie->key_length) goto unm_err_out; - /* Check the consistency of an index entry */ - if (ntfs_index_entry_inconsistent(NULL, vol, ie, COLLATION_FILE_NAME, - dir_ni->mft_no)) - goto unm_err_out; /* * We perform a case sensitive comparison and if that matches * we are done and return the mft reference of the inode (i.e. @@ -453,7 +420,7 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, mref = le64_to_cpu(ie->data.dir.indexed_file); kfree(kaddr); iput(ia_vi); - return mref; + return ntfs_check_mref(mref); } /* * For a case insensitive mount, we also perform a case @@ -578,7 +545,7 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, if (name) { kfree(kaddr); iput(ia_vi); - return name->mref; + return ntfs_check_mref(name->mref); } ntfs_debug("Entry not found."); err = -ENOENT; @@ -892,6 +859,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *actor) ictx->vcn_size_bits = vol->cluster_size_bits; else ictx->vcn_size_bits = NTFS_BLOCK_SIZE_BITS; + ictx->cr = ir->collation_rule; /* The first index entry. */ next = (struct index_entry *)((u8 *)&ir->index + @@ -929,13 +897,6 @@ static int ntfs_readdir(struct file *file, struct dir_context *actor) if (!next) break; nextdir: - /* Check the consistency of an index entry */ - if (ntfs_index_entry_inconsistent(ictx, vol, next, COLLATION_FILE_NAME, - ndir->mft_no)) { - err = -EIO; - goto out; - } - if (ie_pos < actor->pos) { ie_pos += le16_to_cpu(next->length); continue; diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c index c4a4a3e3e5996..0cd192752b7cd 100644 --- a/fs/ntfs/ea.c +++ b/fs/ntfs/ea.c @@ -53,11 +53,11 @@ static int ntfs_ea_lookup(char *ea_buf, s64 ea_buf_size, const char *name, loff_t offset, p_ea_size; unsigned int next; - if (ea_buf_size < sizeof(struct ea_attr)) - goto out; - offset = 0; do { + if (ea_buf_size - offset < sizeof(struct ea_attr)) + break; + p_ea = (const struct ea_attr *)&ea_buf[offset]; next = le32_to_cpu(p_ea->next_entry_offset); p_ea_size = next ? next : (ea_buf_size - offset); @@ -479,13 +479,13 @@ ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size) if (ea_info_qsize > ea_buf_size || ea_info_qsize == 0) goto out; - if (ea_info_qsize < sizeof(struct ea_attr)) { - err = -EIO; - goto out; - } - offset = 0; do { + if (ea_info_qsize - offset < sizeof(struct ea_attr)) { + err = -EIO; + goto out; + } + p_ea = (const struct ea_attr *)&ea_buf[offset]; next = le32_to_cpu(p_ea->next_entry_offset); ea_size = next ? next : (ea_info_qsize - offset); diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index e8bea22b81a75..264cf8404385c 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -707,12 +707,21 @@ static int ntfs_ioctl_get_volume_label(struct file *filp, unsigned long arg) { struct ntfs_volume *vol = NTFS_SB(file_inode(filp)->i_sb); char __user *buf = (char __user *)arg; + char label[FSLABEL_MAX]; + ssize_t len; + mutex_lock(&vol->volume_label_lock); if (!vol->volume_label) { - if (copy_to_user(buf, "", 1)) - return -EFAULT; - } else if (copy_to_user(buf, vol->volume_label, - MIN(FSLABEL_MAX, strlen(vol->volume_label) + 1))) + label[0] = '\0'; + len = 0; + } else { + len = strscpy(label, vol->volume_label, sizeof(label)); + if (len == -E2BIG) + len = FSLABEL_MAX - 1; + } + mutex_unlock(&vol->volume_label_lock); + + if (copy_to_user(buf, label, len + 1)) return -EFAULT; return 0; } diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c index 146e011c1a418..9495b39bd3ceb 100644 --- a/fs/ntfs/index.c +++ b/fs/ntfs/index.c @@ -28,41 +28,10 @@ * length must have been checked beforehand to not overflow from the * index record. */ -int ntfs_index_entry_inconsistent(struct ntfs_index_context *icx, - struct ntfs_volume *vol, const struct index_entry *ie, - __le32 collation_rule, u64 inum) +static int ntfs_index_entry_inconsistent(const struct ntfs_volume *vol, + const struct index_entry *ie, + __le32 collation_rule, u64 inum) { - if (icx) { - struct index_header *ih; - u8 *ie_start, *ie_end; - - if (icx->is_in_root) - ih = &icx->ir->index; - else - ih = &icx->ib->index; - - if ((le32_to_cpu(ih->index_length) > le32_to_cpu(ih->allocated_size)) || - (le32_to_cpu(ih->index_length) > icx->block_size)) { - ntfs_error(vol->sb, "%s Index entry(0x%p)'s length is too big.", - icx->is_in_root ? "Index root" : "Index block", - (u8 *)icx->entry); - return -EINVAL; - } - - ie_start = (u8 *)ih + le32_to_cpu(ih->entries_offset); - ie_end = (u8 *)ih + le32_to_cpu(ih->index_length); - - if (ie_start > (u8 *)ie || - ie_end <= (u8 *)ie + le16_to_cpu(ie->length) || - le16_to_cpu(ie->length) > le32_to_cpu(ih->allocated_size) || - le16_to_cpu(ie->length) > icx->block_size) { - ntfs_error(vol->sb, "Index entry(0x%p) is out of range from %s", - (u8 *)icx->entry, - icx->is_in_root ? "index root" : "index block"); - return -EIO; - } - } - if (ie->key_length && ((le16_to_cpu(ie->key_length) + offsetof(struct index_entry, key)) > le16_to_cpu(ie->length))) { @@ -141,6 +110,10 @@ static int ntfs_ib_write(struct ntfs_index_context *icx, struct index_block *ib) ret = ntfs_inode_attr_pwrite(VFS_I(icx->ia_ni), ntfs_ib_vcn_to_pos(icx, vcn), icx->block_size, (u8 *)ib, icx->sync_write); + + /* Perform data restoration before returning */ + post_write_mst_fixup((struct ntfs_record *)ib); + if (ret != icx->block_size) { ntfs_debug("Failed to write index block %lld, inode %llu", vcn, (unsigned long long)icx->idx_ni->mft_no); @@ -178,7 +151,6 @@ int ntfs_icx_ib_sync_write(struct ntfs_index_context *icx) icx->ib = NULL; icx->ib_dirty = false; } else { - post_write_mst_fixup((struct ntfs_record *)icx->ib); icx->sync_write = false; } @@ -303,6 +275,93 @@ static int ntfs_ie_end(struct index_entry *ie) return ie->flags & INDEX_ENTRY_END || !ie->length; } +static int ntfs_index_header_inconsistent(struct ntfs_volume *vol, + const struct index_header *ih, + u32 bytes_available, u64 inum) +{ + u32 entries_offset, index_length, allocated_size; + + if (bytes_available < sizeof(struct index_header)) { + ntfs_error(vol->sb, + "index block in inode %llu is smaller than an index header.", + (unsigned long long)inum); + return -EIO; + } + + entries_offset = le32_to_cpu(ih->entries_offset); + index_length = le32_to_cpu(ih->index_length); + allocated_size = le32_to_cpu(ih->allocated_size); + + if (entries_offset < sizeof(struct index_header) || + entries_offset > bytes_available) { + ntfs_error(vol->sb, + "Invalid index entry offset in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (index_length <= entries_offset) { + ntfs_error(vol->sb, + "No space for index entries in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (allocated_size < index_length) { + ntfs_error(vol->sb, + "Index entries overflow in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (allocated_size > bytes_available || index_length > bytes_available) { + ntfs_error(vol->sb, + "Index entries in inode %llu exceed the available buffer.", + (unsigned long long)inum); + return -EIO; + } + + return 0; +} + +int ntfs_index_entries_inconsistent(const struct ntfs_volume *vol, + const struct index_header *ih, + __le32 collation_rule, u64 inum) +{ + struct index_entry *ie; + u8 *index_end = (u8 *)ih + le32_to_cpu(ih->index_length); + + for (ie = ntfs_ie_get_first((struct index_header *)ih); + ; ie = ntfs_ie_get_next(ie)) { + if ((u8 *)ie + sizeof(struct index_entry_header) > index_end || + (u8 *)ie + le16_to_cpu(ie->length) > index_end) { + ntfs_error(vol->sb, + "Index entry out of bounds in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (le16_to_cpu(ie->length) < sizeof(struct index_entry_header)) { + ntfs_error(vol->sb, + "Index etnry too small in inode %llu.", + inum); + return -EIO; + } + + if (ntfs_ie_end(ie)) + break; + + if (!ie->key_length) + return -EIO; + + if (ntfs_index_entry_inconsistent(vol, ie, + collation_rule, inum)) + return -EIO; + } + + return 0; +} + /* * Find the last entry in the index block */ @@ -437,7 +496,7 @@ static struct index_entry *ntfs_ie_dup_novcn(struct index_entry *ie) * The size of block is assumed to have been checked to be what is * defined in the index root. * - * Returns 0 if no error was found -1 otherwise (with errno unchanged) + * Returns 0 if no error was found, -EIO otherwise * * |<--->| offsetof(struct index_block, index) * | |<--->| sizeof(struct index_header) @@ -452,21 +511,21 @@ static struct index_entry *ntfs_ie_dup_novcn(struct index_entry *ie) * * size(struct index_header) <= ent_offset < ind_length <= alloc_size < bk_size */ -static int ntfs_index_block_inconsistent(struct ntfs_index_context *icx, - struct index_block *ib, s64 vcn) +int ntfs_index_block_inconsistent(struct ntfs_volume *vol, + const struct index_block *ib, + u32 block_size, s64 vcn, __le32 cr, + u64 inum) { u32 ib_size = (unsigned int)le32_to_cpu(ib->index.allocated_size) + offsetof(struct index_block, index); - struct super_block *sb = icx->idx_ni->vol->sb; - unsigned long long inum = icx->idx_ni->mft_no; + struct super_block *sb = vol->sb; ntfs_debug("Entering\n"); if (!ntfs_is_indx_record(ib->magic)) { - ntfs_error(sb, "Corrupt index block signature: vcn %lld inode %llu\n", - vcn, (unsigned long long)icx->idx_ni->mft_no); - return -1; + vcn, (unsigned long long)inum); + return -EIO; } if (le64_to_cpu(ib->index_block_vcn) != vcn) { @@ -474,32 +533,42 @@ static int ntfs_index_block_inconsistent(struct ntfs_index_context *icx, "Corrupt index block: s64 (%lld) is different from expected s64 (%lld) in inode %llu\n", (long long)le64_to_cpu(ib->index_block_vcn), vcn, inum); - return -1; + return -EIO; } - if (ib_size != icx->block_size) { + if (ib_size != block_size) { ntfs_error(sb, - "Corrupt index block : s64 (%lld) of inode %llu has a size (%u) differing from the index specified size (%u)\n", - vcn, inum, ib_size, icx->block_size); - return -1; + "Corrupt index block : s64 (%lld) of inode %llu has a size (%u) differing from the index specified size (%u)\n", + vcn, inum, ib_size, block_size); + return -EIO; } - if (le32_to_cpu(ib->index.entries_offset) < sizeof(struct index_header)) { - ntfs_error(sb, "Invalid index entry offset in inode %lld\n", inum); - return -1; - } - if (le32_to_cpu(ib->index.index_length) <= - le32_to_cpu(ib->index.entries_offset)) { - ntfs_error(sb, "No space for index entries in inode %lld\n", inum); - return -1; - } - if (le32_to_cpu(ib->index.allocated_size) < - le32_to_cpu(ib->index.index_length)) { - ntfs_error(sb, "Index entries overflow in inode %lld\n", inum); - return -1; + if (ntfs_index_header_inconsistent(vol, &ib->index, + block_size - + offsetof(struct index_block, index), + inum)) + return -EIO; + if (ntfs_index_entries_inconsistent(vol, &ib->index, cr, inum)) + return -EIO; + return 0; +} + +int ntfs_index_root_inconsistent(struct ntfs_volume *vol, + const struct attr_record *a, + const struct index_root *ir, u64 inum) +{ + u32 value_length = le32_to_cpu(a->data.resident.value_length); + + if (value_length < offsetof(struct index_root, index)) { + ntfs_error(vol->sb, "$INDEX_ROOT in inode %llu is too small.", + (unsigned long long)inum); + return -EIO; } - return 0; + return ntfs_index_header_inconsistent(vol, &ir->index, + value_length - + offsetof(struct index_root, index), + inum); } static struct index_root *ntfs_ir_lookup(struct ntfs_inode *ni, __le16 *name, @@ -665,13 +734,14 @@ static int ntfs_ib_read(struct ntfs_index_context *icx, s64 vcn, struct index_bl else ntfs_error(icx->idx_ni->vol->sb, "Failed to read full index block at %lld\n", pos); - return -1; + return -EIO; } post_read_mst_fixup((struct ntfs_record *)((u8 *)dst), icx->block_size); - if (ntfs_index_block_inconsistent(icx, dst, vcn)) - return -1; - + if (ntfs_index_block_inconsistent(icx->idx_ni->vol, dst, + icx->block_size, vcn, icx->cr, + icx->idx_ni->mft_no)) + return -EIO; return 0; } @@ -1173,6 +1243,8 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) struct index_entry *ie; struct index_block *ib = NULL; s64 new_ib_vcn; + u32 index_length; + u32 old_value_length; int ix_root_size; int ret = 0; @@ -1220,6 +1292,21 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) goto clear_bmp; } + old_value_length = le32_to_cpu(ctx->attr->data.resident.value_length); + index_length = le32_to_cpu(ir->index.entries_offset) + + sizeof(struct index_entry_header) + sizeof(s64); + ix_root_size = offsetof(struct index_root, index) + index_length; + /* Grow the resident value before publishing the larger root header. */ + if (ix_root_size > old_value_length) { + ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); + if (ret) + goto resize_failed; + + icx->idx_ni->data_size = ix_root_size; + icx->idx_ni->initialized_size = ix_root_size; + icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; + } + ntfs_ir_nill(ir); ie = ntfs_ie_get_first(&ir->index); @@ -1228,48 +1315,49 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) ir->index.flags = LARGE_INDEX; NInoSetIndexAllocPresent(icx->idx_ni); - ir->index.index_length = cpu_to_le32(le32_to_cpu(ir->index.entries_offset) + - le16_to_cpu(ie->length)); + ir->index.index_length = cpu_to_le32(index_length); ir->index.allocated_size = ir->index.index_length; - ix_root_size = sizeof(struct index_root) - sizeof(struct index_header) + - le32_to_cpu(ir->index.allocated_size); - ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); - if (ret) { - /* - * When there is no space to build a non-resident - * index, we may have to move the root to an extent - */ - if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) { + if (ix_root_size <= old_value_length) { + ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); + if (ret) + goto resize_failed; + + icx->idx_ni->data_size = ix_root_size; + icx->idx_ni->initialized_size = ix_root_size; + icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; + } + ntfs_ie_set_vcn(ie, new_ib_vcn); + goto err_out; + +resize_failed: + /* + * When there is no space to build a non-resident + * index, we may have to move the root to an extent + */ + if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) { + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); + if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size - + le32_to_cpu(ctx->attr->data.resident.value_length))) { + if (ntfs_attrlist_update(ctx->base_ntfs_ino ? + ctx->base_ntfs_ino : ctx->ntfs_ino)) + goto clear_bmp; ntfs_attr_put_search_ctx(ctx); ctx = NULL; - ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); - if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size - - le32_to_cpu(ctx->attr->data.resident.value_length))) { - if (ntfs_attrlist_update(ctx->base_ntfs_ino ? - ctx->base_ntfs_ino : ctx->ntfs_ino)) - goto clear_bmp; - ntfs_attr_put_search_ctx(ctx); - ctx = NULL; - goto retry; - } + goto retry; } - goto clear_bmp; - } else { - icx->idx_ni->data_size = icx->idx_ni->initialized_size = ix_root_size; - icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; } - ntfs_ie_set_vcn(ie, new_ib_vcn); - +clear_bmp: + ntfs_ibm_clear(icx, new_ib_vcn); + goto err_out; err_out: kvfree(ib); if (ctx) ntfs_attr_put_search_ctx(ctx); out: return ret; -clear_bmp: - ntfs_ibm_clear(icx, new_ib_vcn); - goto err_out; } /* @@ -1280,9 +1368,16 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) static int ntfs_ir_truncate(struct ntfs_index_context *icx, int data_size) { int ret; + u32 old_allocated_size; + bool shrink; ntfs_debug("Entering\n"); + old_allocated_size = le32_to_cpu(icx->ir->index.allocated_size); + shrink = data_size < old_allocated_size; + if (shrink) + icx->ir->index.allocated_size = cpu_to_le32(data_size); + /* * INDEX_ROOT must be resident and its entries can be moved to * struct index_block, so ENOSPC isn't a real error. @@ -1294,9 +1389,14 @@ static int ntfs_ir_truncate(struct ntfs_index_context *icx, int data_size) if (!icx->ir) return -ENOENT; - icx->ir->index.allocated_size = cpu_to_le32(data_size); - } else if (ret != -ENOSPC) - ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate INDEX_ROOT"); + if (!shrink) + icx->ir->index.allocated_size = cpu_to_le32(data_size); + } else { + if (shrink) + icx->ir->index.allocated_size = cpu_to_le32(old_allocated_size); + if (ret != -ENOSPC) + ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate INDEX_ROOT"); + } return ret; } diff --git a/fs/ntfs/index.h b/fs/ntfs/index.h index e68d6fabaf9f1..9a03f53bba479 100644 --- a/fs/ntfs/index.h +++ b/fs/ntfs/index.h @@ -89,8 +89,16 @@ struct ntfs_index_context { bool sync_write; }; -int ntfs_index_entry_inconsistent(struct ntfs_index_context *icx, struct ntfs_volume *vol, - const struct index_entry *ie, __le32 collation_rule, u64 inum); +int ntfs_index_root_inconsistent(struct ntfs_volume *vol, + const struct attr_record *a, + const struct index_root *ir, u64 inum); +int ntfs_index_block_inconsistent(struct ntfs_volume *vol, + const struct index_block *ib, + u32 block_size, s64 vcn, + __le32 cr, u64 inum); +int ntfs_index_entries_inconsistent(const struct ntfs_volume *vol, + const struct index_header *ih, + __le32 collation_rule, u64 inum); struct ntfs_index_context *ntfs_index_ctx_get(struct ntfs_inode *ni, __le16 *name, u32 name_len); void ntfs_index_ctx_put(struct ntfs_index_context *ictx); diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 360bebd1ee3fe..f577f7abed544 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -848,6 +848,12 @@ static int ntfs_read_locked_inode(struct inode *vi) a->data.resident.value_offset), le32_to_cpu( a->data.resident.value_length)); + /* A resident list is not validated on load; check it now. */ + if (!ntfs_attr_list_is_valid(ni->attr_list, + ni->attr_list_size)) { + ntfs_error(vi->i_sb, "Corrupt attribute list."); + goto unm_err_out; + } } } skip_attr_list_load: @@ -890,7 +896,6 @@ static int ntfs_read_locked_inode(struct inode *vi) */ if (S_ISDIR(vi->i_mode)) { struct index_root *ir; - u8 *ir_end, *index_end; view_index_meta: /* It is a directory, find index root attribute. */ @@ -940,10 +945,9 @@ static int ntfs_read_locked_inode(struct inode *vi) } ir = (struct index_root *)((u8 *)a + le16_to_cpu(a->data.resident.value_offset)); - ir_end = (u8 *)ir + le32_to_cpu(a->data.resident.value_length); - index_end = (u8 *)&ir->index + - le32_to_cpu(ir->index.index_length); - if (index_end > ir_end) { + if (ntfs_index_root_inconsistent(ni->vol, a, ir, ni->mft_no) || + ntfs_index_entries_inconsistent(ni->vol, &ir->index, + ir->collation_rule, ni->mft_no)) { ntfs_error(vi->i_sb, "Directory index is corrupt."); goto unm_err_out; } @@ -1179,6 +1183,15 @@ static int ntfs_read_locked_inode(struct inode *vi) !S_ISFIFO(vi->i_mode) && !S_ISSOCK(vi->i_mode) && !S_ISLNK(vi->i_mode)) vi->i_flags |= S_IMMUTABLE; + /* + * System files such as $Bitmap and $MFT are maintained by the driver + * itself, and writing them from userspace corrupts the volume. + * Always make them immutable regardless of the sys_immutable option. + * Directories are skipped so the root and $Extend stay usable. + */ + if (ni->mft_no < FILE_first_user && S_ISREG(vi->i_mode)) + vi->i_flags |= S_IMMUTABLE; + /* * The number of 512-byte blocks used on disk (for stat). This is in so * far inaccurate as it doesn't account for any named streams or other @@ -1195,6 +1208,9 @@ static int ntfs_read_locked_inode(struct inode *vi) else vi->i_blocks = ni->allocated_size >> 9; + if (S_ISLNK(vi->i_mode) && ni->target) + vi->i_size = strlen(ni->target); + ntfs_debug("Done."); return 0; unm_err_out: @@ -1483,7 +1499,6 @@ static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi) struct attr_record *a; struct ntfs_attr_search_ctx *ctx; struct index_root *ir; - u8 *ir_end, *index_end; int err = 0; ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no); @@ -1534,9 +1549,9 @@ static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi) } ir = (struct index_root *)((u8 *)a + le16_to_cpu(a->data.resident.value_offset)); - ir_end = (u8 *)ir + le32_to_cpu(a->data.resident.value_length); - index_end = (u8 *)&ir->index + le32_to_cpu(ir->index.index_length); - if (index_end > ir_end) { + if (ntfs_index_root_inconsistent(vol, a, ir, ni->mft_no) || + ntfs_index_entries_inconsistent(vol, &ir->index, + ir->collation_rule, ni->mft_no)) { ntfs_error(vi->i_sb, "Index is corrupt."); goto unm_err_out; } @@ -1994,10 +2009,7 @@ int ntfs_read_inode_mount(struct inode *vi) /* Catch the end of the attribute list. */ if ((u8 *)al_entry == al_end) goto em_put_err_out; - if (!al_entry->length) - goto em_put_err_out; - if ((u8 *)al_entry + 6 > al_end || - (u8 *)al_entry + le16_to_cpu(al_entry->length) > al_end) + if (!ntfs_attr_list_entry_is_valid(al_entry, al_end)) goto em_put_err_out; next_al_entry = (struct attr_list_entry *)((u8 *)al_entry + le16_to_cpu(al_entry->length)); diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c index d3f25d8e29f9d..9bc34572908ef 100644 --- a/fs/ntfs/logfile.c +++ b/fs/ntfs/logfile.c @@ -132,7 +132,7 @@ static bool ntfs_check_restart_area(struct inode *vi, struct restart_page_header { u64 file_size; struct restart_area *ra; - u16 ra_ofs, ra_len, ca_ofs; + u32 ra_ofs, ra_len, ca_ofs; u8 fs_bits; ntfs_debug("Entering."); diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c index a7d10ee41b344..fd20d7abd6f5e 100644 --- a/fs/ntfs/mft.c +++ b/fs/ntfs/mft.c @@ -743,23 +743,6 @@ static int ntfs_test_inode_wb(struct inode *vi, u64 ino, void *data) * * If the mft record is not a FILE record or it is a base mft record, we can * safely write it and return 'true'. - * - * We now know the mft record is an extent mft record. We check if the inode - * corresponding to its base mft record is in icache. If it is not, we cannot - * safely determine the state of the extent inode, so we return 'false'. - * - * We now have the base inode for the extent mft record. We check if it has an - * ntfs inode for the extent mft record attached. If not, it is safe to write - * the extent mft record and we return 'true'. - * - * If the extent inode is attached, we check if it is dirty. If so, we return - * 'false' (letting the standard write_inode path handle it). - * - * If it is not dirty, we attempt to lock the extent mft record. If the lock - * was already taken, it is not safe to write and we return 'false'. - * - * If we manage to obtain the lock we have exclusive access to the extent mft - * record. We set @locked_ni to the now locked ntfs inode and return 'true'. */ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, const struct mft_record *m, struct ntfs_inode **locked_ni, @@ -768,8 +751,7 @@ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, struct super_block *sb = vol->sb; struct inode *mft_vi = vol->mft_ino; struct inode *vi; - struct ntfs_inode *ni, *eni, **extent_nis; - int i; + struct ntfs_inode *ni; struct ntfs_attr na = {0}; ntfs_debug("Entering for inode 0x%llx.", mft_no); @@ -849,100 +831,10 @@ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, mft_no); return true; } - /* - * This is an extent mft record. Check if the inode corresponding to - * its base mft record is in icache and obtain a reference to it if it - * is. - */ - na.mft_no = MREF_LE(m->base_mft_record); - na.state = 0; - ntfs_debug("Mft record 0x%llx is an extent record. Looking for base inode 0x%llx in icache.", - mft_no, na.mft_no); - if (!na.mft_no) { - /* Balance the below iput(). */ - vi = igrab(mft_vi); - WARN_ON(vi != mft_vi); - } else { - vi = find_inode_nowait(sb, na.mft_no, ntfs_test_inode_wb, &na); - if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated) - return false; - } - - if (!vi) - return false; - ntfs_debug("Base inode 0x%llx is in icache.", na.mft_no); - /* - * The base inode is in icache. Check if it has the extent inode - * corresponding to this extent mft record attached. - */ - ni = NTFS_I(vi); - mutex_lock(&ni->extent_lock); - if (ni->nr_extents <= 0) { - /* - * The base inode has no attached extent inodes, write this - * extent mft record. - */ - mutex_unlock(&ni->extent_lock); - *ref_vi = vi; - ntfs_debug("Base inode 0x%llx has no attached extent inodes, write the extent record.", - na.mft_no); - return true; - } - /* Iterate over the attached extent inodes. */ - extent_nis = ni->ext.extent_ntfs_inos; - for (eni = NULL, i = 0; i < ni->nr_extents; ++i) { - if (mft_no == extent_nis[i]->mft_no) { - /* - * Found the extent inode corresponding to this extent - * mft record. - */ - eni = extent_nis[i]; - break; - } - } - /* - * If the extent inode was not attached to the base inode, write this - * extent mft record. - */ - if (!eni) { - mutex_unlock(&ni->extent_lock); - *ref_vi = vi; - ntfs_debug("Extent inode 0x%llx is not attached to its base inode 0x%llx, write the extent record.", - mft_no, na.mft_no); - return true; - } - ntfs_debug("Extent inode 0x%llx is attached to its base inode 0x%llx.", - mft_no, na.mft_no); - /* Take a reference to the extent ntfs inode. */ - atomic_inc(&eni->count); - mutex_unlock(&ni->extent_lock); - /* if extent inode is dirty, write_inode will write it */ - if (NInoDirty(eni)) { - atomic_dec(&eni->count); - *ref_vi = vi; - return false; - } - - /* - * Found the extent inode coresponding to this extent mft record. - * Try to take the mft record lock. - */ - if (unlikely(!mutex_trylock(&eni->mrec_lock))) { - atomic_dec(&eni->count); - *ref_vi = vi; - ntfs_debug("Extent mft record 0x%llx is already locked, do not write it.", - mft_no); - return false; - } - ntfs_debug("Managed to lock extent mft record 0x%llx, write it.", - mft_no); - /* - * The write has to occur while we hold the mft record lock so return - * the locked extent ntfs inode. - */ - *locked_ni = eni; - return true; + ntfs_debug("Mft record 0x%llx is an extent record, skip it.", + mft_no); + return false; } static const char *es = " Leaving inconsistent metadata. Unmount and run chkdsk."; @@ -2745,7 +2637,6 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w s64 vcn = ntfs_pidx_to_cluster(vol, folio->index); s64 end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size); unsigned int folio_sz; - struct runlist_element *rl = NULL; loff_t i_size = i_size_read(vi); ntfs_debug("Entering for inode 0x%llx, attribute type 0x%x, folio index 0x%lx.", @@ -2790,19 +2681,7 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w &tni, &ref_inos[nr_ref_inos])) { unsigned int mft_record_off = 0; s64 vcn_off = vcn; - - /* - * Skip $MFT extent mft records and let them being written - * by writeback to avioid deadlocks. the $MFT runlist - * lock must be taken before $MFT extent mrec_lock is taken. - */ - if (tni && tni->nr_extents < 0 && - tni->ext.base_ntfs_ino == NTFS_I(vol->mft_ino)) { - mutex_unlock(&tni->mrec_lock); - atomic_dec(&tni->count); - iput(vol->mft_ino); - continue; - } + s64 rl_len = 0; /* * The record should be written. If a locked ntfs @@ -2822,8 +2701,12 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w } if (vol->cluster_size < folio_size(folio)) { + struct runlist_element *rl; + down_write(&ni->runlist.lock); rl = ntfs_attr_vcn_to_rl(ni, vcn_off, &lcn); + if (!IS_ERR(rl)) + rl_len = rl->length - (vcn_off - rl->vcn); up_write(&ni->runlist.lock); if (IS_ERR(rl) || lcn < 0) { err = -EIO; @@ -2854,7 +2737,7 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w if (vol->cluster_size == NTFS_BLOCK_SIZE && (mft_record_off || - (rl && rl->length - (vcn_off - rl->vcn) == 1) || + rl_len == 1 || mft_ofs + NTFS_BLOCK_SIZE >= PAGE_SIZE)) folio_sz = NTFS_BLOCK_SIZE; else diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c index c4f82846c58c3..a20ef06087368 100644 --- a/fs/ntfs/namei.c +++ b/fs/ntfs/namei.c @@ -1264,6 +1264,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, struct ntfs_volume *vol = NTFS_SB(sb); struct ntfs_inode *old_ni, *new_ni = NULL; struct ntfs_inode *old_dir_ni = NTFS_I(old_dir), *new_dir_ni = NTFS_I(new_dir); + bool new_dir_first = false; if (NVolShutdown(old_dir_ni->vol)) return -EIO; @@ -1299,36 +1300,39 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, old_inode = old_dentry->d_inode; new_inode = new_dentry->d_inode; old_ni = NTFS_I(old_inode); + if (new_inode) + new_ni = NTFS_I(new_inode); + if (old_dir != new_dir) + new_dir_first = is_subdir(new_dentry->d_parent, + old_dentry->d_parent); if (!(vol->vol_flags & VOLUME_IS_DIRTY)) ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); mutex_lock_nested(&old_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); - mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + if (new_ni) + mutex_lock_nested(&new_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_2); - if (NInoBeingDeleted(old_ni) || NInoBeingDeleted(old_dir_ni)) { + if (old_dir == new_dir) { + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + } else if (new_dir_first) { + mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); + } else { + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); + } + + if (NInoBeingDeleted(old_ni) || NInoBeingDeleted(old_dir_ni) || + (new_ni && NInoBeingDeleted(new_ni)) || + (old_dir != new_dir && NInoBeingDeleted(new_dir_ni))) { err = -ENOENT; - goto unlock_old; + goto err_out; } is_dir = S_ISDIR(old_inode->i_mode); if (new_inode) { - new_ni = NTFS_I(new_inode); - mutex_lock_nested(&new_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_2); - if (old_dir != new_dir) { - mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); - if (NInoBeingDeleted(new_dir_ni)) { - err = -ENOENT; - goto err_out; - } - } - - if (NInoBeingDeleted(new_ni)) { - err = -ENOENT; - goto err_out; - } - if (is_dir) { struct mft_record *ni_mrec; @@ -1346,14 +1350,6 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, err = ntfs_delete(new_ni, new_dir_ni, uname_new, new_name_len, false); if (err) goto err_out; - } else { - if (old_dir != new_dir) { - mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); - if (NInoBeingDeleted(new_dir_ni)) { - err = -ENOENT; - goto err_out; - } - } } err = __ntfs_link(old_ni, new_dir_ni, uname_new, new_name_len); @@ -1384,13 +1380,17 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, inode_inc_iversion(new_dir); err_out: - if (old_dir != new_dir) + if (old_dir == new_dir) { + mutex_unlock(&old_dir_ni->mrec_lock); + } else if (new_dir_first) { + mutex_unlock(&old_dir_ni->mrec_lock); mutex_unlock(&new_dir_ni->mrec_lock); - if (new_inode) + } else { + mutex_unlock(&new_dir_ni->mrec_lock); + mutex_unlock(&old_dir_ni->mrec_lock); + } + if (new_ni) mutex_unlock(&new_ni->mrec_lock); - -unlock_old: - mutex_unlock(&old_dir_ni->mrec_lock); mutex_unlock(&old_ni->mrec_lock); if (uname_new) kmem_cache_free(ntfs_name_cache, uname_new); @@ -1532,8 +1532,7 @@ static int ntfs_link(struct dentry *old_dentry, struct inode *dir, if (uname_len < 0) { if (uname_len != -ENAMETOOLONG) ntfs_error(sb, "Failed to convert name to unicode."); - err = -ENOMEM; - goto out; + return -ENOMEM; } if (!(vol->vol_flags & VOLUME_IS_DIRTY)) @@ -1563,7 +1562,7 @@ static int ntfs_link(struct dentry *old_dentry, struct inode *dir, mutex_unlock(&ni->mrec_lock); out: - kfree(uname); + kmem_cache_free(ntfs_name_cache, uname); return err; } diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c index e7de3d01257e7..15f1ae530ae13 100644 --- a/fs/ntfs/runlist.c +++ b/fs/ntfs/runlist.c @@ -763,7 +763,7 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * buf = (u8 *)attr + le16_to_cpu(attr->data.non_resident.mapping_pairs_offset); attr_end = (u8 *)attr + le32_to_cpu(attr->length); - if (unlikely(buf < (u8 *)attr || buf > attr_end)) { + if (unlikely(buf < (u8 *)attr || buf >= attr_end)) { ntfs_error(vol->sb, "Corrupt attribute."); return ERR_PTR(-EIO); } @@ -811,7 +811,7 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * */ b = *buf & 0xf; if (b) { - if (unlikely(buf + b > attr_end)) + if (unlikely(buf + b >= attr_end)) goto io_error; for (deltaxcn = (s8)buf[b--]; b; b--) deltaxcn = (deltaxcn << 8) + buf[b]; @@ -855,12 +855,16 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * u8 b2 = *buf & 0xf; b = b2 + ((*buf >> 4) & 0xf); - if (buf + b > attr_end) + if (buf + b >= attr_end) goto io_error; for (deltaxcn = (s8)buf[b--]; b > b2; b--) deltaxcn = (deltaxcn << 8) + buf[b]; /* Change the current lcn to its new value. */ - lcn += deltaxcn; + if (unlikely(check_add_overflow(lcn, deltaxcn, &lcn))) { + ntfs_error(vol->sb, + "LCN overflow in mapping pairs array."); + goto err_out; + } #ifdef DEBUG /* * On NTFS 1.2-, apparently can have lcn == -1 to diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c index 9e321cc2febe7..fc9e2724039b9 100644 --- a/fs/ntfs/super.c +++ b/fs/ntfs/super.c @@ -452,25 +452,36 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label) goto out; } - if (!ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0, - ctx)) - ntfs_attr_record_rm(ctx); + ret = ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0, + ctx); + if (!ret) + ret = ntfs_attr_record_rm(ctx); + else if (ret == -ENOENT) + ret = 0; ntfs_attr_put_search_ctx(ctx); + if (ret) + goto out; ret = ntfs_resident_attr_record_add(vol_ni, AT_VOLUME_NAME, AT_UNNAMED, 0, (u8 *)uname, uname_len * sizeof(__le16), 0); out: - mutex_unlock(&vol_ni->mrec_lock); - kvfree(uname); - if (ret >= 0) { - kfree(vol->volume_label); + char *old_label; + + mutex_lock(&vol->volume_label_lock); + old_label = vol->volume_label; vol->volume_label = new_label; + mutex_unlock(&vol->volume_label_lock); + + kfree(old_label); mark_inode_dirty_sync(vol->vol_ino); ret = 0; - } else { - kfree(new_label); } + mutex_unlock(&vol_ni->mrec_lock); + kvfree(uname); + + if (ret < 0) + kfree(new_label); return ret; } @@ -1323,7 +1334,6 @@ static bool load_and_init_upcase(struct ntfs_volume *vol) u8 *addr; pgoff_t index, max_index; unsigned int size; - int i, max; ntfs_debug("Entering."); /* Read upcase table and setup vol->upcase and vol->upcase_len. */ @@ -1374,16 +1384,11 @@ static bool load_and_init_upcase(struct ntfs_volume *vol) mutex_unlock(&ntfs_lock); return true; } - max = default_upcase_len; - if (max > vol->upcase_len) - max = vol->upcase_len; - for (i = 0; i < max; i++) - if (vol->upcase[i] != default_upcase[i]) - break; - if (i == max) { + if (default_upcase_len == vol->upcase_len && + !memcmp(vol->upcase, default_upcase, + default_upcase_len * sizeof(*default_upcase))) { kvfree(vol->upcase); vol->upcase = default_upcase; - vol->upcase_len = max; ntfs_nr_upcase_users++; mutex_unlock(&ntfs_lock); ntfs_debug("Volume specified $UpCase matches default. Using default."); @@ -1531,6 +1536,7 @@ static bool load_system_files(struct ntfs_volume *vol) vol->volume_label = NULL; } + ntfs_attr_reinit_search_ctx(ctx); if (ntfs_attr_lookup(AT_VOLUME_INFORMATION, NULL, 0, 0, 0, NULL, 0, ctx) || ctx->attr->non_resident || ctx->attr->flags) { ntfs_attr_put_search_ctx(ctx); @@ -1954,7 +1960,7 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) struct address_space *mapping = vol->lcnbmp_ino->i_mapping; struct folio *folio; pgoff_t index, max_index; - struct file_ra_state *ra; + struct file_ra_state ra = { 0 }; ntfs_debug("Entering."); /* Serialize accesses to the cluster bitmap. */ @@ -1962,11 +1968,7 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) if (NVolFreeClusterKnown(vol)) return atomic64_read(&vol->free_clusters); - ra = kzalloc(sizeof(*ra), GFP_NOFS); - if (!ra) - return 0; - - file_ra_state_init(ra, mapping); + file_ra_state_init(&ra, mapping); /* * Convert the number of bits into bytes rounded up, then convert into @@ -1985,7 +1987,7 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) * Get folio from page cache, getting it from backing store * if necessary, and increment the use count. */ - folio = ntfs_get_locked_folio(mapping, index, max_index, ra); + folio = ntfs_get_locked_folio(mapping, index, max_index, &ra); /* Ignore pages which errored synchronously. */ if (IS_ERR(folio)) { @@ -2024,7 +2026,6 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) else atomic64_set(&vol->free_clusters, nr_free); - kfree(ra); NVolSetFreeClusterKnown(vol); wake_up_all(&vol->free_waitq); ntfs_debug("Exiting."); @@ -2079,15 +2080,11 @@ static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, struct address_space *mapping = vol->mftbmp_ino->i_mapping; struct folio *folio; pgoff_t index; - struct file_ra_state *ra; + struct file_ra_state ra = { 0 }; ntfs_debug("Entering."); - ra = kzalloc(sizeof(*ra), GFP_NOFS); - if (!ra) - return 0; - - file_ra_state_init(ra, mapping); + file_ra_state_init(&ra, mapping); /* Use multiples of 4 bytes, thus max_size is PAGE_SIZE / 4. */ ntfs_debug("Reading $MFT/$BITMAP, max_index = 0x%lx, max_size = 0x%lx.", @@ -2099,7 +2096,7 @@ static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, * Get folio from page cache, getting it from backing store * if necessary, and increment the use count. */ - folio = ntfs_get_locked_folio(mapping, index, max_index, ra); + folio = ntfs_get_locked_folio(mapping, index, max_index, &ra); /* Ignore pages which errored synchronously. */ if (IS_ERR(folio)) { @@ -2131,7 +2128,6 @@ static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, else atomic64_set(&vol->free_mft_records, nr_free); - kfree(ra); ntfs_debug("Exiting."); return nr_free; } @@ -2530,8 +2526,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) } /* Error exit code path. */ unl_upcase_iput_tmp_ino_err_out_now: - if (vol->lcn_empty_bits_per_page) - kvfree(vol->lcn_empty_bits_per_page); /* * Decrease the number of upcase users and destroy the global default * upcase table if necessary. @@ -2551,6 +2545,9 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) /* Errors at this stage are irrelevant. */ err_out_now: sb->s_fs_info = NULL; + kvfree(vol->lcn_empty_bits_per_page); + kfree(vol->volume_label); + unload_nls(vol->nls_map); kfree(vol); ntfs_debug("Failed, returning -EINVAL."); lockdep_on(); @@ -2631,6 +2628,7 @@ static int ntfs_init_fs_context(struct fs_context *fc) NVolSetCaseSensitive(vol); init_rwsem(&vol->mftbmp_lock); init_rwsem(&vol->lcnbmp_lock); + mutex_init(&vol->volume_label_lock); fc->s_fs_info = vol; fc->ops = &ntfs_context_ops; @@ -2649,7 +2647,7 @@ MODULE_ALIAS_FS("ntfs"); static int ntfs_workqueue_init(void) { - ntfs_wq = alloc_workqueue("ntfs-bg-io", 0, 0); + ntfs_wq = alloc_workqueue("ntfs-bg-io", WQ_PERCPU, 0); if (!ntfs_wq) return -ENOMEM; return 0; diff --git a/fs/ntfs/volume.h b/fs/ntfs/volume.h index af41427ec622c..99e642f475f78 100644 --- a/fs/ntfs/volume.h +++ b/fs/ntfs/volume.h @@ -72,6 +72,7 @@ * @vol_flags: Volume flags. * @major_ver: Ntfs major version of volume. * @minor_ver: Ntfs minor version of volume. + * @volume_label_lock: protects @volume_label. * @volume_label: volume label. * @root_ino: The VFS inode of the root directory. * @secure_ino: The VFS inode of $Secure (NTFS3.0+ only, otherwise NULL). @@ -133,6 +134,7 @@ struct ntfs_volume { struct inode *logfile_ino; struct inode *lcnbmp_ino; struct rw_semaphore lcnbmp_lock; + struct mutex volume_label_lock; struct inode *vol_ino; __le16 vol_flags; u8 major_ver; diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index e61c5bf7e27e4..ff0bf45759480 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -1004,6 +1004,7 @@ int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, struct ATTRIB *attr, *attr_b; struct ATTR_LIST_ENTRY *le, *le_b; struct mft_inode *mi, *mi_b; + struct page *page; CLST hint, svcn, to_alloc, evcn1, next_svcn, asize, end, vcn0; CLST alloc, evcn; unsigned fr; @@ -1042,10 +1043,13 @@ int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, *lcn = RESIDENT_LCN; *len = data_size; if (res && data_size) { - *res = kmemdup(resident_data(attr_b), data_size, - GFP_KERNEL); - if (!*res) + page = alloc_page(GFP_KERNEL); + if (!page) { err = -ENOMEM; + } else { + *res = page_address(page); + memcpy(*res, resident_data(attr_b), data_size); + } } goto out; } diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 7b035da63c121..974c55ae2c014 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -1330,7 +1330,7 @@ int ni_expand_list(struct ntfs_inode *ni) { int err = 0; u32 asize, done = 0; - struct ATTRIB *attr, *ins_attr; + struct ATTRIB *attr, *ins_attr = NULL; struct ATTR_LIST_ENTRY *le; bool is_mft = ni->mi.rno == MFT_REC_MFT; struct MFT_REF ref; @@ -1363,7 +1363,7 @@ int ni_expand_list(struct ntfs_inode *ni) le16_to_cpu(attr->name_off), true, &ins_attr, NULL, NULL); - if (err) + if (err || !ins_attr) goto out; memcpy(ins_attr, attr, asize); @@ -2800,8 +2800,8 @@ int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, err = ni_add_name(new_dir_ni, ni, new_de); if (!err) { err = ni_remove_name(dir_ni, ni, de, &de2, &undo); - WARN_ON(err && - ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)); + if (err && ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)) + _ntfs_bad_inode(&ni->vfs_inode); } /* @@ -2859,6 +2859,11 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) return err; } + if (!clen) { + /* Corrupted file. */ + return -EINVAL; + } + if (lcn == RESIDENT_LCN) { /* clen - resident size in bytes. clen == ni->vfs_inode.i_size */ if (offset >= clen) { @@ -2909,10 +2914,6 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) } } - if (!clen) { - /* Corrupted file. */ - return -EINVAL; - } } } diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index acfa18b84401e..7a75aeef5e5dc 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -764,8 +764,19 @@ static bool check_rstbl(const struct RESTART_TABLE *rt, size_t bytes) /* * Walk through the list headed by the first entry to make * sure none of the entries are currently being used. + * + * Bound traversal by ne (rt->used) to defeat a crafted on-disk + * cycle in the free chain. Each entry in a legitimate free + * list is unique, so a chain that visits more than ne slots + * is malformed. Without this guard, an attacker-controlled + * RESTART_TABLE with a self-loop or A->B->A cycle whose + * offsets satisfy the existing alignment + in-bounds guards + * spins forever at mount time. */ - for (off = ff; off;) { + for (off = ff, i = 0; off; i++) { + if (i > ne) + return false; + if (off == RESTART_ENTRY_ALLOCATED) return false; @@ -1172,7 +1183,7 @@ static int read_log_page(struct ntfs_log *log, u32 vbo, goto out; if (page_buf->rhdr.sign != NTFS_FFFF_SIGNATURE) - ntfs_fix_post_read(&page_buf->rhdr, PAGE_SIZE, false); + ntfs_fix_post_read(&page_buf->rhdr, log->page_size, false); if (page_buf != *buffer) memcpy(*buffer, Add2Ptr(page_buf, page_off), bytes); @@ -3325,6 +3336,17 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, nsize = ALIGN(nsize, 8); data_off = le16_to_cpu(attr->res.data_off); + /* + * aoff comes from the on-disk lrh->attr_off. Forbid + * writes that begin below the resident attribute's + * data_off (which would overwrite the resident header), + * and forbid aoff + dlen < data_off, which would make + * the data_size assignment below underflow to ~4 GiB. + */ + if (aoff < data_off || aoff + dlen < data_off || + aoff + dlen > asize) + goto dirty_vol; + if (nsize < asize) { memmove(Add2Ptr(attr, aoff), data, dlen); data = NULL; // To skip below memmove(). @@ -3368,7 +3390,10 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, memmove(Add2Ptr(attr, aoff), data, dlen); if (run_get_highest_vcn(le64_to_cpu(attr->nres.svcn), - attr_run(attr), &t64)) { + attr_run(attr), + le32_to_cpu(attr->size) - + le16_to_cpu(attr->nres.run_off), + &t64)) { goto dirty_vol; } @@ -3497,6 +3522,18 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + /* + * e->view.data_off and dlen come from the on-disk + * INDEX_ROOT entry / LRH. The neighbouring read sites + * (e.g. fs/ntfs3/index.c) check that + * view.data_off + view.data_size <= e->size; mirror that + * bound here so the memmove cannot reach past the entry. + */ + if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || + le16_to_cpu(e->view.data_off) + dlen > + le16_to_cpu(e->size)) + goto dirty_vol; + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); mi->dirty = true; @@ -3570,9 +3607,23 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, } e1 = Add2Ptr(e, esize); - nsize = esize; used = le32_to_cpu(hdr->used); + /* + * Reject crafted entries whose e->size makes e + esize + * point past the INDEX_HDR's used boundary. Without this, + * PtrOffset(e1, hdr + used) underflows to a quasi-infinite + * size_t when fed to the memmove() below. + * + * Also reject esize == 0: memmove(e, e, ...) is a no-op and + * leaves hdr->used unchanged, masking the crafted entry. + */ + if (!esize || Add2Ptr(e, esize) > Add2Ptr(hdr, used) || + PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + + nsize = esize; + memmove(e, e1, PtrOffset(e1, Add2Ptr(hdr, used))); hdr->used = cpu_to_le32(used - nsize); @@ -3689,6 +3740,12 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, goto dirty_vol; } + /* See UpdateRecordDataRoot for the rationale. */ + if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || + le16_to_cpu(e->view.data_off) + dlen > + le16_to_cpu(e->size)) + goto dirty_vol; + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); a_dirty = true; @@ -3796,11 +3853,7 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) log->l_size = log->orig_file_size = ni->vfs_inode.i_size; /* Get the size of page. NOTE: To replay we can use default page. */ -#if PAGE_SIZE >= DefaultLogPageSize && PAGE_SIZE <= DefaultLogPageSize * 2 log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, true); -#else - log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, false); -#endif if (!log->page_size) { err = -EINVAL; goto out; @@ -3938,9 +3991,28 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) */ t32 = le32_to_cpu(log->rst_info.r_page->sys_page_size); if (log->page_size != t32) { + u32 old_page_size = log->page_size; + log->l_size = log->orig_file_size; log->page_size = norm_file_page(t32, &log->l_size, t32 == DefaultLogPageSize); + + /* + * If the adopted on-disk page size is larger than the size used + * to allocate one_page_buf above, grow the scratch buffer so a + * later read_log_page() cannot overflow it. + */ + if (log->page_size > old_page_size) { + void *buf; + + buf = krealloc(log->one_page_buf, log->page_size, + GFP_NOFS); + if (!buf) { + err = -ENOMEM; + goto out; + } + log->one_page_buf = buf; + } } if (log->page_size != t32 || @@ -4219,13 +4291,26 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) if (rst->major_ver) goto end_conv_1; /* reduce tab pressure. */ + t16 = le16_to_cpu(dptbl->size); + if (t16 < sizeof(struct DIR_PAGE_ENTRY)) { + log->set_dirty = true; + goto out; + } + + t32 = (t16 - sizeof(struct DIR_PAGE_ENTRY)) / sizeof(u64); + dp = NULL; while ((dp = enum_rstbl(dptbl, dp))) { struct DIR_PAGE_ENTRY_32 *dp0 = (struct DIR_PAGE_ENTRY_32 *)dp; - // NOTE: Danger. Check for of boundary. + u32 lcns = le32_to_cpu(dp->lcns_follow); + + if (lcns > t32) { + log->set_dirty = true; + goto out; + } + memmove(&dp->vcn, &dp0->vcn_low, - 2 * sizeof(u64) + - le32_to_cpu(dp->lcns_follow) * sizeof(u64)); + 2 * sizeof(u64) + lcns * sizeof(u64)); } end_conv_1: @@ -4547,12 +4632,34 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) * whole routine a loop, case Lcns do not fit below. */ t16 = le16_to_cpu(lrh->lcns_follow); + t32 = le32_to_cpu(dp->lcns_follow); + if (le64_to_cpu(lrh->target_vcn) < le64_to_cpu(dp->vcn)) { + err = -EINVAL; + goto out; + } + + /* + * find_dp() only validates that target_vcn is the first + * cluster covered by dp. The walk through lrh->lcns_follow + * further entries must stay within the allocated + * dp->page_lcns[] array, which is sized by dp->lcns_follow. + */ + if (le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn) + t16 > + le32_to_cpu(dp->lcns_follow)) { + err = -EINVAL; + log->set_dirty = true; + goto out; + } + for (i = 0; i < t16; i++) { size_t j = (size_t)(le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn)); + if (j >= t32 || i >= t32 - j) { + err = -EINVAL; + goto out; + } dp->page_lcns[j + i] = lrh->page_lcns[i]; } - goto next_log_record_analyze; case DeleteDirtyClusters: { diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 5344b29b0577f..ade2762259995 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -1742,6 +1742,22 @@ static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni, hdr_used = le32_to_cpu(hdr->used); hdr_total = le32_to_cpu(hdr->total); + /* + * The destination INDEX_BUFFER has 'hdr_total' bytes of payload + * available after the header, of which 'hdr_used' are already + * consumed by the single terminal END entry installed by + * indx_new(). A crafted image can present a resident root whose + * non-last entries (summing to 'to_move') exceed what fits in + * this buffer; copying them unchecked would overrun the + * kmalloc(1u << indx->index_bits) allocation backing the new + * buffer. Reject the copy in that case. + */ + if (to_move > hdr_total - hdr_used) { + err = -EINVAL; + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + goto out_put_n; + } + /* Copy root entries into new buffer. */ hdr_insert_head(hdr, re, to_move); @@ -1846,6 +1862,20 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, memcpy(up_e, sp, sp_size); used1 = le32_to_cpu(hdr1->used); + + /* + * hdr_find_split does not validate per-entry sizes, so a crafted + * NTFS_DE whose le16 size field is out of range can place sp such + * that (PtrOffset(hdr1, sp) + sp_size) exceeds used1. Without this + * guard the u32 'used = used1 - to_copy - sp_size' underflows and + * the subsequent memmove count becomes a near-4-GiB value, + * triggering an out-of-bounds kernel write. + */ + if (PtrOffset(hdr1, sp) + sp_size > used1) { + err = -EINVAL; + goto out; + } + hdr1_saved = kmemdup(hdr1, used1, GFP_NOFS); if (!hdr1_saved) { err = -ENOMEM; @@ -2022,13 +2052,21 @@ int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni, static struct indx_node *indx_find_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, const struct INDEX_ROOT *root, - __le64 vbn, struct indx_node *n) + __le64 vbn, struct indx_node *n, + int depth) { int err; const struct NTFS_DE *e; struct indx_node *r; const struct INDEX_HDR *hdr = n ? &n->index->ihdr : &root->ihdr; + /* + * Limit recursion depth to prevent stack overflow from crafted + * images. Use the same bound as the fnd->nodes array (20). + */ + if (depth > ARRAY_SIZE(((struct ntfs_fnd *)NULL)->nodes)) + return ERR_PTR(-EINVAL); + /* Step 1: Scan one level. */ for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) { if (!e) @@ -2049,7 +2087,8 @@ static struct indx_node *indx_find_buffer(struct ntfs_index *indx, if (err) return ERR_PTR(err); - r = indx_find_buffer(indx, ni, root, vbn, n); + r = indx_find_buffer(indx, ni, root, vbn, n, + depth + 1); if (r) return r; } @@ -2462,7 +2501,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, fnd_clear(fnd); - in = indx_find_buffer(indx, ni, root, sub_vbn, NULL); + in = indx_find_buffer(indx, ni, root, sub_vbn, NULL, 0); if (IS_ERR(in)) { err = PTR_ERR(in); goto out; diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 42af1abe17f88..b6c06fe62a9d8 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -609,10 +609,7 @@ static void ntfs_iomap_read_end_io(struct bio *bio) static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter, struct iomap_read_folio_ctx *ctx) { - struct bio *bio = ctx->read_ctx; - - bio->bi_end_io = ntfs_iomap_read_end_io; - submit_bio(bio); + iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io); } static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { @@ -801,7 +798,7 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (lcn == RESIDENT_LCN) { if (offset >= clen) { - kfree(res); + __free_page(virt_to_page(res)); if (flags & IOMAP_REPORT) { /* special code for report. */ return -ENOENT; @@ -921,7 +918,7 @@ static int ntfs_iomap_end(struct inode *inode, loff_t pos, loff_t length, out: if (iomap->type == IOMAP_INLINE) { - kfree(iomap->private); + __free_page(virt_to_page(iomap->private)); iomap->private = NULL; } diff --git a/fs/ntfs3/lznt.c b/fs/ntfs3/lznt.c index fdc9b2ebf3410..f818d97850049 100644 --- a/fs/ntfs3/lznt.c +++ b/fs/ntfs3/lznt.c @@ -240,7 +240,7 @@ static inline ssize_t decompress_chunk(u8 *unc, u8 *unc_end, const u8 *cmpr, if (up - unc > LZNT_CHUNK_SIZE) return -EINVAL; /* Correct index */ - while (unc + s_max_off[index] < up) + while (index < ARRAY_SIZE(s_max_off) - 1 && unc + s_max_off[index] < up) index += 1; /* Check the current flag for zero. */ diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c index b2af8f695e60f..64cde1a856f44 100644 --- a/fs/ntfs3/namei.c +++ b/fs/ntfs3/namei.c @@ -340,7 +340,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, ntfs_sync_inode(dir); if (IS_DIRSYNC(new_dir)) - ntfs_sync_inode(inode); + ntfs_sync_inode(new_dir); } if (dir_ni != new_dir_ni) diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index bbf3b6a1dcbee..d53febc2559c0 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -877,7 +877,8 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, #else #define run_unpack_ex run_unpack #endif -int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn); +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn); int run_clone(const struct runs_tree *run, struct runs_tree *new_run); bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done); CLST run_len(const struct runs_tree *run); diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 1ce7d92fb2748..ad7db67514ef7 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -1205,18 +1205,23 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, * Return the highest vcn from a mapping pairs array * it used while replaying log file. */ -int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn) +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn) { + const u8 *run_last = run_buf + run_buf_size; u64 vcn64 = vcn; u8 size_size; - while ((size_size = *run_buf & 0xF)) { + while (run_buf < run_last && (size_size = *run_buf & 0xF)) { u8 offset_size = *run_buf++ >> 4; u64 len; if (size_size > 8 || offset_size > 8) return -EINVAL; + if (run_buf + size_size + offset_size > run_last) + return -EINVAL; + len = run_unpack_s64(run_buf, size_size, 0); if (!len) return -EINVAL; @@ -1292,9 +1297,12 @@ bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done) if (r_end > end) { /* Remove a middle part, split. */ + CLST tail_lcn = r->lcn == SPARSE_LCN ? + SPARSE_LCN : (r->lcn + (end - r->vcn)); + *done += len; r->len = d; - return run_add_entry(run, end, r->lcn, r_end - end, + return run_add_entry(run, end, tail_lcn, r_end - end, false); } /* Remove tail of run .*/ diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index 9eeac0ab2b714..372cf2f2e89b3 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -851,6 +851,12 @@ static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de, return err; } +static bool ntfs_is_reserved_lxattr(const char *name) +{ + return !strcmp(name, "$LXUID") || !strcmp(name, "$LXGID") || + !strcmp(name, "$LXMOD") || !strcmp(name, "$LXDEV"); +} + /* * ntfs_setxattr - inode_operations::setxattr */ @@ -955,6 +961,12 @@ static noinline int ntfs_setxattr(const struct xattr_handler *handler, goto out; } + /* Do not allow non privileged users to change $LXUID/$LXGID... */ + if (ntfs_is_reserved_lxattr(name) && !capable(CAP_SYS_ADMIN)) { + err = -EPERM; + goto out; + } + /* Deal with NTFS extended attribute. */ err = ntfs_set_ea(inode, name, strlen(name), value, size, flags, 0, NULL); diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c index 6e5fd3f12a84c..be09e766ac1fc 100644 --- a/fs/ocfs2/alloc.c +++ b/fs/ocfs2/alloc.c @@ -7576,7 +7576,7 @@ int ocfs2_trim_mainbm(struct super_block *sb, struct fstrim_range *range) len = range->len >> osb->s_clustersize_bits; minlen = range->minlen >> osb->s_clustersize_bits; - if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize) + if (minlen >= osb->bitmap_cpg || range->len < osb->s_clustersize) return -EINVAL; trace_ocfs2_trim_mainbm(start, len, minlen); diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 6ec198bdab121..4acdbb70882cd 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -2372,6 +2372,15 @@ static int ocfs2_dio_end_io_write(struct inode *inode, unlock: up_write(&oi->ip_alloc_sem); + if (data_ac) { + ocfs2_free_alloc_context(data_ac); + data_ac = NULL; + } + if (meta_ac) { + ocfs2_free_alloc_context(meta_ac); + meta_ac = NULL; + } + /* everything looks good, let's start the cleanup */ if (!ret && dwc->dw_orphaned) { BUG_ON(dwc->dw_writer_pid != task_pid_nr(current)); @@ -2383,10 +2392,6 @@ static int ocfs2_dio_end_io_write(struct inode *inode, ocfs2_inode_unlock(inode, 1); brelse(di_bh); out: - if (data_ac) - ocfs2_free_alloc_context(data_ac); - if (meta_ac) - ocfs2_free_alloc_context(meta_ac); ocfs2_run_deallocs(osb, &dealloc); ocfs2_dio_free_write_ctx(inode, dwc); diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c index 701d27d908d4b..6114299b121e5 100644 --- a/fs/ocfs2/buffer_head_io.c +++ b/fs/ocfs2/buffer_head_io.c @@ -350,8 +350,6 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, wait_on_buffer(bh); put_bh(bh); bhs[i] = NULL; - } else if (bh && buffer_uptodate(bh)) { - clear_buffer_uptodate(bh); } continue; } @@ -380,8 +378,11 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, BUG_ON(buffer_jbd(bh)); clear_buffer_needs_validate(bh); status = validate(sb, bh); - if (status) + if (status) { + if (buffer_uptodate(bh)) + clear_buffer_uptodate(bh); goto read_failure; + } } } diff --git a/fs/ocfs2/dlm/dlmdebug.c b/fs/ocfs2/dlm/dlmdebug.c index fe4fdd09bae39..5645673586201 100644 --- a/fs/ocfs2/dlm/dlmdebug.c +++ b/fs/ocfs2/dlm/dlmdebug.c @@ -560,6 +560,7 @@ static int debug_lockres_open(struct inode *inode, struct file *file) struct dlm_ctxt *dlm = inode->i_private; struct debug_lockres *dl; void *buf; + int status = -ENOMEM; buf = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!buf) @@ -572,16 +573,23 @@ static int debug_lockres_open(struct inode *inode, struct file *file) dl->dl_len = PAGE_SIZE; dl->dl_buf = buf; - dlm_grab(dlm); - dl->dl_ctxt = dlm; + /* ->release uses dl_ctxt after open, so it needs a real pin. */ + dl->dl_ctxt = dlm_grab(dlm); + if (!dl->dl_ctxt) { + status = -ENOENT; + goto bailseq; + } return 0; +bailseq: + seq_release_private(inode, file); bailfree: kfree(buf); bail: - mlog_errno(-ENOMEM); - return -ENOMEM; + if (status != -ENOENT) + mlog_errno(status); + return status; } static int debug_lockres_release(struct inode *inode, struct file *file) diff --git a/fs/ocfs2/dlmglue.c b/fs/ocfs2/dlmglue.c index 7283bb2c5a31b..a23dd8f86c895 100644 --- a/fs/ocfs2/dlmglue.c +++ b/fs/ocfs2/dlmglue.c @@ -3134,6 +3134,22 @@ static void *ocfs2_dlm_seq_next(struct seq_file *m, void *v, loff_t *pos) * - Add last pr/ex unlock times and first lock wait time in usecs */ #define OCFS2_DLM_DEBUG_STR_VERSION 4 + +/* + * The debug iterator snapshots lockres by value, so a userspace-stack LVB + * pointer copied from the original lockres must be rebased to the copied + * lksb before the dump walks the raw bytes. + */ +static void ocfs2_dlm_seq_rebase_lvb(struct ocfs2_lock_res *lockres) +{ + if (!ocfs2_stack_supports_plocks()) + return; + + if (lockres->l_lksb.lksb_fsdlm.sb_lvbptr) + lockres->l_lksb.lksb_fsdlm.sb_lvbptr = + (char *)&lockres->l_lksb + sizeof(struct dlm_lksb); +} + static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) { int i; @@ -3191,6 +3207,7 @@ static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) lockres->l_blocking); /* Dump the raw LVB */ + ocfs2_dlm_seq_rebase_lvb(lockres); lvb = ocfs2_dlm_lvb(&lockres->l_lksb); for(i = 0; i < DLM_LVB_LEN; i++) seq_printf(m, "0x%x\t", lvb[i]); diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index a510a0eb1adcc..662dbc845b8bd 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -64,7 +65,40 @@ static int ocfs2_filecheck_read_inode_block_full(struct inode *inode, static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, struct buffer_head *bh); static int ocfs2_filecheck_repair_inode_block(struct super_block *sb, - struct buffer_head *bh); + struct buffer_head *bh); + +static bool ocfs2_valid_inode_mode(umode_t mode) +{ + return fs_umode_to_ftype(mode) != FT_UNKNOWN; +} + +static bool ocfs2_dinode_has_unexpected_rdev(struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + + return !S_ISCHR(mode) && !S_ISBLK(mode) && di->id1.dev1.i_rdev != 0; +} + +static bool ocfs2_dinode_has_size_without_clusters(struct super_block *sb, + struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) + return false; + if (!le64_to_cpu(di->i_size) || le32_to_cpu(di->i_clusters)) + return false; + + if (S_ISDIR(mode)) + return true; + + return !ocfs2_sparse_alloc(OCFS2_SB(sb)) && S_ISREG(mode); +} void ocfs2_set_inode_flags(struct inode *inode) { @@ -1494,6 +1528,86 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + /* + * Reject dinodes whose i_mode does not name one of the seven + * canonical POSIX file types. ocfs2_populate_inode() copies + * i_mode verbatim into inode->i_mode and then dispatches via + * switch (mode & S_IFMT) to file/dir/symlink/special_file iops; + * an unrecognised type falls into ocfs2_special_file_iops with + * init_special_inode(), which interprets i_rdev. Constrain the + * type here so the dispatch only ever sees a value mkfs.ocfs2 / + * VFS can produce. + */ + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + goto bail; + } + + /* + * id1.dev1.i_rdev is the device-number arm of the id1 union and + * is only meaningful for character and block device inodes. For + * any other regular user-visible file type the on-disk value + * must be zero. ocfs2_populate_inode() currently runs + * + * inode->i_rdev = huge_decode_dev(le64_to_cpu(fe->id1.dev1.i_rdev)); + * + * unconditionally, before the S_IFMT switch decides whether the + * inode is a special file. As a result, an i_rdev value present + * on a non-device inode is silently published into the in-core + * inode; a subsequent forced re-read or in-core mode mutation + * (cluster peer with raw write access to the shared LUN, + * on-disk corruption, or a separately forged dinode) can then + * expose the attacker-controlled device number to + * init_special_inode() without ever showing an unusual i_mode + * at validation time. + * + * System inodes (OCFS2_SYSTEM_FL) legitimately use the bitmap1 + * and journal1 arms of the same union (allocator i_used / + * i_total counters and the journal ij_flags / + * ij_recovery_generation pair); those bytes are not an i_rdev + * and must not be checked here. Restrict the cross-check to + * non-system inodes, which is the full attacker-controllable + * surface. + */ + if (ocfs2_dinode_has_unexpected_rdev(di)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + goto bail; + } + + /* + * Non-inline directories must not have i_size without allocated + * clusters: directory growth adds storage before advancing i_size, + * and readdir walks i_size block-by-block. A forged directory + * with zero clusters and a huge i_size would repeatedly fault on + * holes while advancing through the claimed size. + * + * Non-inline regular files have the same invariant on non-sparse + * volumes. Sparse regular files are different: truncate can + * legitimately grow i_size without allocating clusters, so keep + * the sparse-alloc carveout for S_IFREG only. System inodes and + * inline-data dinodes have their own storage rules. + */ + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + rc = ocfs2_error(sb, + "Invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + rc = ocfs2_error(sb, + "Invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + goto bail; + } + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { struct ocfs2_inline_data *data = &di->id2.i_data; @@ -1525,6 +1639,29 @@ int ocfs2_validate_inode_block(struct super_block *sb, } } + if (S_ISLNK(le16_to_cpu(di->i_mode)) && + !le32_to_cpu(di->i_clusters)) { + int max_inline = ocfs2_fast_symlink_chars(sb); + u64 i_size = le64_to_cpu(di->i_size); + + if (i_size >= max_inline) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: fast symlink i_size %llu exceeds max %d\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)i_size, + max_inline - 1); + goto bail; + } + + if (strnlen((char *)di->id2.i_symlink, i_size + 1) != i_size) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: fast symlink is not NUL-terminated at i_size %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)i_size); + goto bail; + } + } + if (le32_to_cpu(di->i_flags) & OCFS2_CHAIN_FL) { struct ocfs2_chain_list *cl = &di->id2.i_chain; u16 bpc = 1 << (OCFS2_SB(sb)->s_clustersize_bits - @@ -1559,6 +1696,38 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + if (ocfs2_dinode_has_extents(di)) { + struct ocfs2_extent_list *el = &di->id2.i_list; + u16 count = le16_to_cpu(el->l_count); + u16 next_free = le16_to_cpu(el->l_next_free_rec); + + if (count == 0) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_count is zero\n", + (unsigned long long)bh->b_blocknr); + goto bail; + } + /* + * The exact capacity depends on i_xattr_inline_size, another + * unvalidated on-disk field. Inline xattrs only shrink the + * list, so the no-xattr maximum is a safe upper bound that a + * valid l_count never exceeds. + */ + if (count > ocfs2_extent_recs_per_inode(sb)) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_count %u exceeds max %u\n", + (unsigned long long)bh->b_blocknr, count, + ocfs2_extent_recs_per_inode(sb)); + goto bail; + } + if (next_free > count) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_next_free_rec %u exceeds l_count %u\n", + (unsigned long long)bh->b_blocknr, next_free, count); + goto bail; + } + } + rc = 0; bail: @@ -1624,6 +1793,40 @@ static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, (unsigned long long)bh->b_blocknr, le32_to_cpu(di->i_fs_generation)); rc = -OCFS2_FILECHECK_ERR_GENERATION; + goto bail; + } + + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_unexpected_rdev(di)) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; } bail: @@ -1812,4 +2015,3 @@ const struct ocfs2_caching_operations ocfs2_inode_caching_ops = { .co_io_lock = ocfs2_inode_cache_io_lock, .co_io_unlock = ocfs2_inode_cache_io_unlock, }; - diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c index f9bf3bac085db..d8afbc1a76bb8 100644 --- a/fs/ocfs2/journal.c +++ b/fs/ocfs2/journal.c @@ -473,8 +473,12 @@ int ocfs2_extend_trans(handle_t *handle, int nblocks) */ int ocfs2_assure_trans_credits(handle_t *handle, int nblocks) { - int old_nblks = jbd2_handle_buffer_credits(handle); + int old_nblks; + if (is_handle_aborted(handle)) + return -EROFS; + + old_nblks = jbd2_handle_buffer_credits(handle); trace_ocfs2_assure_trans_credits(old_nblks); if (old_nblks >= nblocks) return 0; @@ -1022,11 +1026,8 @@ static int ocfs2_journal_toggle_dirty(struct ocfs2_super *osb, struct ocfs2_dinode *fe; fe = (struct ocfs2_dinode *)bh->b_data; - - /* The journal bh on the osb always comes from ocfs2_journal_init() - * and was validated there inside ocfs2_inode_lock_full(). It's a - * code bug if we mess it up. */ - BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + if (WARN_ON(!OCFS2_IS_VALID_DINODE(fe))) + return -EIO; flags = le32_to_cpu(fe->id1.journal1.ij_flags); if (dirty) diff --git a/fs/ocfs2/journal.h b/fs/ocfs2/journal.h index 6397170f302f2..f8b3b2a3d6309 100644 --- a/fs/ocfs2/journal.h +++ b/fs/ocfs2/journal.h @@ -196,6 +196,9 @@ static inline void ocfs2_checkpoint_inode(struct inode *inode) if (ocfs2_mount_local(osb)) return; + if (!osb->journal) + return; + if (!ocfs2_ci_fully_checkpointed(INODE_CACHE(inode))) { /* WARNING: This only kicks off a single * checkpoint. If someone races you and adds more diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c index c53de4439d933..ad1678ee7cc46 100644 --- a/fs/ocfs2/move_extents.c +++ b/fs/ocfs2/move_extents.c @@ -534,6 +534,8 @@ static void ocfs2_probe_alloc_group(struct inode *inode, struct buffer_head *bh, u32 base_cpos = ocfs2_blocks_to_clusters(inode->i_sb, le64_to_cpu(gd->bg_blkno)); + *phys_cpos = 0; + for (i = base_bit; i < le16_to_cpu(gd->bg_bits); i++) { used = ocfs2_test_bit(i, (unsigned long *)gd->bg_bitmap); @@ -555,7 +557,7 @@ static void ocfs2_probe_alloc_group(struct inode *inode, struct buffer_head *bh, last_free_bits++; if (last_free_bits == move_len) { - i -= move_len; + i = i - move_len + 1; *goal_bit = i; *phys_cpos = base_cpos + i; break; diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h index 7b50e03dfa664..62cad6522c7a3 100644 --- a/fs/ocfs2/ocfs2.h +++ b/fs/ocfs2/ocfs2.h @@ -494,8 +494,6 @@ struct ocfs2_super struct rb_root osb_rf_lock_tree; struct ocfs2_refcount_tree *osb_ref_tree_lru; - struct mutex system_file_mutex; - /* * OCFS2 needs to schedule several different types of work which * require cluster locking, disk I/O, recovery waits, etc. Since these diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c index 12cbb4fccda0d..f55810c59b1b1 100644 --- a/fs/ocfs2/quota_local.c +++ b/fs/ocfs2/quota_local.c @@ -302,7 +302,7 @@ static int ocfs2_add_recovery_chunk(struct super_block *sb, if (!rc) return -ENOMEM; rc->rc_chunk = chunk; - rc->rc_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); + rc->rc_bitmap = kzalloc(sb->s_blocksize, GFP_NOFS); if (!rc->rc_bitmap) { kfree(rc); return -ENOMEM; diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 8eee5be4d1ed4..7323bde70caaf 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -2131,10 +2131,15 @@ static int ocfs2_remove_refcount_extent(handle_t *handle, rb->rf_flags = 0; rb->rf_parent = 0; rb->rf_cpos = 0; - memset(&rb->rf_records, 0, sb->s_blocksize - - offsetof(struct ocfs2_refcount_block, rf_records)); + rb->rf_records.rl_used = 0; + rb->rf_records.rl_reserved2 = 0; + rb->rf_records.rl_reserved1 = 0; + /* rl_count determines the memset size and fortify object size. */ rb->rf_records.rl_count = cpu_to_le16(ocfs2_refcount_recs_per_rb(sb)); + memset(rb->rf_records.rl_recs, 0, + le16_to_cpu(rb->rf_records.rl_count) * + sizeof(*rb->rf_records.rl_recs)); } ocfs2_journal_dirty(handle, ref_root_bh); diff --git a/fs/ocfs2/stack_user.c b/fs/ocfs2/stack_user.c index 5803f1dee6790..91e19d33847c1 100644 --- a/fs/ocfs2/stack_user.c +++ b/fs/ocfs2/stack_user.c @@ -327,18 +327,14 @@ static int ocfs2_control_install_private(struct file *file) ocfs2_control_this_node = p->op_this_node; running_proto.pv_major = p->op_proto.pv_major; running_proto.pv_minor = p->op_proto.pv_minor; - } - -out_unlock: - mutex_unlock(&ocfs2_control_lock); - - if (!rc && set_p) { - /* We set the global values successfully */ atomic_inc(&ocfs2_control_opened); ocfs2_control_set_handshake_state(file, OCFS2_CONTROL_HANDSHAKE_VALID); } +out_unlock: + mutex_unlock(&ocfs2_control_lock); + return rc; } diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index d284e0e372525..a4a2b87a45fe3 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -231,8 +231,16 @@ static int ocfs2_validate_gd_parent(struct super_block *sb, int resize) { unsigned int max_bits; + unsigned int max_bitmap_bits; + unsigned int max_bitmap_size; + int suballocator; struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data; + suballocator = le64_to_cpu(di->i_blkno) != OCFS2_SB(sb)->bitmap_blkno; + max_bitmap_size = ocfs2_group_bitmap_size(sb, suballocator, + OCFS2_SB(sb)->s_feature_incompat); + max_bitmap_bits = max_bitmap_size * 8; + if (di->i_blkno != gd->bg_parent_dinode) { do_error("Group descriptor #%llu has bad parent pointer (%llu, expected %llu)\n", (unsigned long long)bh->b_blocknr, @@ -240,6 +248,20 @@ static int ocfs2_validate_gd_parent(struct super_block *sb, (unsigned long long)le64_to_cpu(di->i_blkno)); } + if (le16_to_cpu(gd->bg_size) > max_bitmap_size) { + do_error("Group descriptor #%llu has bitmap size %u but physical max of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_size), + max_bitmap_size); + } + + if (le16_to_cpu(gd->bg_bits) > max_bitmap_bits) { + do_error("Group descriptor #%llu has bit count %u but physical max of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_bits), + max_bitmap_bits); + } + max_bits = le16_to_cpu(di->id2.i_chain.cl_cpg) * le16_to_cpu(di->id2.i_chain.cl_bpc); if (le16_to_cpu(gd->bg_bits) > max_bits) { do_error("Group descriptor #%llu has bit count of %u\n", diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index b875f01c97564..6dd45c2153f88 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1997,8 +1997,6 @@ static int ocfs2_initialize_super(struct super_block *sb, spin_lock_init(&osb->osb_xattr_lock); ocfs2_init_steal_slots(osb); - mutex_init(&osb->system_file_mutex); - atomic_set(&osb->alloc_stats.moves, 0); atomic_set(&osb->alloc_stats.local_data, 0); atomic_set(&osb->alloc_stats.bitmap_data, 0); diff --git a/fs/ocfs2/sysfile.c b/fs/ocfs2/sysfile.c index d53a6cc866bef..67e492f4b828b 100644 --- a/fs/ocfs2/sysfile.c +++ b/fs/ocfs2/sysfile.c @@ -98,11 +98,9 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb, } else arr = get_local_system_inode(osb, type, slot); - mutex_lock(&osb->system_file_mutex); if (arr && ((inode = *arr) != NULL)) { /* get a ref in addition to the array ref */ inode = igrab(inode); - mutex_unlock(&osb->system_file_mutex); BUG_ON(!inode); return inode; @@ -112,11 +110,10 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb, inode = _ocfs2_get_system_file_inode(osb, type, slot); /* add one more if putting into array for first time */ - if (arr && inode) { - *arr = igrab(inode); - BUG_ON(!*arr); + if (inode && arr && !*arr && !cmpxchg(&(*arr), NULL, inode)) { + inode = igrab(inode); + BUG_ON(!inode); } - mutex_unlock(&osb->system_file_mutex); return inode; } diff --git a/fs/orangefs/dir.c b/fs/orangefs/dir.c index 6e2ebc8b9867b..115b2c2f52694 100644 --- a/fs/orangefs/dir.c +++ b/fs/orangefs/dir.c @@ -191,7 +191,8 @@ static int fill_from_part(struct orangefs_dir_part *part, { const int offset = sizeof(struct orangefs_readdir_response_s); struct orangefs_khandle *khandle; - __u32 *len, padlen; + __u32 *len; + u64 padlen; loff_t i; char *s; i = ctx->pos & ~PART_MASK; @@ -215,8 +216,8 @@ static int fill_from_part(struct orangefs_dir_part *part, * len is the size of the string itself. padlen is the * total size of the encoded string. */ - padlen = (sizeof *len + *len + 1) + - (8 - (sizeof *len + *len + 1)%8)%8; + padlen = (u64)sizeof *len + *len + 1; + padlen += (8 - padlen % 8) % 8; if (part->len < i + padlen + sizeof *khandle) goto next; s = (void *)part + offset + i + sizeof *len; diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 13cb60b52bd6e..e963701b4c877 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -853,7 +853,7 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) { struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); struct inode *udir = d_inode(c->destdir); - struct dentry *temp, *upper; + struct dentry *temp, *upper, *newdentry = NULL; struct file *tmpfile; int err; @@ -889,6 +889,14 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) err = PTR_ERR(upper); if (!IS_ERR(upper)) { err = ovl_do_link(ofs, temp, udir, upper); + if (!err) { + /* + * Record the linked dentry -- not the disconnected + * O_TMPFILE dentry -- so that ->d_revalidate() on + * the upper fs sees the real parent/name. + */ + newdentry = dget(upper); + } end_creating(upper); } @@ -903,7 +911,7 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) if (!c->metacopy) ovl_set_upperdata(d_inode(c->dentry)); - ovl_inode_update(d_inode(c->dentry), dget(temp)); + ovl_inode_update(d_inode(c->dentry), newdentry); out: ovl_end_write(c->dentry); diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 00c69707bda98..bc71231cad532 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -783,8 +783,8 @@ static const struct address_space_operations ovl_aops = { * * This chain is valid: * - inode->i_rwsem (inode_lock[2]) - * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) * - OVL_I(inode)->lock (ovl_inode_lock[2]) + * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) * - OVL_I(lowerinode)->lock (ovl_inode_lock[1]) * * And this chain is valid: @@ -797,8 +797,8 @@ static const struct address_space_operations ovl_aops = { * held, because it is in reverse order of the non-nested case using the same * upper fs: * - inode->i_rwsem (inode_lock[1]) - * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) * - OVL_I(inode)->lock (ovl_inode_lock[1]) + * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) */ #define OVL_MAX_NESTING FILESYSTEM_MAX_STACK_DEPTH diff --git a/fs/pidfs.c b/fs/pidfs.c index 1cce4f34a0512..143d0aec16af1 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -37,6 +37,8 @@ static struct kmem_cache *pidfs_attr_cachep __ro_after_init; static struct path pidfs_root_path = {}; +static struct simple_xattr_cache pidfs_xa_cache; + void pidfs_get_root(struct path *path) { *path = pidfs_root_path; @@ -96,7 +98,7 @@ static const struct rhashtable_params pidfs_ino_ht_params = { * use file handles. */ struct pidfs_attr { - struct simple_xattrs *xattrs; + struct list_head xattrs; union { struct pidfs_anon_attr; struct llist_node pidfs_llist; @@ -196,12 +198,7 @@ static void pidfs_free_attr_work(struct work_struct *work) head = llist_del_all(&pidfs_free_list); llist_for_each_entry_safe(attr, next, head, pidfs_llist) { - struct simple_xattrs *xattrs = attr->xattrs; - - if (xattrs) { - simple_xattrs_free(xattrs, NULL); - kfree(xattrs); - } + simple_xattrs_free(&pidfs_xa_cache, &attr->xattrs, NULL); kfree(attr); } } @@ -229,7 +226,7 @@ void pidfs_free_pid(struct pid *pid) if (IS_ERR(attr)) return; - if (likely(!attr->xattrs)) + if (likely(list_empty(&attr->xattrs))) kfree(attr); else if (llist_add(&attr->pidfs_llist, &pidfs_free_list)) schedule_work(&pidfs_free_work); @@ -815,14 +812,8 @@ static ssize_t pidfs_listxattr(struct dentry *dentry, char *buf, size_t size) { struct inode *inode = d_inode(dentry); struct pid *pid = inode->i_private; - struct pidfs_attr *attr = pid->attr; - struct simple_xattrs *xattrs; - - xattrs = READ_ONCE(attr->xattrs); - if (!xattrs) - return 0; - return simple_xattr_list(inode, xattrs, buf, size); + return simple_xattr_list(inode, &pid->attr->xattrs, buf, size); } static const struct inode_operations pidfs_inode_operations = { @@ -1018,6 +1009,8 @@ int pidfs_register_pid(struct pid *pid) if (!new_attr) return -ENOMEM; + INIT_LIST_HEAD_RCU(&new_attr->xattrs); + /* Synchronize with pidfs_exit(). */ guard(spinlock_irq)(&pid->wait_pidfd.lock); @@ -1057,16 +1050,9 @@ static int pidfs_xattr_get(const struct xattr_handler *handler, const char *suffix, void *value, size_t size) { struct pid *pid = inode->i_private; - struct pidfs_attr *attr = pid->attr; - const char *name; - struct simple_xattrs *xattrs; - - xattrs = READ_ONCE(attr->xattrs); - if (!xattrs) - return -ENODATA; + const char *name = xattr_full_name(handler, suffix); - name = xattr_full_name(handler, suffix); - return simple_xattr_get(xattrs, name, value, size); + return simple_xattr_get(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size); } static int pidfs_xattr_set(const struct xattr_handler *handler, @@ -1075,20 +1061,13 @@ static int pidfs_xattr_set(const struct xattr_handler *handler, const void *value, size_t size, int flags) { struct pid *pid = inode->i_private; - struct pidfs_attr *attr = pid->attr; - const char *name; - struct simple_xattrs *xattrs; + const char *name = xattr_full_name(handler, suffix); struct simple_xattr *old_xattr; /* Ensure we're the only one to set @attr->xattrs. */ WARN_ON_ONCE(!inode_is_locked(inode)); - xattrs = simple_xattrs_lazy_alloc(&attr->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); - - name = xattr_full_name(handler, suffix); - old_xattr = simple_xattr_set(xattrs, name, value, size, flags); + old_xattr = simple_xattr_set(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size, flags); if (IS_ERR(old_xattr)) return PTR_ERR(old_xattr); diff --git a/fs/proc/array.c b/fs/proc/array.c index 90fb0c6b5f99f..479ea8cb4ef43 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -482,6 +482,11 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, unsigned long flags; int exit_code = task->exit_code; struct signal_struct *sig = task->signal; + int ret; + + ret = down_read_killable(&task->signal->exec_update_lock); + if (ret) + return ret; state = *get_task_state(task); vsize = eip = esp = 0; @@ -657,6 +662,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, seq_puts(m, " 0"); seq_putc(m, '\n'); + up_read(&task->signal->exec_update_lock); if (mm) mmput(mm); return 0; diff --git a/fs/proc/base.c b/fs/proc/base.c index d9acfa89c894b..5042f14d24f36 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -218,33 +218,24 @@ static int get_task_root(struct task_struct *task, struct path *root) return result; } -static int proc_cwd_link(struct dentry *dentry, struct path *path) +static int proc_cwd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task = get_proc_task(d_inode(dentry)); int result = -ENOENT; - if (task) { - task_lock(task); - if (task->fs) { - get_fs_pwd(task->fs, path); - result = 0; - } - task_unlock(task); - put_task_struct(task); + task_lock(task); + if (task->fs) { + get_fs_pwd(task->fs, path); + result = 0; } + task_unlock(task); return result; } -static int proc_root_link(struct dentry *dentry, struct path *path) +static int proc_root_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task = get_proc_task(d_inode(dentry)); - int result = -ENOENT; - - if (task) { - result = get_task_root(task, path); - put_task_struct(task); - } - return result; + return get_task_root(task, path); } /* @@ -423,18 +414,24 @@ static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns, { unsigned long wchan; char symname[KSYM_NAME_LEN]; + int err; + err = down_read_killable(&task->signal->exec_update_lock); + if (err) + return err; if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) goto print0; wchan = get_wchan(task); if (wchan && !lookup_symbol_name(wchan, symname)) { seq_puts(m, symname); + up_read(&task->signal->exec_update_lock); return 0; } print0: seq_putc(m, '0'); + up_read(&task->signal->exec_update_lock); return 0; } #endif /* CONFIG_KALLSYMS */ @@ -704,23 +701,6 @@ static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns, /* Here the fs part begins */ /************************************************************************/ -/* permission checks */ -static bool proc_fd_access_allowed(struct inode *inode) -{ - struct task_struct *task; - bool allowed = false; - /* Allow access to a task's file descriptors if it is us or we - * may use ptrace attach to the process and find out that - * information. - */ - task = get_proc_task(inode); - if (task) { - allowed = ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS); - put_task_struct(task); - } - return allowed; -} - int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -1777,16 +1757,12 @@ static const struct file_operations proc_pid_set_comm_operations = { .release = single_release, }; -static int proc_exe_link(struct dentry *dentry, struct path *exe_path) +static int proc_exe_link(struct dentry *dentry, struct path *exe_path, + struct task_struct *task) { - struct task_struct *task; struct file *exe_file; - task = get_proc_task(d_inode(dentry)); - if (!task) - return -ENOENT; exe_file = get_task_exe_file(task); - put_task_struct(task); if (exe_file) { *exe_path = exe_file->f_path; path_get(&exe_file->f_path); @@ -1796,26 +1772,42 @@ static int proc_exe_link(struct dentry *dentry, struct path *exe_path) return -ENOENT; } +static int call_proc_get_link(struct dentry *dentry, struct inode *inode, struct path *path_out) +{ + struct task_struct *task; + int ret; + + task = get_proc_task(inode); + if (!task) + return -ENOENT; + ret = down_read_killable(&task->signal->exec_update_lock); + if (ret) + goto out_put_task; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + ret = -EACCES; + goto out; + } + ret = PROC_I(inode)->op.proc_get_link(dentry, path_out, task); + +out: + up_read(&task->signal->exec_update_lock); +out_put_task: + put_task_struct(task); + return ret; +} + static const char *proc_pid_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) { struct path path; - int error = -EACCES; + int error; if (!dentry) return ERR_PTR(-ECHILD); - - /* Are we allowed to snoop on the tasks file descriptors? */ - if (!proc_fd_access_allowed(inode)) - goto out; - - error = PROC_I(inode)->op.proc_get_link(dentry, &path); - if (error) - goto out; - - error = nd_jump_link(&path); -out: + error = call_proc_get_link(dentry, inode, &path); + if (!error) + error = nd_jump_link(&path); return ERR_PTR(error); } @@ -1849,17 +1841,11 @@ static int proc_pid_readlink(struct dentry * dentry, char __user * buffer, int b struct inode *inode = d_inode(dentry); struct path path; - /* Are we allowed to snoop on the tasks file descriptors? */ - if (!proc_fd_access_allowed(inode)) - goto out; - - error = PROC_I(inode)->op.proc_get_link(dentry, &path); - if (error) - goto out; - - error = do_proc_readlink(&path, buffer, buflen); - path_put(&path); -out: + error = call_proc_get_link(dentry, inode, &path); + if (!error) { + error = do_proc_readlink(&path, buffer, buflen); + path_put(&path); + } return error; } @@ -2250,21 +2236,16 @@ static const struct dentry_operations tid_map_files_dentry_operations = { .d_delete = pid_delete_dentry, }; -static int map_files_get_link(struct dentry *dentry, struct path *path) +static int map_files_get_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { unsigned long vm_start, vm_end; struct vm_area_struct *vma; - struct task_struct *task; struct mm_struct *mm; int rc; rc = -ENOENT; - task = get_proc_task(d_inode(dentry)); - if (!task) - goto out; - mm = get_task_mm(task); - put_task_struct(task); if (!mm) goto out; @@ -2360,17 +2341,15 @@ static struct dentry *proc_map_files_lookup(struct inode *dir, if (!task) goto out; - result = ERR_PTR(-EACCES); - if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) - goto out_put_task; - result = ERR_PTR(-ENOENT); if (dname_to_vma_addr(dentry, &vm_start, &vm_end)) goto out_put_task; - mm = get_task_mm(task); - if (!mm) + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR(mm)) { + result = ERR_CAST(mm); goto out_put_task; + } result = ERR_PTR(-EINTR); if (mmap_read_lock_killable(mm)) @@ -2420,23 +2399,22 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) if (!task) goto out; - ret = -EACCES; - if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) - goto out_put_task; - ret = 0; if (!dir_emit_dots(file, ctx)) goto out_put_task; - mm = get_task_mm(task); - if (!mm) + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR(mm)) { + ret = PTR_ERR(mm); + /* if the task has no mm, the directory should just be empty */ + if (ret == -ESRCH) + ret = 0; goto out_put_task; + } ret = mmap_read_lock_killable(mm); - if (ret) { - mmput(mm); - goto out_put_task; - } + if (ret) + goto out_put_mm; nr_files = 0; @@ -2462,8 +2440,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) if (!p) { ret = -ENOMEM; mmap_read_unlock(mm); - mmput(mm); - goto out_put_task; + goto out_put_mm; } p->start = vma->vm_start; @@ -2471,7 +2448,6 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) p->mode = vma->vm_file->f_mode; } mmap_read_unlock(mm); - mmput(mm); for (i = 0; i < nr_files; i++) { char buf[4 * sizeof(long) + 2]; /* max: %lx-%lx\0 */ @@ -2488,6 +2464,8 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) ctx->pos++; } +out_put_mm: + mmput(mm); out_put_task: put_task_struct(task); out: diff --git a/fs/proc/fd.c b/fs/proc/fd.c index 05c7513e77c7e..0f9a1556f2a34 100644 --- a/fs/proc/fd.c +++ b/fs/proc/fd.c @@ -171,24 +171,19 @@ static const struct dentry_operations tid_fd_dentry_operations = { .d_delete = pid_delete_dentry, }; -static int proc_fd_link(struct dentry *dentry, struct path *path) +static int proc_fd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task; int ret = -ENOENT; - - task = get_proc_task(d_inode(dentry)); - if (task) { - unsigned int fd = proc_fd(d_inode(dentry)); - struct file *fd_file; - - fd_file = fget_task(task, fd); - if (fd_file) { - *path = fd_file->f_path; - path_get(&fd_file->f_path); - ret = 0; - fput(fd_file); - } - put_task_struct(task); + unsigned int fd = proc_fd(d_inode(dentry)); + struct file *fd_file; + + fd_file = fget_task(task, fd); + if (fd_file) { + *path = fd_file->f_path; + path_get(&fd_file->f_path); + ret = 0; + fput(fd_file); } return ret; diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 8bb81e58c9d8c..32b6b0f979679 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -112,6 +112,8 @@ static bool pde_subdir_insert(struct proc_dir_entry *dir, /* Add new node and rebalance tree. */ rb_link_node(&de->subdir_node, parent, new); rb_insert_color(&de->subdir_node, root); + if (S_ISDIR(de->mode)) + dir->nlink++; return true; } @@ -404,7 +406,6 @@ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, write_unlock(&proc_subdir_lock); goto out_free_inum; } - dir->nlink++; write_unlock(&proc_subdir_lock); return dp; @@ -702,6 +703,8 @@ static void pde_erase(struct proc_dir_entry *pde, struct proc_dir_entry *parent) { rb_erase(&pde->subdir_node, &parent->subdir); RB_CLEAR_NODE(&pde->subdir_node); + if (S_ISDIR(pde->mode)) + parent->nlink--; } /* @@ -727,8 +730,6 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent) de = NULL; } else { pde_erase(de, parent); - if (S_ISDIR(de->mode)) - parent->nlink--; } } write_unlock(&proc_subdir_lock); @@ -787,8 +788,6 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent) continue; } next = de->parent; - if (S_ISDIR(de->mode)) - next->nlink--; write_unlock(&proc_subdir_lock); proc_entry_rundown(de); diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 64dc44832808a..d31984c3c7979 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -107,7 +107,7 @@ extern struct kmem_cache *proc_dir_entry_cache; void pde_free(struct proc_dir_entry *pde); union proc_op { - int (*proc_get_link)(struct dentry *, struct path *); + int (*proc_get_link)(struct dentry *, struct path *, struct task_struct *); int (*proc_show)(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task); diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c index 39f4169f669f6..2f46f13967445 100644 --- a/fs/proc/namespaces.c +++ b/fs/proc/namespaces.c @@ -55,6 +55,10 @@ static const char *proc_ns_get_link(struct dentry *dentry, if (!task) return ERR_PTR(-EACCES); + error = down_read_killable(&task->signal->exec_update_lock); + if (error) + goto out_put_task; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) goto out; @@ -64,6 +68,8 @@ static const char *proc_ns_get_link(struct dentry *dentry, error = nd_jump_link(&ns_path); out: + up_read(&task->signal->exec_update_lock); +out_put_task: put_task_struct(task); return ERR_PTR(error); } @@ -80,11 +86,17 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl if (!task) return res; + res = down_read_killable(&task->signal->exec_update_lock); + if (res) + goto out_put_task; + if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { res = ns_get_name(name, sizeof(name), task, ns_ops); if (res >= 0) res = readlink_copy(buffer, buflen, name, strlen(name)); } + up_read(&task->signal->exec_update_lock); +out_put_task: put_task_struct(task); return res; } diff --git a/fs/proc/page.c b/fs/proc/page.c index f9b2c2c906cd7..7d93871434357 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -173,7 +173,7 @@ u64 stable_page_flags(const struct page *page) u |= 1 << KPF_MMAP; if (is_anon) { u |= 1 << KPF_ANON; - if (mapping & FOLIO_MAPPING_KSM) + if ((mapping & FOLIO_MAPPING_FLAGS) == FOLIO_MAPPING_KSM) u |= 1 << KPF_KSM; } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 751b9ba160fbf..29f4bdf6e4ab9 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2042,7 +2042,6 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_SOFT_DIRTY; if (pmd_swp_uffd_wp(pmd)) flags |= PM_UFFD_WP; - VM_WARN_ON_ONCE(!pmd_is_migration_entry(pmd)); page = softleaf_to_page(entry); } @@ -2523,12 +2522,16 @@ static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry)) return; - if (softleaf_is_migration(entry)) + if (softleaf_is_migration(entry)) { set_huge_pte_at(vma->vm_mm, addr, ptep, pte_swp_mkuffd_wp(ptent), psize); - else - huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, - huge_pte_mkuffd_wp(ptent)); + } else { + pte_t old_pte, new_pte; + + old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); + new_pte = huge_pte_mkuffd_wp(old_pte); + huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); + } } #endif /* CONFIG_HUGETLB_PAGE */ @@ -2869,7 +2872,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, if (~categories & PAGE_IS_WRITTEN) goto out_unlock; - if (end != start + HPAGE_SIZE) { + if (end != start + huge_page_size(hstate_vma(vma))) { /* Partial HugeTLB page WP isn't possible. */ pagemap_scan_backout_range(p, start, end); p->arg.walk_end = start; @@ -2886,8 +2889,62 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, return ret; } + +/* + * Write-protect the unpopulated hugetlb entries covering [addr, end) by + * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() + * does for populated entries. + * + * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the + * loop normally runs a single iteration; it is written to cover the full range + * in case the walker ever coalesces adjacent holes. + * + * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- + * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table + * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the + * same VMA, so the scanning thread would deadlock against itself. PMD sharing + * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so + * the vma lock guards nothing that matters for these entries anyway. + */ +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + struct hstate *h = hstate_vma(vma); + unsigned long psize = huge_page_size(h); + struct mm_struct *mm = vma->vm_mm; + spinlock_t *ptl; + pte_t *ptep; + pte_t pte; + + for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { + ptep = huge_pte_alloc(mm, vma, addr, psize); + if (!ptep) + return -ENOMEM; + + i_mmap_lock_write(vma->vm_file->f_mapping); + ptl = huge_pte_lock(h, mm, ptep); + pte = huge_ptep_get(mm, addr, ptep); + make_uffd_wp_huge_pte(vma, addr, ptep, pte); + /* + * A none entry has no cached translation, so installing the + * marker needs no TLB flush. Flush only if a fault populated + * the entry between huge_pte_alloc() and the page table lock. + */ + if (!huge_pte_none(pte)) + flush_hugetlb_tlb_range(vma, addr, addr + psize); + spin_unlock(ptl); + i_mmap_unlock_write(vma->vm_file->f_mapping); + } + + return 0; +} #else #define pagemap_scan_hugetlb_entry NULL +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + return 0; +} #endif static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, @@ -2907,7 +2964,10 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, if (~p->arg.flags & PM_SCAN_WP_MATCHING) return ret; - err = uffd_wp_range(vma, addr, end - addr, true); + if (is_vm_hugetlb_page(vma)) + err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); + else + err = uffd_wp_range(vma, addr, end - addr, true); if (err < 0) ret = err; diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 9fd901c78dc66..3d869fc35e837 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -135,10 +135,10 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); struct rmid_entry *entry; + bool rmid_dirty = true; u32 idx, cur_idx = 1; void *arch_mon_ctx; void *arch_priv; - bool rmid_dirty; u64 val = 0; arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv; @@ -161,22 +161,27 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) break; entry = __rmid_entry(idx); - if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, entry->rmid, - QOS_L3_OCCUP_EVENT_ID, arch_priv, &val, - arch_mon_ctx)) { - rmid_dirty = true; - } else { - rmid_dirty = (val >= resctrl_rmid_realloc_threshold); - - /* - * x86's CLOSID and RMID are independent numbers, so the entry's - * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the - * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't - * used to select the configuration. It is thus necessary to track both - * CLOSID and RMID because there may be dependencies between them - * on some architectures. - */ - trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val); + if (!force_free) { + if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, + entry->rmid, QOS_L3_OCCUP_EVENT_ID, + arch_priv, &val, arch_mon_ctx)) { + rmid_dirty = true; + } else { + rmid_dirty = (val >= resctrl_rmid_realloc_threshold); + + /* + * x86's CLOSID and RMID are independent numbers, + * so the entry's CLOSID is an empty CLOSID + * (X86_RESCTRL_EMPTY_CLOSID). On Arm the RMID + * (PMG) extends the CLOSID (PARTID) space with + * bits that aren't used to select the configuration. + * It is thus necessary to track both CLOSID and + * RMID because there may be dependencies between + * them on some architectures. + */ + trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, + d->hdr.id, val); + } } if (force_free || !rmid_dirty) { diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 5dfdaa6f9d8ff..e7c5a4e0590df 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -74,6 +74,8 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx); static void rdtgroup_destroy_root(void); +static void mon_put_kn_priv(void); + struct dentry *debugfs_resctrl; /* @@ -585,14 +587,20 @@ static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, * * On resource group creation via a mkdir, an extra kernfs_node reference is * taken to ensure that the rdtgroup structure remains accessible for the - * rdtgroup_kn_unlock() calls where it is removed. + * rdtgroup_kn_unlock() calls where it is removed. The default group is + * statically allocated: it does not have an extra reference but will have + * RDT_DELETED set on unmount to support safe access to its associated files + * via rdtgroup_kn_lock_live/rdtgroup_kn_unlock(). * - * Drop the extra reference here, then free the rdtgroup structure. + * For all but the default group: drop the extra reference, then free the + * rdtgroup structure. * * Return: void */ static void rdtgroup_remove(struct rdtgroup *rdtgrp) { + if (rdtgrp == &rdtgroup_default) + return; kernfs_put(rdtgrp->kn); kfree(rdtgrp); } @@ -2802,6 +2810,12 @@ static int rdt_get_tree(struct fs_context *fc) goto out; } + /* Avoid races from pending operations from a previous mount */ + if (atomic_read(&rdtgroup_default.waitcount) != 0) { + ret = -EBUSY; + goto out; + } + ret = setup_rmid_lru_list(); if (ret) goto out; @@ -2883,6 +2897,7 @@ static int rdt_get_tree(struct fs_context *fc) kernfs_remove(kn_mondata); out_mongrp: if (resctrl_arch_mon_capable()) { + mon_put_kn_priv(); rdtgroup_unassign_cntrs(&rdtgroup_default); kernfs_remove(kn_mongrp); } @@ -3059,10 +3074,6 @@ static void rmdir_all_sub(void) if (rdtgrp == &rdtgroup_default) continue; - if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || - rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) - rdtgroup_pseudo_lock_remove(rdtgrp); - /* * Give any CPUs back to the default group. We cannot copy * cpu_online_mask because a CPU might have executed the @@ -3073,7 +3084,13 @@ static void rmdir_all_sub(void) rdtgroup_unassign_cntrs(rdtgrp); - free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + rdtgroup_pseudo_lock_remove(rdtgrp); + } else { + /* Pseudo-locked group's RMID is freed during setup. */ + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + } kernfs_remove(rdtgrp->kn); list_del(&rdtgrp->rdtgroup_list); @@ -3164,6 +3181,7 @@ static void resctrl_fs_teardown(void) mon_put_kn_priv(); rdt_pseudo_lock_release(); rdtgroup_default.mode = RDT_MODE_SHAREABLE; + rdtgroup_default.flags = RDT_DELETED; closid_exit(); schemata_list_destroy(); rdtgroup_destroy_root(); @@ -4264,6 +4282,7 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx) ctx->kfc.root = rdt_root; rdtgroup_default.kn = kernfs_root_to_node(rdt_root); + rdtgroup_default.flags = 0; return 0; } diff --git a/fs/smb/client/cifs_fs_sb.h b/fs/smb/client/cifs_fs_sb.h index 84e7e366b0ff4..d6494e1d93ccf 100644 --- a/fs/smb/client/cifs_fs_sb.h +++ b/fs/smb/client/cifs_fs_sb.h @@ -56,6 +56,7 @@ struct cifs_sb_info { struct smb3_fs_context *ctx; atomic_t active; atomic_t mnt_cifs_flags; + atomic_t outstanding_rreq; /* nr of rreqs not yet fully deinitialized */ struct delayed_work prune_tlinks; struct rcu_head rcu; diff --git a/fs/smb/client/cifs_swn.c b/fs/smb/client/cifs_swn.c index 9753a432d0998..9951817d0d7ff 100644 --- a/fs/smb/client/cifs_swn.c +++ b/fs/smb/client/cifs_swn.c @@ -28,10 +28,54 @@ struct cifs_swn_reg { bool net_name_notify; bool share_name_notify; bool ip_notify; +}; - struct cifs_tcon *tcon; +struct cifs_swn_reg_info { + int id; + unsigned int ref_count; + const char *net_name; + const char *share_name; + bool net_name_notify; + bool share_name_notify; + bool ip_notify; }; +static void cifs_swn_snapshot_reg(struct cifs_swn_reg *swnreg, + struct cifs_swn_reg_info *info) +{ + info->id = swnreg->id; + info->ref_count = kref_read(&swnreg->ref_count); + info->net_name = swnreg->net_name; + info->share_name = swnreg->share_name; + info->net_name_notify = swnreg->net_name_notify; + info->share_name_notify = swnreg->share_name_notify; + info->ip_notify = swnreg->ip_notify; +} + +static int cifs_swn_dup_reg(struct cifs_swn_reg *swnreg, + struct cifs_swn_reg_info *info) +{ + cifs_swn_snapshot_reg(swnreg, info); + + info->net_name = kstrdup(swnreg->net_name, GFP_KERNEL); + if (!info->net_name) + return -ENOMEM; + + info->share_name = kstrdup(swnreg->share_name, GFP_KERNEL); + if (!info->share_name) { + kfree(info->net_name); + return -ENOMEM; + } + + return 0; +} + +static void cifs_swn_free_reg_info(struct cifs_swn_reg_info *info) +{ + kfree(info->net_name); + kfree(info->share_name); +} + static int cifs_swn_auth_info_krb(struct cifs_tcon *tcon, struct sk_buff *skb) { int ret; @@ -73,7 +117,8 @@ static int cifs_swn_auth_info_ntlm(struct cifs_tcon *tcon, struct sk_buff *skb) * The authentication information to connect to the witness service is bundled * into the message. */ -static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) +static int cifs_swn_send_register_message(struct cifs_swn_reg_info *swnreg, + struct cifs_tcon *tcon) { struct sk_buff *skb; struct genlmsghdr *hdr; @@ -109,10 +154,10 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) * told to switch to it (client move message). In these cases we unregister from the * server address and register to the new address when we receive the notification. */ - if (swnreg->tcon->ses->server->use_swn_dstaddr) - addr = &swnreg->tcon->ses->server->swn_dstaddr; + if (tcon->ses->server->use_swn_dstaddr) + addr = &tcon->ses->server->swn_dstaddr; else - addr = &swnreg->tcon->ses->server->dstaddr; + addr = &tcon->ses->server->dstaddr; ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage), addr); if (ret < 0) @@ -136,10 +181,10 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) goto nlmsg_fail; } - authtype = cifs_select_sectype(swnreg->tcon->ses->server, swnreg->tcon->ses->sectype); + authtype = cifs_select_sectype(tcon->ses->server, tcon->ses->sectype); switch (authtype) { case Kerberos: - ret = cifs_swn_auth_info_krb(swnreg->tcon, skb); + ret = cifs_swn_auth_info_krb(tcon, skb); if (ret < 0) { cifs_dbg(VFS, "%s: Failed to get kerberos auth info: %d\n", __func__, ret); goto nlmsg_fail; @@ -147,7 +192,7 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) break; case NTLMv2: case RawNTLMSSP: - ret = cifs_swn_auth_info_ntlm(swnreg->tcon, skb); + ret = cifs_swn_auth_info_ntlm(tcon, skb); if (ret < 0) { cifs_dbg(VFS, "%s: Failed to get NTLM auth info: %d\n", __func__, ret); goto nlmsg_fail; @@ -176,7 +221,8 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) /* * Sends an uregister message to the userspace daemon based on the registration */ -static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg) +static int cifs_swn_send_unregister_message(struct cifs_swn_reg_info *swnreg, + struct cifs_tcon *tcon) { struct sk_buff *skb; struct genlmsghdr *hdr; @@ -205,7 +251,7 @@ static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg) goto nlmsg_fail; ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage), - &swnreg->tcon->ses->server->dstaddr); + &tcon->ses->server->dstaddr); if (ret < 0) goto nlmsg_fail; @@ -241,6 +287,88 @@ static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg) return ret; } +/* + * Allocation-free mirror of extract_hostname() + extract_sharename() from + * fs/smb/client/unc.c. Those helpers kmalloc(GFP_KERNEL); this runs under + * cifs_tcp_ses_lock and tcon->tc_lock, both spinlocks, so we mirror their + * parsing in place against the caller's stable net_name/share_name strings. + * Keep in sync with unc.c. + */ +static bool cifs_swn_tcon_matches(struct cifs_tcon *tcon, + const char *net_name, + const char *share_name) +{ + const char *unc = tcon->tree_name; + const char *host, *share, *delim; + size_t host_len, share_len; + + if (!tcon->use_witness) + return false; + + /* extract_hostname: require strlen(unc) >= 3 */ + if (strnlen(unc, 3) < 3) + return false; + /* extract_hostname: skip all leading '\' characters */ + for (host = unc; *host == '\\'; host++) + ; + if (!*host) + return false; + delim = strchr(host, '\\'); + if (!delim) + return false; + host_len = delim - host; + if (strlen(net_name) != host_len || + strncasecmp(host, net_name, host_len)) + return false; + + /* extract_sharename: start at unc + 2, then first '\' onward */ + share = unc + 2; + delim = strchr(share, '\\'); + if (!delim) + return false; + share = delim + 1; + share_len = strlen(share); + + return strlen(share_name) == share_len && + !strncasecmp(share, share_name, share_len); +} + +/* + * One SWN registration id represents one net/share name pair. Multiple + * mounted tcons can therefore share the id. Pick a live representative at + * use time instead of caching the first tcon pointer in the registration. + */ +static struct cifs_tcon *cifs_swn_get_tcon(struct cifs_swn_reg_info *swnreg) +{ + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_EXITING || + !cifs_swn_tcon_matches(tcon, swnreg->net_name, + swnreg->share_name)) { + spin_unlock(&tcon->tc_lock); + continue; + } + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, + tcon->tc_count, + netfs_trace_tcon_ref_get_swn_notify); + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + return tcon; + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + return NULL; +} + /* * Try to find a matching registration for the tcon's server name and share name. * Calls to this function must be protected by cifs_swnreg_idr_mutex. @@ -347,8 +475,6 @@ static struct cifs_swn_reg *cifs_get_swn_reg(struct cifs_tcon *tcon) reg->net_name_notify = true; reg->share_name_notify = true; reg->ip_notify = (tcon->capabilities & SMB2_SHARE_CAP_SCALEOUT); - - reg->tcon = tcon; unlock: mutex_unlock(&cifs_swnreg_idr_mutex); @@ -368,11 +494,6 @@ static struct cifs_swn_reg *cifs_get_swn_reg(struct cifs_tcon *tcon) static void cifs_swn_reg_release(struct kref *ref) { struct cifs_swn_reg *swnreg = container_of(ref, struct cifs_swn_reg, ref_count); - int ret; - - ret = cifs_swn_send_unregister_message(swnreg); - if (ret < 0) - cifs_dbg(VFS, "%s: Failed to send unregister message: %d\n", __func__, ret); idr_remove(&cifs_swnreg_idr, swnreg->id); kfree(swnreg->net_name); @@ -380,23 +501,33 @@ static void cifs_swn_reg_release(struct kref *ref) kfree(swnreg); } -static void cifs_put_swn_reg(struct cifs_swn_reg *swnreg) +static void cifs_put_swn_reg_locked(struct cifs_swn_reg *swnreg, + struct cifs_tcon *tcon) { - mutex_lock(&cifs_swnreg_idr_mutex); + if (kref_read(&swnreg->ref_count) == 1) { + struct cifs_swn_reg_info swnreg_info; + int ret; + + cifs_swn_snapshot_reg(swnreg, &swnreg_info); + ret = cifs_swn_send_unregister_message(&swnreg_info, tcon); + if (ret < 0) + cifs_dbg(VFS, "%s: Failed to send unregister message: %d\n", + __func__, ret); + } + kref_put(&swnreg->ref_count, cifs_swn_reg_release); - mutex_unlock(&cifs_swnreg_idr_mutex); } -static int cifs_swn_resource_state_changed(struct cifs_swn_reg *swnreg, const char *name, int state) +static int cifs_swn_resource_state_changed(struct cifs_tcon *tcon, const char *name, int state) { switch (state) { case CIFS_SWN_RESOURCE_STATE_UNAVAILABLE: cifs_dbg(FYI, "%s: resource name '%s' become unavailable\n", __func__, name); - cifs_signal_cifsd_for_reconnect(swnreg->tcon->ses->server, true); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); break; case CIFS_SWN_RESOURCE_STATE_AVAILABLE: cifs_dbg(FYI, "%s: resource name '%s' become available\n", __func__, name); - cifs_signal_cifsd_for_reconnect(swnreg->tcon->ses->server, true); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); break; case CIFS_SWN_RESOURCE_STATE_UNKNOWN: cifs_dbg(FYI, "%s: resource name '%s' changed to unknown state\n", __func__, name); @@ -502,7 +633,7 @@ static int cifs_swn_reconnect(struct cifs_tcon *tcon, struct sockaddr_storage *a return ret; } -static int cifs_swn_client_move(struct cifs_swn_reg *swnreg, struct sockaddr_storage *addr) +static int cifs_swn_client_move(struct cifs_tcon *tcon, struct sockaddr_storage *addr) { struct sockaddr_in *ipv4 = (struct sockaddr_in *)addr; struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)addr; @@ -512,14 +643,17 @@ static int cifs_swn_client_move(struct cifs_swn_reg *swnreg, struct sockaddr_sto else if (addr->ss_family == AF_INET6) cifs_dbg(FYI, "%s: move to %pI6\n", __func__, &ipv6->sin6_addr); - return cifs_swn_reconnect(swnreg->tcon, addr); + return cifs_swn_reconnect(tcon, addr); } int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) { struct cifs_swn_reg *swnreg; + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; char name[256]; int type; + int ret = 0; if (info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]) { int swnreg_id; @@ -527,21 +661,34 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) swnreg_id = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]); mutex_lock(&cifs_swnreg_idr_mutex); swnreg = idr_find(&cifs_swnreg_idr, swnreg_id); - mutex_unlock(&cifs_swnreg_idr_mutex); if (swnreg == NULL) { + mutex_unlock(&cifs_swnreg_idr_mutex); cifs_dbg(FYI, "%s: registration id %d not found\n", __func__, swnreg_id); return -EINVAL; } + ret = cifs_swn_dup_reg(swnreg, &swnreg_info); + mutex_unlock(&cifs_swnreg_idr_mutex); + if (ret) + return ret; } else { cifs_dbg(FYI, "%s: missing registration id attribute\n", __func__); return -EINVAL; } + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + cifs_dbg(FYI, "%s: registration id %d has no live tcon\n", + __func__, swnreg_info.id); + ret = -ENODEV; + goto free_info; + } + if (info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]) { type = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]); } else { cifs_dbg(FYI, "%s: missing notification type attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } switch (type) { @@ -553,15 +700,18 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) sizeof(name)); } else { cifs_dbg(FYI, "%s: missing resource name attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } if (info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]) { state = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]); } else { cifs_dbg(FYI, "%s: missing resource state attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } - return cifs_swn_resource_state_changed(swnreg, name, state); + ret = cifs_swn_resource_state_changed(tcon, name, state); + break; } case CIFS_SWN_NOTIFICATION_CLIENT_MOVE: { struct sockaddr_storage addr; @@ -570,28 +720,36 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) nla_memcpy(&addr, info->attrs[CIFS_GENL_ATTR_SWN_IP], sizeof(addr)); } else { cifs_dbg(FYI, "%s: missing IP address attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } - return cifs_swn_client_move(swnreg, &addr); + ret = cifs_swn_client_move(tcon, &addr); + break; } default: cifs_dbg(FYI, "%s: unknown notification type %d\n", __func__, type); break; } - return 0; +out: + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +free_info: + cifs_swn_free_reg_info(&swnreg_info); + return ret; } int cifs_swn_register(struct cifs_tcon *tcon) { struct cifs_swn_reg *swnreg; + struct cifs_swn_reg_info swnreg_info; int ret; swnreg = cifs_get_swn_reg(tcon); if (IS_ERR(swnreg)) return PTR_ERR(swnreg); - ret = cifs_swn_send_register_message(swnreg); + cifs_swn_snapshot_reg(swnreg, &swnreg_info); + ret = cifs_swn_send_register_message(&swnreg_info, tcon); if (ret < 0) { cifs_dbg(VFS, "%s: Failed to send swn register message: %d\n", __func__, ret); /* Do not put the swnreg or return error, the echo task will retry */ @@ -612,35 +770,68 @@ int cifs_swn_unregister(struct cifs_tcon *tcon) return PTR_ERR(swnreg); } + cifs_put_swn_reg_locked(swnreg, tcon); mutex_unlock(&cifs_swnreg_idr_mutex); - cifs_put_swn_reg(swnreg); - return 0; } -void cifs_swn_dump(struct seq_file *m) +/* + * Snapshot one registration under cifs_swnreg_idr_mutex and return. Callers + * intentionally do the per-registration network/genlmsg work without the + * mutex held, both to keep the critical section short and to avoid nesting + * cifs_swnreg_idr_mutex inside the higher tc_lock when a live tcon is then + * pinned for the send. + */ +static int cifs_swn_get_next_reg_info(int *id, struct cifs_swn_reg_info *info) { struct cifs_swn_reg *swnreg; + int ret = 0; + + mutex_lock(&cifs_swnreg_idr_mutex); + swnreg = idr_get_next(&cifs_swnreg_idr, id); + if (swnreg) { + ret = cifs_swn_dup_reg(swnreg, info); + if (!ret) { + *id = swnreg->id + 1; + ret = 1; + } + } + mutex_unlock(&cifs_swnreg_idr_mutex); + + return ret; +} + +void cifs_swn_dump(struct seq_file *m) +{ + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; struct sockaddr_in *sa; struct sockaddr_in6 *sa6; - int id; + int id = 0; + int ret; seq_puts(m, "Witness registrations:"); - mutex_lock(&cifs_swnreg_idr_mutex); - idr_for_each_entry(&cifs_swnreg_idr, swnreg, id) { + while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) { seq_printf(m, "\nId: %u Refs: %u Network name: '%s'%s Share name: '%s'%s Ip address: ", - id, kref_read(&swnreg->ref_count), - swnreg->net_name, swnreg->net_name_notify ? "(y)" : "(n)", - swnreg->share_name, swnreg->share_name_notify ? "(y)" : "(n)"); - switch (swnreg->tcon->ses->server->dstaddr.ss_family) { + swnreg_info.id, swnreg_info.ref_count, + swnreg_info.net_name, swnreg_info.net_name_notify ? "(y)" : "(n)", + swnreg_info.share_name, swnreg_info.share_name_notify ? "(y)" : "(n)"); + + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + seq_puts(m, "(no live tcon)"); + goto next; + } + + switch (tcon->ses->server->dstaddr.ss_family) { case AF_INET: - sa = (struct sockaddr_in *) &swnreg->tcon->ses->server->dstaddr; + sa = (struct sockaddr_in *)&tcon->ses->server->dstaddr; seq_printf(m, "%pI4", &sa->sin_addr.s_addr); break; case AF_INET6: - sa6 = (struct sockaddr_in6 *) &swnreg->tcon->ses->server->dstaddr; + sa6 = (struct sockaddr_in6 *)&tcon->ses->server->dstaddr; seq_printf(m, "%pI6", &sa6->sin6_addr.s6_addr); if (sa6->sin6_scope_id) seq_printf(m, "%%%u", sa6->sin6_scope_id); @@ -648,23 +839,38 @@ void cifs_swn_dump(struct seq_file *m) default: seq_puts(m, "(unknown)"); } - seq_printf(m, "%s", swnreg->ip_notify ? "(y)" : "(n)"); + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +next: + seq_printf(m, "%s", swnreg_info.ip_notify ? "(y)" : "(n)"); + cifs_swn_free_reg_info(&swnreg_info); } - mutex_unlock(&cifs_swnreg_idr_mutex); + if (ret < 0) + seq_printf(m, "\nFailed to snapshot witness registration: %d", ret); seq_puts(m, "\n"); } void cifs_swn_check(void) { - struct cifs_swn_reg *swnreg; - int id; + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; + int id = 0; int ret; - mutex_lock(&cifs_swnreg_idr_mutex); - idr_for_each_entry(&cifs_swnreg_idr, swnreg, id) { - ret = cifs_swn_send_register_message(swnreg); + while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) { + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + cifs_dbg(FYI, "%s: registration id %d has no live tcon\n", + __func__, swnreg_info.id); + goto free_info; + } + + ret = cifs_swn_send_register_message(&swnreg_info, tcon); if (ret < 0) cifs_dbg(FYI, "%s: Failed to send register message: %d\n", __func__, ret); + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +free_info: + cifs_swn_free_reg_info(&swnreg_info); } - mutex_unlock(&cifs_swnreg_idr_mutex); + if (ret < 0) + cifs_dbg(FYI, "%s: Failed to snapshot registration: %d\n", __func__, ret); } diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index 786dbbc43c5b9..59a2d702fce15 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -889,7 +889,7 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, */ fattr->cf_mode &= ~07777; fattr->cf_mode |= - le32_to_cpu(ppace[i]->sid.sub_auth[2]); + le32_to_cpu(ppace[i]->sid.sub_auth[2]) & 07777; break; } else { if (compare_sids(&(ppace[i]->sid), pownersid) == 0) { diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index ce23924f01b3a..d6c30f8552e08 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -309,6 +309,18 @@ static void cifs_kill_sb(struct super_block *sb) /* Wait for all opened files to release */ flush_workqueue(deferredclose_wq); + /* + * Wait for all in-flight netfs I/O requests to finish their + * cleanup_work so that any cifsFileInfo final puts they queue + * to fileinfo_put_wq/serverclose_wq have been queued, then + * drain the workqueue so the cfile dentry refs are dropped to + * avoid the busy dentry warning. + */ + wait_var_event(&cifs_sb->outstanding_rreq, + !atomic_read(&cifs_sb->outstanding_rreq)); + flush_workqueue(serverclose_wq); + flush_workqueue(fileinfo_put_wq); + /* finally release root dentry */ dput(cifs_sb->root); cifs_sb->root = NULL; diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index dcde25da468df..9511deef7084f 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -3479,6 +3479,7 @@ int cifs_setup_cifs_sb(struct cifs_sb_info *cifs_sb) spin_lock_init(&cifs_sb->tlink_tree_lock); cifs_sb->tlink_tree = RB_ROOT; + atomic_set(&cifs_sb->outstanding_rreq, 0); cifs_dbg(FYI, "file mode: %04ho dir mode: %04ho\n", ctx->file_mode, ctx->dir_mode); @@ -3996,6 +3997,9 @@ cifs_umount(struct cifs_sb_info *cifs_sb) } spin_unlock(&cifs_sb->tlink_tree_lock); + flush_workqueue(serverclose_wq); + flush_workqueue(fileinfo_put_wq); + kfree(cifs_sb->prepath); call_rcu(&cifs_sb->rcu, delayed_free); } diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index b60344125f271..a26a464d6242b 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -241,6 +241,7 @@ static void cifs_issue_read(struct netfs_io_subrequest *subreq) return; failed: + add_credits_and_wake_if(rdata->server, &rdata->credits, 0); subreq->error = rc; netfs_read_subreq_terminated(subreq); } @@ -287,6 +288,7 @@ static int cifs_init_request(struct netfs_io_request *rreq, struct file *file) return smb_EIO1(smb_eio_trace_not_netfs_writeback, rreq->origin); } + atomic_inc(&cifs_sb->outstanding_rreq); return 0; } @@ -301,16 +303,20 @@ static void cifs_rreq_done(struct netfs_io_request *rreq) /* we do not want atime to be less than mtime, it broke some apps */ atime = inode_set_atime_to_ts(inode, current_time(inode)); mtime = inode_get_mtime(inode); - if (timespec64_compare(&atime, &mtime)) + if (timespec64_compare(&atime, &mtime) < 0) inode_set_atime_to_ts(inode, inode_get_mtime(inode)); } static void cifs_free_request(struct netfs_io_request *rreq) { struct cifs_io_request *req = container_of(rreq, struct cifs_io_request, rreq); + struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); if (req->cfile) cifsFileInfo_put(req->cfile); + + if (atomic_dec_and_test(&cifs_sb->outstanding_rreq)) + wake_up_var(&cifs_sb->outstanding_rreq); } static void cifs_free_subrequest(struct netfs_io_subrequest *subreq) diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index 2f86158f85d7b..fd4b13cd654d9 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -693,6 +693,41 @@ static int smb3_handle_conflicting_options(struct fs_context *fc) { struct smb3_fs_context *ctx = smb3_fc2context(fc); + if (ctx->rdma && ctx->vals->protocol_id < SMB30_PROT_ID) { + cifs_errorf(fc, "SMB Direct requires Version >=3.0\n"); + return -EOPNOTSUPP; + } + + if (ctx->multiuser && !IS_ENABLED(CONFIG_KEYS)) { + cifs_errorf(fc, "Multiuser mounts require kernels with CONFIG_KEYS enabled\n"); + return -EOPNOTSUPP; + } + + if (ctx->multiuser && ctx->upcall_target == UPTARGET_MOUNT) { + cifs_errorf(fc, "multiuser mount option not supported with upcalltarget set as 'mount'\n"); + return -EINVAL; + } + + if (ctx->uid_specified && !ctx->forceuid_specified) { + ctx->override_uid = 1; + pr_notice("enabling forceuid mount option implicitly because uid= option is specified\n"); + } + + if (ctx->gid_specified && !ctx->forcegid_specified) { + ctx->override_gid = 1; + pr_notice("enabling forcegid mount option implicitly because gid= option is specified\n"); + } + + if (ctx->override_uid && !ctx->uid_specified) { + ctx->override_uid = 0; + pr_notice("ignoring forceuid mount option specified with no uid= option\n"); + } + + if (ctx->override_gid && !ctx->gid_specified) { + ctx->override_gid = 0; + pr_notice("ignoring forcegid mount option specified with no gid= option\n"); + } + if (ctx->multichannel_specified) { if (ctx->multichannel) { if (!ctx->max_channels_specified) { @@ -711,19 +746,14 @@ static int smb3_handle_conflicting_options(struct fs_context *fc) return -EINVAL; } } - } else { - if (ctx->max_channels_specified) { - if (ctx->max_channels > 1) - ctx->multichannel = true; - else - ctx->multichannel = false; - } else { + } else if (ctx->max_channels_specified) { + if (ctx->max_channels > 1) + ctx->multichannel = true; + else ctx->multichannel = false; - ctx->max_channels = 1; - } } - //resetting default values as remount doesn't initialize fs_context again + /* clear parse-time latches so they don't persist across remounts */ ctx->multichannel_specified = false; ctx->max_channels_specified = false; @@ -804,28 +834,23 @@ static int smb3_fs_context_parse_monolithic(struct fs_context *fc, if (ret < 0) break; } - return ret ?: smb3_handle_conflicting_options(fc); + return ret; } /* - * Validate the preparsed information in the config. + * smb3_fs_context_validate - check initial-mount-only constraints: + * UNC presence, address resolution, dialect warnings + * + * @fc: generic mount context */ static int smb3_fs_context_validate(struct fs_context *fc) { struct smb3_fs_context *ctx = smb3_fc2context(fc); + int rc; - if (ctx->rdma && ctx->vals->protocol_id < SMB30_PROT_ID) { - cifs_errorf(fc, "SMB Direct requires Version >=3.0\n"); - return -EOPNOTSUPP; - } - -#ifndef CONFIG_KEYS - /* Muliuser mounts require CONFIG_KEYS support */ - if (ctx->multiuser) { - cifs_errorf(fc, "Multiuser mounts require kernels with CONFIG_KEYS enabled\n"); - return -1; - } -#endif + rc = smb3_handle_conflicting_options(fc); + if (rc) + return rc; if (ctx->got_version == false) pr_warn_once("No dialect specified on mount. Default has changed to a more secure dialect, SMB2.1 or later (e.g. SMB3.1.1), from CIFS (SMB1). To use the less secure SMB1 dialect to access old servers which do not support SMB3.1.1 (or even SMB3 or SMB2.1) specify vers=1.0 on mount.\n"); @@ -860,26 +885,6 @@ static int smb3_fs_context_validate(struct fs_context *fc) /* set the port that we got earlier */ cifs_set_port((struct sockaddr *)&ctx->dstaddr, ctx->port); - if (ctx->uid_specified && !ctx->forceuid_specified) { - ctx->override_uid = 1; - pr_notice("enabling forceuid mount option implicitly because uid= option is specified\n"); - } - - if (ctx->gid_specified && !ctx->forcegid_specified) { - ctx->override_gid = 1; - pr_notice("enabling forcegid mount option implicitly because gid= option is specified\n"); - } - - if (ctx->override_uid && !ctx->uid_specified) { - ctx->override_uid = 0; - pr_notice("ignoring forceuid mount option specified with no uid= option\n"); - } - - if (ctx->override_gid && !ctx->gid_specified) { - ctx->override_gid = 0; - pr_notice("ignoring forcegid mount option specified with no gid= option\n"); - } - return 0; } @@ -1078,6 +1083,10 @@ static int smb3_reconfigure(struct fs_context *fc) if (rc) return rc; + rc = smb3_handle_conflicting_options(fc); + if (rc) + return rc; + old_ctx = kzalloc_obj(*old_ctx); if (!old_ctx) return -ENOMEM; @@ -1933,11 +1942,6 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, } /* case Opt_ignore: - is ignored as expected ... */ - if (ctx->multiuser && ctx->upcall_target == UPTARGET_MOUNT) { - cifs_errorf(fc, "multiuser mount option not supported with upcalltarget set as 'mount'\n"); - goto cifs_parse_mount_err; - } - return 0; cifs_parse_mount_err: diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index 9472c0a6c187c..826d36ed13ec9 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -28,6 +28,23 @@ #include "cached_dir.h" #include "reparse.h" +static void cifs_invalidate_cached_dir(struct cifs_tcon *tcon, + struct dentry *parent) +{ + struct cached_fid *parent_cfid = NULL; + + if (!tcon || !parent) + return; + + if (!open_cached_dir_by_dentry(tcon, parent, &parent_cfid)) { + mutex_lock(&parent_cfid->dirents.de_mutex); + parent_cfid->dirents.is_valid = false; + parent_cfid->dirents.is_failed = true; + mutex_unlock(&parent_cfid->dirents.de_mutex); + close_cached_dir(parent_cfid); + } +} + /* * Set parameters for the netfs library */ @@ -2067,6 +2084,9 @@ static int __cifs_unlink(struct inode *dir, struct dentry *dentry, bool sillyren cifs_set_file_info(inode, attrs, xid, full_path, origattr); out_reval: + if (!rc && dentry->d_parent) + cifs_invalidate_cached_dir(tcon, dentry->d_parent); + if (inode) { cifs_inode = CIFS_I(inode); cifs_inode->time = 0; /* will force revalidate to get info @@ -2378,7 +2398,6 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry) } rc = server->ops->rmdir(xid, tcon, full_path, cifs_sb); - cifs_put_tlink(tlink); cifsInode = CIFS_I(d_inode(direntry)); @@ -2388,6 +2407,8 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry) i_size_write(d_inode(direntry), 0); clear_nlink(d_inode(direntry)); spin_unlock(&d_inode(direntry)->i_lock); + if (direntry->d_parent) + cifs_invalidate_cached_dir(tcon, direntry->d_parent); } /* force revalidate to go get info when needed */ @@ -2402,6 +2423,7 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry) inode_set_ctime_current(d_inode(direntry)); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + cifs_put_tlink(tlink); rmdir_exit: free_dentry_path(page); @@ -2668,6 +2690,12 @@ cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir, } /* force revalidate to go get info when needed */ + if (!rc) { + cifs_invalidate_cached_dir(tcon, source_dentry->d_parent); + if (target_dentry->d_parent != source_dentry->d_parent) + cifs_invalidate_cached_dir(tcon, target_dentry->d_parent); + } + CIFS_I(source_dir)->time = CIFS_I(target_dir)->time = 0; cifs_rename_exit: @@ -3348,7 +3376,8 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (attrs->ia_valid & ATTR_GID) gid = attrs->ia_gid; - if (sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) { + if ((sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) || + cifs_sb_master_tcon(cifs_sb)->posix_extensions) { if (uid_valid(uid) || gid_valid(gid)) { mode = NO_CHANGE_64; rc = id_mode_to_cifs_acl(inode, full_path, &mode, diff --git a/fs/smb/client/ioctl.c b/fs/smb/client/ioctl.c index 17408bb8ab65b..746d70091f3d1 100644 --- a/fs/smb/client/ioctl.c +++ b/fs/smb/client/ioctl.c @@ -392,13 +392,11 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) } #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ #endif /* CONFIG_CIFS_POSIX */ - rc = 0; - if (CIFS_I(inode)->cifsAttrs & ATTR_COMPRESSED) { - /* add in the compressed bit */ - ExtAttrBits = FS_COMPR_FL; - rc = put_user(ExtAttrBits & FS_FL_USER_VISIBLE, - (int __user *)arg); - } + if (CIFS_I(inode)->cifsAttrs & FILE_ATTRIBUTE_COMPRESSED) + ExtAttrBits |= FS_COMPR_FL; + + rc = put_user(ExtAttrBits & FS_FL_USER_VISIBLE, + (int __user *)arg); break; case FS_IOC_SETFLAGS: if (pSMBFile == NULL) diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c index 0c54b9b79a2ce..ee1728eec8aa0 100644 --- a/fs/smb/client/misc.c +++ b/fs/smb/client/misc.c @@ -752,6 +752,10 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, node->ref_flag = le16_to_cpu(ref->ReferralEntryFlags); /* copy DfsPath */ + if (le16_to_cpu(ref->DfsPathOffset) > data_end - (char *)ref) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } temp = (char *)ref + le16_to_cpu(ref->DfsPathOffset); max_len = data_end - temp; node->path_name = cifs_strndup_from_utf16(temp, max_len, @@ -762,6 +766,10 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, } /* copy link target UNC */ + if (le16_to_cpu(ref->NetworkAddressOffset) > data_end - (char *)ref) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } temp = (char *)ref + le16_to_cpu(ref->NetworkAddressOffset); max_len = data_end - temp; node->node_name = cifs_strndup_from_utf16(temp, max_len, diff --git a/fs/smb/client/reparse.c b/fs/smb/client/reparse.c index cd1e1eaee67a6..5cc5b0410d489 100644 --- a/fs/smb/client/reparse.c +++ b/fs/smb/client/reparse.c @@ -67,6 +67,7 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, char *sym = NULL; struct kvec iov; bool directory; + int path_len; int rc = 0; if (strlen(symname) > REPARSE_SYM_PATH_MAX) @@ -168,7 +169,21 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') sym[0] = sym[1] = sym[2] = sym[5] = '_'; - path = cifs_convert_path_to_utf16(sym, cifs_sb); + /* + * On a POSIX paths mount the symlink target is stored verbatim, so + * convert it with cifs_strndup_to_utf16(). cifs_convert_path_to_utf16() + * must not be used here: it strips a leading path separator (it is + * meant for share-relative SMB paths), which would corrupt an absolute + * POSIX symlink target such as "/foo/bar". Using NO_MAP_UNI_RSVD also + * matches the readback path in smb2_parse_native_symlink(). + */ + if (sbflags & CIFS_MOUNT_POSIX_PATHS) + path = cifs_strndup_to_utf16(sym, strlen(sym), &path_len, + cifs_sb->local_nls, + NO_MAP_UNI_RSVD); + else + path = cifs_convert_path_to_utf16(sym, cifs_sb); + if (!path) { rc = -ENOMEM; goto out; diff --git a/fs/smb/client/smb1pdu.h b/fs/smb/client/smb1pdu.h index 7584e94d9b2bf..0870949144abf 100644 --- a/fs/smb/client/smb1pdu.h +++ b/fs/smb/client/smb1pdu.h @@ -1211,11 +1211,6 @@ typedef struct smb_com_transaction_compr_ioctl_req { __le16 compression_state; /* See below for valid flags */ } __packed TRANSACT_COMPR_IOCTL_REQ; -/* compression state flags */ -#define COMPRESSION_FORMAT_NONE 0x0000 -#define COMPRESSION_FORMAT_DEFAULT 0x0001 -#define COMPRESSION_FORMAT_LZNT1 0x0002 - typedef struct smb_com_transaction_ioctl_rsp { struct smb_hdr hdr; /* wct = 19 */ __u8 Reserved[3]; diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 2a7355ce1a078..9068175e57cd0 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -19,6 +19,9 @@ #include "nterr.h" #include "cached_dir.h" +static unsigned int __smb2_calc_size(void *buf, bool *have_data, + bool *data_area_overlap); + static int check_smb2_hdr(struct smb2_hdr *shdr, __u64 mid) { @@ -145,6 +148,8 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, int command; __u32 calc_len; /* calculated length */ __u64 mid; + bool have_data; + bool data_area_overlap; /* If server is a channel, select the primary channel */ pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; @@ -228,7 +233,13 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, } } - calc_len = smb2_calc_size(buf); + have_data = false; + data_area_overlap = false; + calc_len = __smb2_calc_size(buf, &have_data, &data_area_overlap); + + /* Reject responses whose data area overlaps the fixed area. */ + if (data_area_overlap) + return 1; /* For SMB2_IOCTL, OutputOffset and OutputLength are optional, so might * be 0, and not a real miscalculation */ @@ -247,8 +258,13 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, /* Windows 7 server returns 24 bytes more */ if (calc_len + 24 == len && command == SMB2_OPLOCK_BREAK_HE) return 0; - /* server can return one byte more due to implied bcc[0] */ - if (calc_len == len + 1) + /* + * Server can return one byte more due to implied bcc[0]. + * Allow it only when there is no data area; if data_length > 0 + * the +1 gap indicates an overreported data length rather than + * the bcc[0] omission. + */ + if (calc_len == len + 1 && !have_data) return 0; /* @@ -407,19 +423,28 @@ smb2_get_data_area_len(int *off, int *len, struct smb2_hdr *shdr) } /* - * Calculate the size of the SMB message based on the fixed header - * portion, the number of word parameters and the data portion of the message. + * Calculate the size of the SMB message based on the fixed header, fixed + * parameter area, and variable data area. + * + * If have_data is not NULL, it is set when a non-empty data area is found. + * If data_area_overlap is not NULL, it is set when the data area overlaps + * the fixed area. */ -unsigned int -smb2_calc_size(void *buf) +static unsigned int +__smb2_calc_size(void *buf, bool *have_data, bool *data_area_overlap) { struct smb2_pdu *pdu = buf; struct smb2_hdr *shdr = &pdu->hdr; int offset; /* the offset from the beginning of SMB to data area */ - int data_length; /* the length of the variable length data area */ + int data_length = 0; /* the length of the variable length data area */ /* Structure Size has already been checked to make sure it is 64 */ int len = le16_to_cpu(shdr->StructureSize); + if (have_data) + *have_data = false; + if (data_area_overlap) + *data_area_overlap = false; + /* * StructureSize2, ie length of fixed parameter area has already * been checked to make sure it is the correct length. @@ -442,16 +467,27 @@ smb2_calc_size(void *buf) if (offset + 1 < len) { cifs_dbg(VFS, "data area offset %d overlaps SMB2 header %d\n", offset + 1, len); + if (data_area_overlap) + *data_area_overlap = true; data_length = 0; + goto calc_size_exit; } else { len = offset + data_length; } } calc_size_exit: cifs_dbg(FYI, "SMB2 len %d\n", len); + if (have_data) + *have_data = (data_length > 0); return len; } +unsigned int +smb2_calc_size(void *buf) +{ + return __smb2_calc_size(buf, NULL, NULL); +} + /* Note: caller must free return buffer */ __le16 * cifs_convert_path_to_utf16(const char *from, struct cifs_sb_info *cifs_sb) diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index d4875f9532b4d..02c2f83353e27 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -1772,8 +1772,8 @@ smb2_ioctl_query_info(const unsigned int xid, if (le32_to_cpu(io_rsp->OutputCount) < qi.input_buffer_length) qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount); if (qi.input_buffer_length > 0 && - le32_to_cpu(io_rsp->OutputOffset) + qi.input_buffer_length - > rsp_iov[1].iov_len) { + size_add(le32_to_cpu(io_rsp->OutputOffset), + qi.input_buffer_length) > rsp_iov[1].iov_len) { rc = -EFAULT; goto out; } @@ -2117,8 +2117,9 @@ smb2_sync_write(const unsigned int xid, struct cifs_fid *pfid, } /* Set or clear the SPARSE_FILE attribute based on value passed in setsparse */ -static bool smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, - struct cifsFileInfo *cfile, struct inode *inode, __u8 setsparse) +static int smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct inode *inode, + __u8 setsparse) { struct cifsInodeInfo *cifsi; int rc; @@ -2127,31 +2128,31 @@ static bool smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, /* if file already sparse don't bother setting sparse again */ if ((cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) && setsparse) - return true; /* already sparse */ + return 0; /* already sparse */ if (!(cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) && !setsparse) - return true; /* already not sparse */ + return 0; /* already not sparse */ /* * Can't check for sparse support on share the usual way via the * FS attribute info (FILE_SUPPORTS_SPARSE_FILES) on the share * since Samba server doesn't set the flag on the share, yet * supports the set sparse FSCTL and returns sparse correctly - * in the file attributes. If we fail setting sparse though we - * mark that server does not support sparse files for this share - * to avoid repeatedly sending the unsupported fsctl to server - * if the file is repeatedly extended. + * in the file attributes. If the server returns EOPNOTSUPP, mark + * that sparse files are not supported on this share to avoid + * repeatedly sending the unsupported FSCTL. */ if (tcon->broken_sparse_sup) - return false; + return -EOPNOTSUPP; rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, FSCTL_SET_SPARSE, &setsparse, 1, CIFSMaxBufSize, NULL, NULL); if (rc) { - tcon->broken_sparse_sup = true; + if (rc == -EOPNOTSUPP) + tcon->broken_sparse_sup = true; cifs_dbg(FYI, "set sparse rc = %d\n", rc); - return false; + return rc; } if (setsparse) @@ -2159,7 +2160,7 @@ static bool smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, else cifsi->cifsAttrs &= (~FILE_ATTRIBUTE_SPARSE_FILE); - return true; + return 0; } static int @@ -3483,10 +3484,9 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, /* Need to make file sparse, if not already, before freeing range. */ /* Consider adding equivalent for compressed since it could also work */ - if (!smb2_set_sparse(xid, tcon, cfile, inode, set_sparse)) { - rc = -EOPNOTSUPP; + rc = smb2_set_sparse(xid, tcon, cfile, inode, set_sparse); + if (rc) goto out; - } filemap_invalidate_lock(inode->i_mapping); /* @@ -3595,7 +3595,7 @@ static int smb3_simple_fallocate_range(unsigned int xid, if (rc) goto out; - buf = kzalloc(1024 * 1024, GFP_KERNEL); + buf = kvzalloc(1024 * 1024, GFP_KERNEL); if (buf == NULL) { rc = -ENOMEM; goto out; @@ -3652,7 +3652,7 @@ static int smb3_simple_fallocate_range(unsigned int xid, out: kfree(out_data); - kfree(buf); + kvfree(buf); return rc; } @@ -4359,11 +4359,13 @@ static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst unsigned int req_size = sizeof(**req) + crypto_aead_reqsize(tfm); unsigned int iv_size = crypto_aead_ivsize(tfm); unsigned int len; + int ret; u8 *p; - *num_sgs = cifs_get_num_sgs(rqst, num_rqst, sig); - if (IS_ERR_VALUE((long)(int)*num_sgs)) - return ERR_PTR(*num_sgs); + ret = cifs_get_num_sgs(rqst, num_rqst, sig); + if (ret < 0) + return ERR_PTR(ret); + *num_sgs = ret; len = iv_size; len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1); @@ -5109,6 +5111,12 @@ receive_encrypted_standard(struct TCP_Server_Info *server, one_more: shdr = (struct smb2_hdr *)buf; next_cmd = le32_to_cpu(shdr->NextCommand); + + if (*num_mids >= MAX_COMPOUND) { + cifs_server_dbg(VFS, "too many PDUs in compound\n"); + return -1; + } + if (next_cmd) { if (WARN_ON_ONCE(next_cmd > pdu_length)) return -1; @@ -5132,10 +5140,6 @@ receive_encrypted_standard(struct TCP_Server_Info *server, mid_entry->resp_buf_size = server->pdu_size; } - if (*num_mids >= MAX_COMPOUND) { - cifs_server_dbg(VFS, "too many PDUs in compound\n"); - return -1; - } bufs[*num_mids] = buf; mids[(*num_mids)++] = mid_entry; diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index fbeb2156ddb6b..69b618e2f67a9 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -2371,9 +2371,9 @@ parse_posix_ctxt(struct create_context *cc, struct smb2_file_all_info *info, memset(posix, 0, sizeof(*posix)); - posix->nlink = le32_to_cpu(*(__le32 *)(beg + 0)); - posix->reparse_tag = le32_to_cpu(*(__le32 *)(beg + 4)); - posix->mode = le32_to_cpu(*(__le32 *)(beg + 8)); + posix->nlink = get_unaligned_le32(beg); + posix->reparse_tag = get_unaligned_le32(beg + 4); + posix->mode = get_unaligned_le32(beg + 8); sid = beg + 12; sid_len = posix_info_sid_size(sid, end); @@ -3280,6 +3280,8 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); oparms->replay = !!(retries); @@ -3505,6 +3507,8 @@ SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); @@ -3699,6 +3703,8 @@ __SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; query_attrs = false; server = cifs_pick_channel(ses); @@ -3911,6 +3917,8 @@ query_info(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; allocated = false; server = cifs_pick_channel(ses); @@ -4083,6 +4091,8 @@ SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); @@ -4425,6 +4435,8 @@ SMB2_flush(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); @@ -5368,7 +5380,7 @@ int posix_info_sid_size(const void *beg, const void *end) size_t subauth; int total; - if (beg + 1 > end) + if (beg + 2 > end) return -1; subauth = *(u8 *)(beg+1); @@ -5683,6 +5695,8 @@ SMB2_query_directory(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); diff --git a/fs/smb/client/smb2pdu.h b/fs/smb/client/smb2pdu.h index 30d70097fe2fa..b9bf2fa989d5b 100644 --- a/fs/smb/client/smb2pdu.h +++ b/fs/smb/client/smb2pdu.h @@ -195,10 +195,6 @@ struct network_resiliency_req { #define NO_FILE_ID 0xFFFFFFFFFFFFFFFFULL /* general ioctls to srv not to file */ -struct compress_ioctl { - __le16 CompressionState; /* See cifspdu.h for possible flag values */ -} __packed; - /* * Maximum number of iovs we need for an ioctl request. * [0] : struct smb2_ioctl_req diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index b99ec5a417fad..5b21ad3c15fb5 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -181,6 +181,7 @@ EM(netfs_trace_tcon_ref_get_find, "GET Find ") \ EM(netfs_trace_tcon_ref_get_find_sess_tcon, "GET FndSes") \ EM(netfs_trace_tcon_ref_get_reconnect_server, "GET Reconn") \ + EM(netfs_trace_tcon_ref_get_swn_notify, "GET SwnNot") \ EM(netfs_trace_tcon_ref_new, "NEW ") \ EM(netfs_trace_tcon_ref_new_ipc, "NEW Ipc ") \ EM(netfs_trace_tcon_ref_new_reconnect_server, "NEW Reconn") \ @@ -192,6 +193,7 @@ EM(netfs_trace_tcon_ref_put_mnt_ctx, "PUT MntCtx") \ EM(netfs_trace_tcon_ref_put_dfs_refer, "PUT DfsRfr") \ EM(netfs_trace_tcon_ref_put_reconnect_server, "PUT Reconn") \ + EM(netfs_trace_tcon_ref_put_swn_notify, "PUT SwnNot") \ EM(netfs_trace_tcon_ref_put_tlink, "PUT Tlink ") \ EM(netfs_trace_tcon_ref_see_cancelled_close, "SEE Cn-Cls") \ EM(netfs_trace_tcon_ref_see_fscache_collision, "SEE FV-CO!") \ diff --git a/fs/smb/common/fscc.h b/fs/smb/common/fscc.h index bc3012cc295da..859849a42fec5 100644 --- a/fs/smb/common/fscc.h +++ b/fs/smb/common/fscc.h @@ -100,6 +100,24 @@ struct duplicate_extents_to_file_ex { __le32 Reserved; } __packed; +/* + * compression state flags + * See MS-FSCC 2.3.18 + * MS-FSCC 2.3.67 + * MS-FSCC 2.4.9 + */ +#define COMPRESSION_FORMAT_NONE 0x0000 +#define COMPRESSION_FORMAT_DEFAULT 0x0001 +#define COMPRESSION_FORMAT_LZNT1 0x0002 + +/* + * See MS-FSCC 2.3.18 + * MS-FSCC 2.3.67 + */ +struct compress_ioctl { + __le16 CompressionState; +} __packed; + /* See MS-FSCC 2.3.20 */ struct fsctl_get_integrity_information_rsp { __le16 ChecksumAlgorithm; diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index e99409fa721cd..2c7096a782daa 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -133,15 +133,17 @@ static int calc_ntlmv2_hash(struct ksmbd_conn *conn, struct ksmbd_session *sess, * @blen: NTLMv2 blob length * @domain_name: domain name * @cryptkey: session crypto key + * @sess_key: derived session key output buffer * * Return: 0 on success, error number on error */ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, struct ntlmv2_resp *ntlmv2, int blen, char *domain_name, - char *cryptkey) + char *cryptkey, char *sess_key) { char ntlmv2_hash[CIFS_ENCPWD_SIZE]; char ntlmv2_rsp[CIFS_HMAC_MD5_HASH_SIZE]; + char base_key[SMB2_NTLMV2_SESSKEY_SIZE]; struct hmac_md5_ctx ctx; int rc; @@ -164,12 +166,21 @@ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, /* Generate the session key */ hmac_md5_usingrawkey(ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE, ntlmv2_rsp, CIFS_HMAC_MD5_HASH_SIZE, - sess->sess_key); + base_key); if (crypto_memneq(ntlmv2->ntlmv2_hash, ntlmv2_rsp, - CIFS_HMAC_MD5_HASH_SIZE)) - return -EINVAL; - return 0; + CIFS_HMAC_MD5_HASH_SIZE)) { + rc = -EINVAL; + goto out; + } + + memcpy(sess_key, base_key, sizeof(base_key)); + rc = 0; +out: + memzero_explicit(ntlmv2_hash, sizeof(ntlmv2_hash)); + memzero_explicit(ntlmv2_rsp, sizeof(ntlmv2_rsp)); + memzero_explicit(base_key, sizeof(base_key)); + return rc; } /** @@ -179,12 +190,13 @@ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, * @blob_len: length of the @authblob message * @conn: connection * @sess: session of connection + * @sess_key: derived session key output buffer * * Return: 0 on success, error number on error */ int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, int blob_len, struct ksmbd_conn *conn, - struct ksmbd_session *sess) + struct ksmbd_session *sess, char *sess_key) { char *domain_name; unsigned int nt_off, dn_off; @@ -224,8 +236,10 @@ int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, ret = ksmbd_auth_ntlmv2(conn, sess, (struct ntlmv2_resp *)((char *)authblob + nt_off), nt_len - CIFS_ENCPWD_SIZE, - domain_name, conn->ntlmssp.cryptkey); + domain_name, conn->ntlmssp.cryptkey, sess_key); kfree(domain_name); + if (ret) + return ret; /* The recovered secondary session key */ if (conn->ntlmssp.client_flags & NTLMSSP_NEGOTIATE_KEY_XCH) { @@ -245,8 +259,8 @@ int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, if (!ctx_arc4) return -ENOMEM; - arc4_setkey(ctx_arc4, sess->sess_key, SMB2_NTLMV2_SESSKEY_SIZE); - arc4_crypt(ctx_arc4, sess->sess_key, + arc4_setkey(ctx_arc4, sess_key, SMB2_NTLMV2_SESSKEY_SIZE); + arc4_crypt(ctx_arc4, sess_key, (char *)authblob + sess_key_off, sess_key_len); kfree_sensitive(ctx_arc4); } @@ -388,7 +402,8 @@ ksmbd_build_ntlmssp_challenge_blob(struct challenge_message *chgblob, #ifdef CONFIG_SMB_SERVER_KERBEROS5 int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, - int in_len, char *out_blob, int *out_len) + int in_len, char *out_blob, int *out_len, + char *sess_key) { struct ksmbd_spnego_authen_response *resp; struct ksmbd_login_response_ext *resp_ext = NULL; @@ -443,7 +458,7 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, ksmbd_free_user(user); } - memcpy(sess->sess_key, resp->payload, resp->session_key_len); + memcpy(sess_key, resp->payload, resp->session_key_len); memcpy(out_blob, resp->payload + resp->session_key_len, resp->spnego_blob_len); *out_len = resp->spnego_blob_len; @@ -454,7 +469,8 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, } #else int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, - int in_len, char *out_blob, int *out_len) + int in_len, char *out_blob, int *out_len, + char *sess_key) { return -EOPNOTSUPP; } @@ -513,7 +529,7 @@ struct derivation { bool binding; }; -static void generate_key(struct ksmbd_conn *conn, struct ksmbd_session *sess, +static void generate_key(struct ksmbd_conn *conn, const char *sess_key, struct kvec label, struct kvec context, __u8 *key, unsigned int key_size) { @@ -524,7 +540,7 @@ static void generate_key(struct ksmbd_conn *conn, struct ksmbd_session *sess, unsigned char prfhash[SMB2_HMACSHA256_SIZE]; struct hmac_sha256_ctx ctx; - hmac_sha256_init_usingrawkey(&ctx, sess->sess_key, + hmac_sha256_init_usingrawkey(&ctx, sess_key, SMB2_NTLMV2_SESSKEY_SIZE); hmac_sha256_update(&ctx, i, 4); hmac_sha256_update(&ctx, label.iov_base, label.iov_len); @@ -547,18 +563,21 @@ static int generate_smb3signingkey(struct ksmbd_session *sess, const struct derivation *signing) { struct channel *chann; - char *key; + char *key, *sess_key; chann = lookup_chann_list(sess, conn); if (!chann) return 0; - if (conn->dialect >= SMB30_PROT_ID && signing->binding) + if (conn->dialect >= SMB30_PROT_ID && signing->binding) { key = chann->smb3signingkey; - else + sess_key = chann->sess_key; + } else { key = sess->smb3signingkey; + sess_key = sess->sess_key; + } - generate_key(conn, sess, signing->label, signing->context, key, + generate_key(conn, sess_key, signing->label, signing->context, key, SMB3_SIGN_KEY_SIZE); if (!(conn->dialect >= SMB30_PROT_ID && signing->binding)) @@ -615,11 +634,11 @@ static void generate_smb3encryptionkey(struct ksmbd_conn *conn, struct ksmbd_session *sess, const struct derivation_twin *ptwin) { - generate_key(conn, sess, ptwin->encryption.label, + generate_key(conn, sess->sess_key, ptwin->encryption.label, ptwin->encryption.context, sess->smb3encryptionkey, SMB3_ENC_DEC_KEY_SIZE); - generate_key(conn, sess, ptwin->decryption.label, + generate_key(conn, sess->sess_key, ptwin->decryption.label, ptwin->decryption.context, sess->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE); diff --git a/fs/smb/server/auth.h b/fs/smb/server/auth.h index 5767aabc63c9b..f14b7c0332646 100644 --- a/fs/smb/server/auth.h +++ b/fs/smb/server/auth.h @@ -41,17 +41,18 @@ int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, void ksmbd_copy_gss_neg_header(void *buf); int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, struct ntlmv2_resp *ntlmv2, int blen, char *domain_name, - char *cryptkey); + char *cryptkey, char *sess_key); int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, int blob_len, struct ksmbd_conn *conn, - struct ksmbd_session *sess); + struct ksmbd_session *sess, char *sess_key); int ksmbd_decode_ntlmssp_neg_blob(struct negotiate_message *negblob, int blob_len, struct ksmbd_conn *conn); unsigned int ksmbd_build_ntlmssp_challenge_blob(struct challenge_message *chgblob, struct ksmbd_conn *conn); int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, - int in_len, char *out_blob, int *out_len); + int in_len, char *out_blob, int *out_len, + char *sess_key); void ksmbd_sign_smb2_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, int n_vec, char *sig); void ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index de58aed76cb42..d6331184ebfcb 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -255,7 +255,7 @@ static void free_channel_list(struct ksmbd_session *sess) down_write(&sess->chann_lock); xa_for_each(&sess->ksmbd_chann_list, index, chann) { xa_erase(&sess->ksmbd_chann_list, index); - kfree(chann); + kfree_sensitive(chann); } xa_destroy(&sess->ksmbd_chann_list); @@ -449,7 +449,7 @@ static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) if (!chann) return -ENOENT; - kfree(chann); + kfree_sensitive(chann); return 0; } diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index 6aebd385be847..4637a8c8436d0 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -19,6 +19,7 @@ struct ksmbd_file_table; struct channel { + char sess_key[CIFS_KEY_SIZE]; __u8 smb3signingkey[SMB3_SIGN_KEY_SIZE]; struct ksmbd_conn *conn; }; diff --git a/fs/smb/server/misc.c b/fs/smb/server/misc.c index a543ec9d35817..966004c414a8c 100644 --- a/fs/smb/server/misc.c +++ b/fs/smb/server/misc.c @@ -283,39 +283,6 @@ char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename) return ksmbd_casefold_sharename(um, name); } -/** - * convert_to_unix_name() - convert windows name to unix format - * @share: ksmbd_share_config pointer - * @name: file name that is relative to share - * - * Return: converted name on success, otherwise NULL - */ -char *convert_to_unix_name(struct ksmbd_share_config *share, const char *name) -{ - int no_slash = 0, name_len, path_len; - char *new_name; - - if (name[0] == '/') - name++; - - path_len = share->path_sz; - name_len = strlen(name); - new_name = kmalloc(path_len + name_len + 2, KSMBD_DEFAULT_GFP); - if (!new_name) - return new_name; - - memcpy(new_name, share->path, path_len); - if (new_name[path_len - 1] != '/') { - new_name[path_len] = '/'; - no_slash = 1; - } - - memcpy(new_name + path_len + no_slash, name, name_len); - path_len += name_len + no_slash; - new_name[path_len] = 0x00; - return new_name; -} - char *ksmbd_convert_dir_info_name(struct ksmbd_dir_info *d_info, const struct nls_table *local_nls, int *conv_len) diff --git a/fs/smb/server/misc.h b/fs/smb/server/misc.h index 13423696ae8c6..3909104e18ad4 100644 --- a/fs/smb/server/misc.h +++ b/fs/smb/server/misc.h @@ -25,7 +25,6 @@ void ksmbd_strip_last_slash(char *path); void ksmbd_conv_path_to_windows(char *path); char *ksmbd_casefold_sharename(struct unicode_map *um, const char *name); char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename); -char *convert_to_unix_name(struct ksmbd_share_config *share, const char *name); #define KSMBD_DIR_INFO_ALIGNMENT 8 struct ksmbd_dir_info; diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index b193dde4810dc..60e7e821c2455 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -528,7 +528,12 @@ static struct oplock_info *same_client_has_lease(struct ksmbd_inode *ci, ret = compare_guid_key(opinfo, client_guid, lctx->lease_key); if (ret) { + if (!atomic_inc_not_zero(&opinfo->refcount)) + continue; + if (m_opinfo) + opinfo_put(m_opinfo); m_opinfo = opinfo; + /* skip upgrading lease about breaking lease */ if (atomic_read(&opinfo->breaking_cnt)) continue; @@ -1246,6 +1251,7 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, if (atomic_read(&m_opinfo->breaking_cnt)) opinfo->o_lease->flags = SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE; + opinfo_put(m_opinfo); goto out; } } diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 3eb3b1711acb3..ef65b26270813 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -89,6 +89,47 @@ struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn return chann; } +#define KSMBD_MAX_CHANNELS 32 + +static int register_session_channel(struct ksmbd_session *sess, + struct ksmbd_conn *conn, + const char *sess_key) +{ + struct channel *chann, *old; + unsigned long index; + unsigned int count = 0; + int rc = 0; + + down_write(&sess->chann_lock); + if (xa_load(&sess->ksmbd_chann_list, (long)conn)) + goto out; + + xa_for_each(&sess->ksmbd_chann_list, index, chann) + count++; + if (count >= KSMBD_MAX_CHANNELS) { + rc = -ENOSPC; + goto out; + } + + chann = kmalloc_obj(struct channel, KSMBD_DEFAULT_GFP); + if (!chann) { + rc = -ENOMEM; + goto out; + } + + chann->conn = conn; + memcpy(chann->sess_key, sess_key, sizeof(chann->sess_key)); + old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, + KSMBD_DEFAULT_GFP); + if (xa_is_err(old)) { + kfree_sensitive(chann); + rc = xa_err(old); + } +out: + up_write(&sess->chann_lock); + return rc; +} + /** * smb2_get_ksmbd_tcon() - get tree connection information using a tree id. * @work: smb work @@ -613,6 +654,11 @@ int smb2_check_user_session(struct ksmbd_work *work) sess_id, work->sess->id); return -EINVAL; } + if (work->sess->state != SMB2_SESSION_VALID) { + pr_err("compound request on a non-valid session (state %d)\n", + work->sess->state); + return -EINVAL; + } return 1; } @@ -1460,9 +1506,11 @@ static int ntlm_authenticate(struct ksmbd_work *work, { struct ksmbd_conn *conn = work->conn; struct ksmbd_session *sess = work->sess; - struct channel *chann = NULL, *old; struct ksmbd_user *user; + char channel_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : sess->sess_key; u64 prev_id; + bool binding = conn->binding; int sz, rc; ksmbd_debug(SMB, "authenticate phase\n"); @@ -1521,11 +1569,13 @@ static int ntlm_authenticate(struct ksmbd_work *work, sz = conn->mechTokenLen; else sz = le16_to_cpu(req->SecurityBufferLength); - rc = ksmbd_decode_ntlmssp_auth_blob(authblob, sz, conn, sess); + rc = ksmbd_decode_ntlmssp_auth_blob(authblob, sz, conn, sess, + auth_key); if (rc) { set_user_flag(sess->user, KSMBD_USER_FLAG_BAD_PASSWORD); ksmbd_debug(SMB, "authentication failed\n"); - return -EPERM; + rc = -EPERM; + goto out; } } @@ -1560,37 +1610,30 @@ static int ntlm_authenticate(struct ksmbd_work *work, binding_session: if (conn->dialect >= SMB30_PROT_ID) { - chann = lookup_chann_list(sess, conn); - if (!chann) { - chann = kmalloc_obj(struct channel, KSMBD_DEFAULT_GFP); - if (!chann) - return -ENOMEM; - - chann->conn = conn; - down_write(&sess->chann_lock); - old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, - KSMBD_DEFAULT_GFP); - up_write(&sess->chann_lock); - if (xa_is_err(old)) { - kfree(chann); - return xa_err(old); - } - } + rc = register_session_channel(sess, conn, auth_key); + if (rc) + goto out; } if (conn->ops->generate_signingkey) { rc = conn->ops->generate_signingkey(sess, conn); if (rc) { ksmbd_debug(SMB, "SMB3 signing key generation failed\n"); - return -EINVAL; + rc = -EINVAL; + goto out; } } if (!ksmbd_conn_lookup_dialect(conn)) { pr_err("fail to verify the dialect\n"); - return -ENOENT; + rc = -ENOENT; + goto out; } - return 0; + rc = 0; +out: + if (binding) + memzero_explicit(channel_key, sizeof(channel_key)); + return rc; } #ifdef CONFIG_SMB_SERVER_KERBEROS5 @@ -1601,8 +1644,10 @@ static int krb5_authenticate(struct ksmbd_work *work, struct ksmbd_conn *conn = work->conn; struct ksmbd_session *sess = work->sess; char *in_blob, *out_blob; - struct channel *chann = NULL, *old; + char channel_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : sess->sess_key; u64 prev_sess_id; + bool binding = conn->binding; int in_len, out_len; int retval; @@ -1615,10 +1660,11 @@ static int krb5_authenticate(struct ksmbd_work *work, (le16_to_cpu(rsp->SecurityBufferOffset) + 4); retval = ksmbd_krb5_authenticate(sess, in_blob, in_len, - out_blob, &out_len); + out_blob, &out_len, auth_key); if (retval) { ksmbd_debug(SMB, "krb5 authentication failed\n"); - return -EINVAL; + retval = -EINVAL; + goto out; } /* Check previous session */ @@ -1655,37 +1701,30 @@ static int krb5_authenticate(struct ksmbd_work *work, binding_session: if (conn->dialect >= SMB30_PROT_ID) { - chann = lookup_chann_list(sess, conn); - if (!chann) { - chann = kmalloc_obj(struct channel, KSMBD_DEFAULT_GFP); - if (!chann) - return -ENOMEM; - - chann->conn = conn; - down_write(&sess->chann_lock); - old = xa_store(&sess->ksmbd_chann_list, (long)conn, - chann, KSMBD_DEFAULT_GFP); - up_write(&sess->chann_lock); - if (xa_is_err(old)) { - kfree(chann); - return xa_err(old); - } - } + retval = register_session_channel(sess, conn, auth_key); + if (retval) + goto out; } if (conn->ops->generate_signingkey) { retval = conn->ops->generate_signingkey(sess, conn); if (retval) { ksmbd_debug(SMB, "SMB3 signing key generation failed\n"); - return -EINVAL; + retval = -EINVAL; + goto out; } } if (!ksmbd_conn_lookup_dialect(conn)) { pr_err("fail to verify the dialect\n"); - return -ENOENT; + retval = -ENOENT; + goto out; } - return 0; + retval = 0; +out: + if (binding) + memzero_explicit(channel_key, sizeof(channel_key)); + return retval; } #else static int krb5_authenticate(struct ksmbd_work *work, @@ -1907,7 +1946,7 @@ int smb2_sess_setup(struct ksmbd_work *work) rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; else if (rc == -EFAULT) rsp->hdr.Status = STATUS_NETWORK_SESSION_EXPIRED; - else if (rc == -ENOMEM) + else if (rc == -ENOMEM || rc == -ENOSPC) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; else if (rc == -EOPNOTSUPP) rsp->hdr.Status = STATUS_NOT_SUPPORTED; @@ -1946,8 +1985,16 @@ int smb2_sess_setup(struct ksmbd_work *work) sess->last_active = jiffies; sess->state = SMB2_SESSION_EXPIRED; } - ksmbd_user_session_put(sess); - work->sess = NULL; + /* + * Keep the binding session reference until the response is + * signed and sent. Error responses for a signed binding + * request are signed with the existing session signing key. + */ + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING) || + work->sess != sess) { + ksmbd_user_session_put(sess); + work->sess = NULL; + } if (try_delay) { ksmbd_conn_set_need_reconnect(conn); ssleep(5); @@ -4465,6 +4512,8 @@ int smb2_query_dir(struct ksmbd_work *work) ksmbd_debug(SMB, "Search pattern is %s\n", srch_ptr); } + mutex_lock(&dir_fp->readdir_lock); + if (srch_flag & SMB2_REOPEN || srch_flag & SMB2_RESTART_SCANS) { ksmbd_debug(SMB, "Restart directory scan\n"); generic_file_llseek(dir_fp->filp, 0, SEEK_SET); @@ -4569,6 +4618,7 @@ int smb2_query_dir(struct ksmbd_work *work) goto err_out; } + mutex_unlock(&dir_fp->readdir_lock); kfree(srch_ptr); ksmbd_fd_put(work, dir_fp); ksmbd_revert_fsids(work); @@ -4576,6 +4626,7 @@ int smb2_query_dir(struct ksmbd_work *work) err_out: pr_err("error while processing smb2 query dir rc = %d\n", rc); + mutex_unlock(&dir_fp->readdir_lock); kfree(srch_ptr); err_out2: @@ -5328,6 +5379,12 @@ static int find_file_posix_info(struct smb2_query_info_rsp *rsp, int out_buf_len = sizeof(struct smb311_posix_qinfo) + 32; int ret; + if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { + pr_err("no right to read the attributes : 0x%x\n", + fp->daccess); + return -EACCES; + } + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); if (ret) @@ -6325,6 +6382,7 @@ static int set_file_allocation_info(struct ksmbd_work *work, */ loff_t alloc_blks; + u64 alloc_size; struct inode *inode; struct kstat stat; int rc; @@ -6340,7 +6398,19 @@ static int set_file_allocation_info(struct ksmbd_work *work, if (rc) return rc; - alloc_blks = (le64_to_cpu(file_alloc_info->AllocationSize) + 511) >> 9; + /* + * AllocationSize is fully client-controlled (the caller only + * validates the fixed 8-byte buffer length). Reject values that + * would overflow the "round up to 512-byte blocks" conversion + * below instead of silently wrapping it to a tiny block count, + * which would truncate the file to a size the client never + * asked for. + */ + alloc_size = le64_to_cpu(file_alloc_info->AllocationSize); + if (alloc_size > MAX_LFS_FILESIZE - 511) + return -EINVAL; + + alloc_blks = (alloc_size + 511) >> 9; inode = file_inode(fp->filp); if (alloc_blks > stat.blocks) { @@ -6549,11 +6619,18 @@ static int smb2_set_info_file(struct ksmbd_work *work, struct ksmbd_file *fp, } case FILE_LINK_INFORMATION: { + struct smb2_file_link_info *file_info; + if (buf_len < sizeof(struct smb2_file_link_info)) return -EMSGSIZE; - return smb2_create_link(work, work->tcon->share_conf, - (struct smb2_file_link_info *)buffer, + file_info = (struct smb2_file_link_info *)buffer; + if (file_info->ReplaceIfExists && !(fp->daccess & FILE_DELETE_LE)) { + pr_err("no right to delete : 0x%x\n", fp->daccess); + return -EACCES; + } + + return smb2_create_link(work, work->tcon->share_conf, file_info, buf_len, fp->filp, work->conn->local_nls); } @@ -6606,6 +6683,9 @@ static int smb2_set_info_sec(struct ksmbd_file *fp, int addition_info, fp->saccess |= FILE_SHARE_DELETE_LE; + if (!(fp->daccess & (FILE_WRITE_DAC_LE | FILE_WRITE_OWNER_LE))) + return -EACCES; + return set_info_sec(fp->conn, fp->tcon, &fp->filp->f_path, pntsd, buf_len, false, true); } @@ -6618,6 +6698,7 @@ static int smb2_set_info_sec(struct ksmbd_file *fp, int addition_info, */ int smb2_set_info(struct ksmbd_work *work) { + const struct cred *saved_cred; struct smb2_set_info_req *req; struct smb2_set_info_rsp *rsp; struct ksmbd_file *fp = NULL; @@ -6659,6 +6740,7 @@ int smb2_set_info(struct ksmbd_work *work) goto err_out; } + saved_cred = override_creds(fp->filp->f_cred); switch (req->InfoType) { case SMB2_O_INFO_FILE: ksmbd_debug(SMB, "GOT SMB2_O_INFO_FILE\n"); @@ -6666,19 +6748,15 @@ int smb2_set_info(struct ksmbd_work *work) break; case SMB2_O_INFO_SECURITY: ksmbd_debug(SMB, "GOT SMB2_O_INFO_SECURITY\n"); - if (ksmbd_override_fsids(work)) { - rc = -ENOMEM; - goto err_out; - } rc = smb2_set_info_sec(fp, le32_to_cpu(req->AdditionalInformation), (char *)req + le16_to_cpu(req->BufferOffset), le32_to_cpu(req->BufferLength)); - ksmbd_revert_fsids(work); break; default: rc = -EOPNOTSUPP; } + revert_creds(saved_cred); if (rc < 0) goto err_out; @@ -7323,14 +7401,14 @@ int smb2_cancel(struct ksmbd_work *work) continue; /* - * A cancelled deferred byte-range lock frees its - * file_lock and takes the smb2_lock() early-exit that - * skips release_async_work(), so the work stays on - * conn->async_requests with a live cancel_fn pointing - * at the freed file_lock. Re-firing it on a second - * SMB2_CANCEL is a use-after-free. + * Only an ACTIVE deferred work may have its cancel_fn + * fired. A CANCELLED or CLOSED work already took the + * smb2_lock() non-ACTIVE early-exit that frees the + * file_lock and skips release_async_work(), so it is + * still on conn->async_requests with a live cancel_fn + * pointing at the freed file_lock. */ - if (iter->state == KSMBD_WORK_CANCELLED) + if (iter->state != KSMBD_WORK_ACTIVE) break; ksmbd_debug(SMB, @@ -7623,9 +7701,11 @@ int smb2_lock(struct ksmbd_work *work) nolock = 0; list_del(&cmp_lock->flist); list_del(&cmp_lock->clist); + cmp_lock->conn = NULL; spin_unlock(&conn->llist_lock); up_read(&conn_list_lock); + ksmbd_conn_put(conn); locks_free_lock(cmp_lock->fl); kfree(cmp_lock); goto out_check_cl; @@ -7737,31 +7817,30 @@ int smb2_lock(struct ksmbd_work *work) list_del(&work->fp_entry); spin_unlock(&fp->f_lock); - if (work->state != KSMBD_WORK_ACTIVE) { - list_del(&smb_lock->llist); - locks_free_lock(flock); + list_del(&smb_lock->llist); + release_async_work(work); - if (work->state == KSMBD_WORK_CANCELLED) { - rsp->hdr.Status = - STATUS_CANCELLED; - kfree(smb_lock); - smb2_send_interim_resp(work, - STATUS_CANCELLED); - work->send_no_response = 1; - goto out; - } + if (work->state == KSMBD_WORK_ACTIVE) + goto retry; + + locks_free_lock(flock); - rsp->hdr.Status = - STATUS_RANGE_NOT_LOCKED; + if (work->state == KSMBD_WORK_CANCELLED) { + rsp->hdr.Status = STATUS_CANCELLED; kfree(smb_lock); - goto out2; + smb2_send_interim_resp(work, + STATUS_CANCELLED); + work->send_no_response = 1; + goto out; } - list_del(&smb_lock->llist); - release_async_work(work); - goto retry; + rsp->hdr.Status = + STATUS_RANGE_NOT_LOCKED; + kfree(smb_lock); + goto out2; } else if (!rc) { list_add(&smb_lock->llist, &rollback_list); + smb_lock->conn = ksmbd_conn_get(work->conn); spin_lock(&work->conn->llist_lock); list_add_tail(&smb_lock->clist, &work->conn->lock_list); @@ -7816,11 +7895,14 @@ int smb2_lock(struct ksmbd_work *work) } list_del(&smb_lock->llist); - spin_lock(&work->conn->llist_lock); + conn = smb_lock->conn; + spin_lock(&conn->llist_lock); if (!list_empty(&smb_lock->flist)) list_del(&smb_lock->flist); list_del(&smb_lock->clist); - spin_unlock(&work->conn->llist_lock); + smb_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + ksmbd_conn_put(conn); locks_free_lock(smb_lock->fl); if (rlock) @@ -8238,6 +8320,7 @@ static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id, if (fp->f_ci->m_fattr != old_fattr && test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_STORE_DOS_ATTRS)) { + const struct cred *saved_cred; struct xattr_dos_attrib da; ret = ksmbd_vfs_get_dos_attrib_xattr(idmap, @@ -8246,9 +8329,11 @@ static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id, goto out; da.attr = le32_to_cpu(fp->f_ci->m_fattr); + saved_cred = override_creds(fp->filp->f_cred); ret = ksmbd_vfs_set_dos_attrib_xattr(idmap, &fp->filp->f_path, &da, true); + revert_creds(saved_cred); if (ret) fp->f_ci->m_fattr = old_fattr; } @@ -8482,6 +8567,12 @@ int smb2_ioctl(struct ksmbd_work *work) goto out; } + if (!(fp->daccess & FILE_WRITE_DATA_LE)) { + ksmbd_fd_put(work, fp); + ret = -EACCES; + goto out; + } + ret = ksmbd_vfs_zero_data(work, fp, off, len); ksmbd_fd_put(work, fp); if (ret < 0) @@ -8557,6 +8648,21 @@ int smb2_ioctl(struct ksmbd_work *work) goto dup_ext_out; } + if (!test_tree_conn_flag(work->tcon, + KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ret = -EACCES; + goto dup_ext_out; + } + + if (!(fp_out->daccess & FILE_WRITE_DATA_LE)) { + ret = -EACCES; + goto dup_ext_out; + } + if (!(fp_in->daccess & FILE_READ_DATA_LE)) { + ret = -EACCES; + goto dup_ext_out; + } + src_off = le64_to_cpu(dup_ext->SourceFileOffset); dst_off = le64_to_cpu(dup_ext->TargetFileOffset); length = le64_to_cpu(dup_ext->ByteCount); @@ -8974,7 +9080,6 @@ bool smb2_is_sign_req(struct ksmbd_work *work, unsigned int command) if ((rcv_hdr2->Flags & SMB2_FLAGS_SIGNED) && command != SMB2_NEGOTIATE_HE && - command != SMB2_SESSION_SETUP_HE && command != SMB2_OPLOCK_BREAK_HE) return true; @@ -9123,13 +9228,14 @@ void smb3_set_sign_rsp(struct ksmbd_work *work) struct channel *chann; char signature[SMB2_CMACAES_SIZE]; struct kvec *iov; + u16 command = conn->ops->get_cmd_val(work); int n_vec = 1; char *signing_key; hdr = ksmbd_resp_buf_curr(work); - if (conn->binding == false && - le16_to_cpu(hdr->Command) == SMB2_SESSION_SETUP_HE) { + if (command == SMB2_SESSION_SETUP_HE && + (!conn->binding || hdr->Status != STATUS_SUCCESS)) { signing_key = work->sess->smb3signingkey; } else { chann = lookup_chann_list(work->sess, work->conn); @@ -9172,10 +9278,13 @@ void smb3_preauth_hash_rsp(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); - if (le16_to_cpu(req->Command) == SMB2_NEGOTIATE_HE && - conn->preauth_info) - ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, - conn->preauth_info->Preauth_HashValue); + if (le16_to_cpu(req->Command) == SMB2_NEGOTIATE_HE) { + ksmbd_conn_lock(conn); + if (conn->preauth_info) + ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, + conn->preauth_info->Preauth_HashValue); + ksmbd_conn_unlock(conn); + } if (le16_to_cpu(rsp->Command) == SMB2_SESSION_SETUP_HE && sess) { __u8 *hash_value; diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h index e7cf573e59f00..3bed676bb5ad4 100644 --- a/fs/smb/server/smb2pdu.h +++ b/fs/smb/server/smb2pdu.h @@ -230,9 +230,6 @@ struct smb2_file_mode_info { __le32 Mode; } __packed; -#define COMPRESSION_FORMAT_NONE 0x0000 -#define COMPRESSION_FORMAT_LZNT1 0x0002 - struct smb2_file_comp_info { __le64 CompressedFileSize; __le16 CompressionFormat; diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index 664b1b4a3233d..fc9937cedb012 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -374,6 +374,7 @@ static void parse_dacl(struct mnt_idmap *idmap, { int i, ret; u16 num_aces = 0; + u16 dacl_size; unsigned int acl_size; char *acl_base; struct smb_ace **ppace; @@ -403,7 +404,11 @@ static void parse_dacl(struct mnt_idmap *idmap, if (num_aces <= 0) return; - if (num_aces > (le16_to_cpu(pdacl->size) - sizeof(struct smb_acl)) / + dacl_size = le16_to_cpu(pdacl->size); + if (dacl_size < sizeof(struct smb_acl)) + return; + + if (num_aces > (dacl_size - sizeof(struct smb_acl)) / (offsetof(struct smb_ace, sid) + offsetof(struct smb_sid, sub_auth) + sizeof(__le16))) return; @@ -1477,7 +1482,9 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, break; aces_size -= ace_size; - if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + ace_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE + + sizeof(__le32) * ace->sid.num_subauth) break; if (!compare_sids(&sid, &ace->sid) || diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index d08973b288e58..8b6b2ec896286 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -254,17 +254,20 @@ static ssize_t ksmbd_vfs_getcasexattr(struct mnt_idmap *idmap, static int ksmbd_vfs_stream_read(struct ksmbd_file *fp, char *buf, loff_t *pos, size_t count) { + const struct cred *saved_cred; ssize_t v_len; char *stream_buf = NULL; ksmbd_debug(VFS, "read stream data pos : %llu, count : %zd\n", *pos, count); + saved_cred = override_creds(fp->filp->f_cred); v_len = ksmbd_vfs_getcasexattr(file_mnt_idmap(fp->filp), fp->filp->f_path.dentry, fp->stream.name, fp->stream.size, &stream_buf); + revert_creds(saved_cred); if ((int)v_len <= 0) return (int)v_len; @@ -388,6 +391,7 @@ int ksmbd_vfs_read(struct ksmbd_work *work, struct ksmbd_file *fp, size_t count, static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, size_t count) { + const struct cred *saved_cred; char *stream_buf = NULL, *wbuf; struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); size_t size; @@ -408,6 +412,7 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, count = XATTR_SIZE_MAX - *pos; } + saved_cred = override_creds(fp->filp->f_cred); v_len = ksmbd_vfs_getcasexattr(idmap, fp->filp->f_path.dentry, fp->stream.name, @@ -416,14 +421,14 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, if (v_len < 0) { pr_err("not found stream in xattr : %zd\n", v_len); err = v_len; - goto out; + goto out_revert; } if (v_len < size) { wbuf = kvzalloc(size, KSMBD_DEFAULT_GFP); if (!wbuf) { err = -ENOMEM; - goto out; + goto out_revert; } if (v_len > 0) @@ -441,6 +446,8 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, size, 0, true); +out_revert: + revert_creds(saved_cred); if (err < 0) goto out; else @@ -919,15 +926,21 @@ void ksmbd_vfs_set_fadvise(struct file *filp, __le32 option) int ksmbd_vfs_zero_data(struct ksmbd_work *work, struct ksmbd_file *fp, loff_t off, loff_t len) { + const struct cred *saved_cred; + int err; + smb_break_all_levII_oplock(work, fp, 1); + saved_cred = override_creds(fp->filp->f_cred); if (fp->f_ci->m_fattr & FILE_ATTRIBUTE_SPARSE_FILE_LE) - return vfs_fallocate(fp->filp, - FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - off, len); - - return vfs_fallocate(fp->filp, - FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE, - off, len); + err = vfs_fallocate(fp->filp, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, len); + else + err = vfs_fallocate(fp->filp, + FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE, + off, len); + revert_creds(saved_cred); + return err; } int ksmbd_vfs_fqar_lseek(struct ksmbd_file *fp, loff_t start, loff_t length, @@ -1009,13 +1022,15 @@ int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap, int ksmbd_vfs_unlink(struct file *filp) { + const struct cred *saved_cred; int err = 0; struct dentry *dir, *dentry = filp->f_path.dentry; struct mnt_idmap *idmap = file_mnt_idmap(filp); + saved_cred = override_creds(filp->f_cred); err = mnt_want_write(filp->f_path.mnt); if (err) - return err; + goto out_revert; dir = dget_parent(dentry); dentry = start_removing_dentry(dir, dentry); @@ -1034,7 +1049,8 @@ int ksmbd_vfs_unlink(struct file *filp) out: dput(dir); mnt_drop_write(filp->f_path.mnt); - +out_revert: + revert_creds(saved_cred); return err; } @@ -1149,7 +1165,7 @@ int __ksmbd_vfs_kern_path(struct ksmbd_work *work, char *filepath, retry: err = ksmbd_vfs_path_lookup(share_conf, filepath, flags, path, for_remove); - if (!err || !caseless) + if (!err || err != -ENOENT || !caseless) return err; path_len = strlen(filepath); @@ -1251,15 +1267,36 @@ struct dentry *ksmbd_vfs_kern_path_create(struct ksmbd_work *work, unsigned int flags, struct path *path) { - char *abs_name; + struct ksmbd_share_config *share_conf = work->tcon->share_conf; + struct qstr last; struct dentry *dent; + int err, type; + + /* resolve the name beneath the share root so ".." cannot escape */ + CLASS(filename_kernel, filename)(name); + + err = vfs_path_parent_lookup(filename, flags | LOOKUP_BENEATH, + path, &last, &type, + &share_conf->vfs_path); + if (err) + return ERR_PTR(err); + + if (unlikely(type != LAST_NORM)) { + path_put(path); + return ERR_PTR(-EINVAL); + } - abs_name = convert_to_unix_name(work->tcon->share_conf, name); - if (!abs_name) - return ERR_PTR(-ENOMEM); + err = mnt_want_write(path->mnt); + if (err) { + path_put(path); + return ERR_PTR(err); + } - dent = start_creating_path(AT_FDCWD, abs_name, path, flags); - kfree(abs_name); + dent = start_creating_noperm(path->dentry, &last); + if (IS_ERR(dent)) { + mnt_drop_write(path->mnt); + path_put(path); + } return dent; } diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index ba3355a6057a2..aa0924c9fdf96 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -385,10 +385,14 @@ static void __ksmbd_inode_close(struct ksmbd_file *fp) up_write(&ci->m_lock); if (remove_stream_xattr) { + const struct cred *saved_cred; + + saved_cred = override_creds(filp->f_cred); err = ksmbd_vfs_remove_xattr(file_mnt_idmap(filp), &filp->f_path, fp->stream.name, true); + revert_creds(saved_cred); if (err) pr_err("remove xattr failed : %s\n", fp->stream.name); @@ -480,10 +484,14 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) * there are not accesses to fp->lock_list. */ list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - if (!list_empty(&smb_lock->clist) && fp->conn) { - spin_lock(&fp->conn->llist_lock); - list_del(&smb_lock->clist); - spin_unlock(&fp->conn->llist_lock); + struct ksmbd_conn *conn = smb_lock->conn; + + if (conn) { + spin_lock(&conn->llist_lock); + list_del_init(&smb_lock->clist); + smb_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + ksmbd_conn_put(conn); } list_del(&smb_lock->flist); @@ -790,6 +798,7 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp) INIT_LIST_HEAD(&fp->node); INIT_LIST_HEAD(&fp->lock_list); spin_lock_init(&fp->f_lock); + mutex_init(&fp->readdir_lock); atomic_set(&fp->refcount, 1); fp->filp = filp; @@ -1220,16 +1229,21 @@ void ksmbd_stop_durable_scavenger(void) static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user) { + char *name; + if (!user) return -EINVAL; /* Duplicate the user name to ensure identity persistence */ - fp->owner.name = kstrdup(user->name, GFP_KERNEL); - if (!fp->owner.name) + name = kstrdup(user->name, GFP_KERNEL); + if (!name) return -ENOMEM; + spin_lock(&fp->f_lock); fp->owner.uid = user->uid; fp->owner.gid = user->gid; + fp->owner.name = name; + spin_unlock(&fp->f_lock); return 0; } @@ -1247,18 +1261,24 @@ static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user) { - if (!user || !fp->owner.name) + bool ret = false; + + if (!user) return false; + spin_lock(&fp->f_lock); + if (!fp->owner.name) + goto out; + /* Check if the UID and GID match first (fast path) */ if (fp->owner.uid != user->uid || fp->owner.gid != user->gid) - return false; + goto out; /* Validate the account name to ensure the same SecurityContext */ - if (strcmp(fp->owner.name, user->name)) - return false; - - return true; + ret = (strcmp(fp->owner.name, user->name) == 0); +out: + spin_unlock(&fp->f_lock); + return ret; } static bool session_fd_check(struct ksmbd_tree_connect *tcon, @@ -1298,9 +1318,15 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, up_write(&ci->m_lock); list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - spin_lock(&conn->llist_lock); + struct ksmbd_conn *lock_conn = smb_lock->conn; + + if (!lock_conn) + continue; + spin_lock(&lock_conn->llist_lock); list_del_init(&smb_lock->clist); - spin_unlock(&conn->llist_lock); + smb_lock->conn = NULL; + spin_unlock(&lock_conn->llist_lock); + ksmbd_conn_put(lock_conn); } fp->conn = NULL; @@ -1430,6 +1456,7 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) } list_for_each_entry(smb_lock, &fp->lock_list, flist) { + smb_lock->conn = ksmbd_conn_get(conn); spin_lock(&conn->llist_lock); list_add_tail(&smb_lock->clist, &conn->lock_list); spin_unlock(&conn->llist_lock); @@ -1444,9 +1471,11 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) } up_write(&ci->m_lock); + spin_lock(&fp->f_lock); fp->owner.uid = fp->owner.gid = 0; kfree(fp->owner.name); fp->owner.name = NULL; + spin_unlock(&fp->f_lock); return 0; } diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index e6871266a94ba..a3a9fda6de917 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,7 @@ struct ksmbd_session; struct ksmbd_lock { struct file_lock *fl; + struct ksmbd_conn *conn; struct list_head clist; struct list_head flist; struct list_head llist; @@ -113,6 +115,7 @@ struct ksmbd_file { /* if ls is happening on directory, below is valid*/ struct ksmbd_readdir_data readdir_data; + struct mutex readdir_lock; int dot_dotdot[2]; unsigned int f_state; bool reserve_lease_break; diff --git a/fs/super.c b/fs/super.c index 378e81efe643b..97df9e574d8bf 100644 --- a/fs/super.c +++ b/fs/super.c @@ -741,12 +741,13 @@ struct super_block *sget_fc(struct fs_context *fc, int err; /* - * Never allow s_user_ns != &init_user_ns when FS_USERNS_MOUNT is - * not set, as the filesystem is likely unprepared to handle it. - * This can happen when fsconfig() is called from init_user_ns with - * an fs_fd opened in another user namespace. + * Never allow s_user_ns != &init_user_ns when FS_USERNS_MOUNT or + * FS_USERNS_DELEGATABLE is not set, as the filesystem is likely + * unprepared to handle it. This can happen when fsconfig() is called + * from init_user_ns with an fs_fd opened in another user namespace. */ - if (user_ns != &init_user_ns && !(fc->fs_type->fs_flags & FS_USERNS_MOUNT)) { + if (user_ns != &init_user_ns && + !(fc->fs_type->fs_flags & (FS_USERNS_MOUNT | FS_USERNS_DELEGATABLE))) { errorfc(fc, "VFS: Mounting from non-initial user namespace is not allowed"); return ERR_PTR(-EPERM); } diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c index 5709cede1d756..25b44fe171a32 100644 --- a/fs/sysfs/file.c +++ b/fs/sysfs/file.c @@ -120,6 +120,10 @@ static ssize_t sysfs_kf_read(struct kernfs_open_file *of, char *buf, len = ops->show(kobj, of->kn->priv, buf); if (len < 0) return len; + if (len >= (ssize_t)PAGE_SIZE) { + printk("fill_read_buffer: %pS returned bad count\n", ops->show); + len = PAGE_SIZE - 1; + } if (pos) { if (len <= pos) return 0; diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c index 807c493ed0cd5..cc6dc6e1d84de 100644 --- a/fs/udf/balloc.c +++ b/fs/udf/balloc.c @@ -82,8 +82,9 @@ static int load_block_bitmap(struct super_block *sb, int nr_groups = bitmap->s_nr_groups; if (block_group >= nr_groups) { - udf_debug("block_group (%u) > nr_groups (%d)\n", + udf_debug("block_group (%u) >= nr_groups (%d)\n", block_group, nr_groups); + return -EFSCORRUPTED; } if (bitmap->s_block_bitmap[block_group]) { @@ -662,7 +663,7 @@ void udf_free_blocks(struct super_block *sb, struct inode *inode, if (check_add_overflow(bloc->logicalBlockNum, offset, &blk) || check_add_overflow(blk, count, &blk) || - bloc->logicalBlockNum + count > map->s_partition_len) { + blk > map->s_partition_len) { udf_debug("Invalid request to free blocks: (%d, %u), off %u, " "len %u, partition len %u\n", partition, bloc->logicalBlockNum, offset, count, diff --git a/fs/udf/super.c b/fs/udf/super.c index cad2e15d633b6..f5328c0084dca 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1263,6 +1263,14 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) map->s_type_specific.s_virtual.s_start_offset = le16_to_cpu(vat20->lengthHeader); + if (map->s_type_specific.s_virtual.s_start_offset + > sbi->s_vat_inode->i_size) { + udf_err(sb, "Corrupted VAT header length %u (VAT inode size %lld)\n", + map->s_type_specific.s_virtual.s_start_offset, + sbi->s_vat_inode->i_size); + brelse(bh); + return -EFSCORRUPTED; + } map->s_type_specific.s_virtual.s_num_entries = (sbi->s_vat_inode->i_size - map->s_type_specific.s_virtual. @@ -1418,7 +1426,8 @@ static int udf_load_sparable_map(struct super_block *sb, if (ident != 0 || strncmp(st->sparingIdent.ident, UDF_ID_SPARING, strlen(UDF_ID_SPARING)) || - sizeof(*st) + le16_to_cpu(st->reallocationTableLen) > + struct_size(st, mapEntry, + le16_to_cpu(st->reallocationTableLen)) > sb->s_blocksize) { brelse(bh); continue; @@ -2321,7 +2330,7 @@ static int udf_fill_super(struct super_block *sb, struct fs_context *fc) error_out: iput(sbi->s_vat_inode); - unload_nls(uopt->nls_map); + unload_nls(sbi->s_nls_map); if (lvid_open) udf_close_lvid(sb); brelse(sbi->s_lvid_bh); diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 4b53dc4a32664..dba1172436b74 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -253,6 +253,15 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, /* UFFD PTE markers require userspace to resolve the fault. */ if (pte_is_uffd_marker(pte)) return true; + /* + * Concurrent migration may have replaced the present PTE with a + * non-marker swap entry between fault delivery and this lockless + * re-check. huge_pte_write() on a swap entry decodes random offset + * bits, so gate it on pte_present(). The migration completion path + * will re-deliver the fault if it still needs userspace. + */ + if (!pte_present(pte)) + return false; /* * If VMA has UFFD WP faults enabled and WP fault, wait for userspace to * resolve the fault. @@ -339,6 +348,17 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, /* UFFD PTE markers require userspace to resolve the fault. */ if (pte_is_uffd_marker(ptent)) goto out; + /* + * Concurrent swap-out / migration may have replaced the present PTE + * with a non-marker swap entry between fault delivery and this + * lockless re-check. pte_write() on a swap entry decodes random + * offset bits, so gate it on pte_present(). The page-in path will + * re-deliver the fault if it still needs userspace. + */ + if (!pte_present(ptent)) { + ret = false; + goto out; + } /* * If VMA has UFFD WP faults enabled and WP fault, wait for userspace to * resolve the fault. @@ -786,6 +806,8 @@ void mremap_userfaultfd_fail(struct vm_userfaultfd_ctx *vm_ctx) if (!ctx) return; + atomic_dec(&ctx->mmap_changing); + VM_WARN_ON_ONCE(atomic_read(&ctx->mmap_changing) < 0); userfaultfd_ctx_put(ctx); } diff --git a/fs/xattr.c b/fs/xattr.c index 09ecbaaa16608..89374cd9029a7 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -28,6 +28,11 @@ #include "internal.h" +struct sx_key { + const struct list_head *parent; + const char *name; +}; + static const char * strcmp_prefix(const char *a, const char *a_prefix) { @@ -1269,23 +1274,32 @@ struct simple_xattr *simple_xattr_alloc(const void *value, size_t size) return new_xattr; } +static u32 sx_hashfn(const char *name, const struct list_head *parent, u32 seed) +{ + return jhash(name, strlen(name), jhash(&parent, sizeof(parent), seed)); +} + static u32 simple_xattr_hashfn(const void *data, u32 len, u32 seed) { - const char *name = data; - return jhash(name, strlen(name), seed); + const struct sx_key *key = data; + + return sx_hashfn(key->name, key->parent, seed); } static u32 simple_xattr_obj_hashfn(const void *obj, u32 len, u32 seed) { const struct simple_xattr *xattr = obj; - return jhash(xattr->name, strlen(xattr->name), seed); + + return sx_hashfn(xattr->name, xattr->parent, seed); } static int simple_xattr_obj_cmpfn(struct rhashtable_compare_arg *arg, const void *obj) { const struct simple_xattr *xattr = obj; - return strcmp(xattr->name, arg->key); + const struct sx_key *key = arg->key; + + return xattr->parent != key->parent || strcmp(xattr->name, key->name); } static const struct rhashtable_params simple_xattr_params = { @@ -1298,6 +1312,7 @@ static const struct rhashtable_params simple_xattr_params = { /** * simple_xattr_get - get an xattr object + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @name: the name of the xattr to retrieve * @buffer: the buffer to store the value into @@ -1311,14 +1326,19 @@ static const struct rhashtable_params simple_xattr_params = { * Return: On success the length of the xattr value is returned. On error a * negative error code is returned. */ -int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, - void *buffer, size_t size) +int simple_xattr_get(struct simple_xattr_cache *cache, struct list_head *xattrs, + const char *name, void *buffer, size_t size) { struct simple_xattr *xattr; + struct sx_key key = { .parent = xattrs, .name = name }; + struct rhashtable *ht = READ_ONCE(cache->ht); int ret = -ENODATA; + if (!ht) + return ret; + guard(rcu)(); - xattr = rhashtable_lookup(&xattrs->ht, name, simple_xattr_params); + xattr = rhashtable_lookup(ht, &key, simple_xattr_params); if (xattr) { ret = xattr->size; if (buffer) { @@ -1331,8 +1351,45 @@ int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, return ret; } +static struct rhashtable *simple_xattrs_lazy_alloc(struct simple_xattr_cache *cache, + const void *value, int flags) +{ + struct rhashtable *oldht, *ht = READ_ONCE(cache->ht); + int err; + + if (unlikely(!ht)) { + if (!value) + return (flags & XATTR_REPLACE) ? ERR_PTR(-ENODATA) : NULL; + + ht = kzalloc_obj(*ht); + if (!ht) + return ERR_PTR(-ENOMEM); + + err = rhashtable_init(ht, &simple_xattr_params); + if (err) { + kfree(ht); + return ERR_PTR(err); + } + + /* + * Provides release semantics on success, so that use of a + * non-NULL READ_ONCE(cache->ht) will be ordered relative to the + * above initialization, due to implicit address dependency. + */ + oldht = cmpxchg_release(&cache->ht, NULL, ht); + if (oldht) { + /* Race lost */ + rhashtable_destroy(ht); + kfree(ht); + ht = oldht; + } + } + return ht; +} + /** * simple_xattr_set - set an xattr object + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @name: the name of the xattr to retrieve * @value: the value to store along the xattr @@ -1362,45 +1419,58 @@ int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, * Return: On success, the removed or replaced xattr is returned, to be freed * by the caller; or NULL if none. On failure a negative error code is returned. */ -struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs, +struct simple_xattr *simple_xattr_set(struct simple_xattr_cache *cache, struct list_head *xattrs, const char *name, const void *value, size_t size, int flags) { + struct sx_key key = { .parent = xattrs, .name = name }; struct simple_xattr *old_xattr = NULL; + struct rhashtable *ht; int err; + ht = simple_xattrs_lazy_alloc(cache, value, flags); + if (IS_ERR_OR_NULL(ht)) + return ERR_CAST(ht); + CLASS(simple_xattr, new_xattr)(value, size); if (IS_ERR(new_xattr)) return new_xattr; if (new_xattr) { + new_xattr->parent = xattrs; new_xattr->name = kstrdup(name, GFP_KERNEL_ACCOUNT); if (!new_xattr->name) return ERR_PTR(-ENOMEM); } - /* Lookup is safe without RCU here since writes are serialized. */ - old_xattr = rhashtable_lookup_fast(&xattrs->ht, name, - simple_xattr_params); - + /* + * Hash table lookup/replace/remove will grab RCU read lock themselves. + * This makes sure that hash table lookup is safe against concurrent + * modification on another inode. + */ + old_xattr = rhashtable_lookup_fast(ht, &key, simple_xattr_params); if (old_xattr) { /* Fail if XATTR_CREATE is requested and the xattr exists. */ if (flags & XATTR_CREATE) return ERR_PTR(-EEXIST); if (new_xattr) { - err = rhashtable_replace_fast(&xattrs->ht, + err = rhashtable_replace_fast(ht, &old_xattr->hash_node, &new_xattr->hash_node, simple_xattr_params); if (err) return ERR_PTR(err); + + list_replace_rcu(&old_xattr->node, &new_xattr->node); } else { - err = rhashtable_remove_fast(&xattrs->ht, + err = rhashtable_remove_fast(ht, &old_xattr->hash_node, simple_xattr_params); if (err) return ERR_PTR(err); + + list_del_rcu(&old_xattr->node); } } else { /* Fail if XATTR_REPLACE is requested but no xattr is found. */ @@ -1412,11 +1482,13 @@ struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs, * new value simply insert it. */ if (new_xattr) { - err = rhashtable_insert_fast(&xattrs->ht, + err = rhashtable_insert_fast(ht, &new_xattr->hash_node, simple_xattr_params); if (err) return ERR_PTR(err); + + list_add_tail_rcu(&new_xattr->node, xattrs); } /* @@ -1453,6 +1525,7 @@ static inline int simple_xattr_limits_inc(struct simple_xattr_limits *limits, /** * simple_xattr_set_limited - set an xattr with per-inode user.* limits + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @limits: per-inode limit counters for user.* xattrs * @name: the name of the xattr to set or remove @@ -1467,7 +1540,7 @@ static inline int simple_xattr_limits_inc(struct simple_xattr_limits *limits, * Return: On success zero is returned. On failure a negative error code is * returned. */ -int simple_xattr_set_limited(struct simple_xattrs *xattrs, +int simple_xattr_set_limited(struct simple_xattr_cache *cache, struct list_head *xattrs, struct simple_xattr_limits *limits, const char *name, const void *value, size_t size, int flags) @@ -1481,7 +1554,7 @@ int simple_xattr_set_limited(struct simple_xattrs *xattrs, return ret; } - old_xattr = simple_xattr_set(xattrs, name, value, size, flags); + old_xattr = simple_xattr_set(cache, xattrs, name, value, size, flags); if (IS_ERR(old_xattr)) { if (value) simple_xattr_limits_dec(limits, size); @@ -1527,11 +1600,10 @@ static bool xattr_is_maclabel(const char *name) * Return: On success the required size or the size of the copied xattrs is * returned. On error a negative error code is returned. */ -ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, +ssize_t simple_xattr_list(struct inode *inode, struct list_head *xattrs, char *buffer, size_t size) { bool trusted = ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN); - struct rhashtable_iter iter; struct simple_xattr *xattr; ssize_t remaining_size = size; int err = 0; @@ -1555,17 +1627,8 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, if (!xattrs) return size - remaining_size; - rhashtable_walk_enter(&xattrs->ht, &iter); - rhashtable_walk_start(&iter); - - while ((xattr = rhashtable_walk_next(&iter)) != NULL) { - if (IS_ERR(xattr)) { - if (PTR_ERR(xattr) == -EAGAIN) - continue; - err = PTR_ERR(xattr); - break; - } - + rcu_read_lock(); + list_for_each_entry_rcu(xattr, xattrs, node) { /* skip "trusted." attributes for unprivileged callers */ if (!trusted && xattr_is_trusted(xattr->name)) continue; @@ -1578,15 +1641,14 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, if (err) break; } - - rhashtable_walk_stop(&iter); - rhashtable_walk_exit(&iter); + rcu_read_unlock(); return err ? err : size - remaining_size; } /** * simple_xattr_add - add xattr objects + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @new_xattr: the xattr object to add * @@ -1597,112 +1659,67 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, * Return: On success zero is returned. On failure a negative error code is * returned. */ -int simple_xattr_add(struct simple_xattrs *xattrs, +int simple_xattr_add(struct simple_xattr_cache *cache, struct list_head *xattrs, struct simple_xattr *new_xattr) { - return rhashtable_insert_fast(&xattrs->ht, &new_xattr->hash_node, - simple_xattr_params); -} - -/** - * simple_xattrs_init - initialize new xattr header - * @xattrs: header to initialize - * - * Initialize the rhashtable used to store xattr objects. - * - * Return: On success zero is returned. On failure a negative error code is - * returned. - */ -int simple_xattrs_init(struct simple_xattrs *xattrs) -{ - return rhashtable_init(&xattrs->ht, &simple_xattr_params); -} - -/** - * simple_xattrs_alloc - allocate and initialize a new xattr header - * - * Dynamically allocate a simple_xattrs header and initialize the - * underlying rhashtable. This is intended for consumers that want - * to lazily allocate xattr storage only when the first xattr is set, - * avoiding the per-inode rhashtable overhead when no xattrs are used. - * - * Return: On success a new simple_xattrs is returned. On failure an - * ERR_PTR is returned. - */ -struct simple_xattrs *simple_xattrs_alloc(void) -{ - struct simple_xattrs *xattrs __free(kfree) = NULL; - int ret; + struct rhashtable *ht; + int err; - xattrs = kzalloc(sizeof(*xattrs), GFP_KERNEL); - if (!xattrs) - return ERR_PTR(-ENOMEM); + ht = simple_xattrs_lazy_alloc(cache, new_xattr->value, 0); + if (IS_ERR(ht)) + return PTR_ERR(ht); - ret = simple_xattrs_init(xattrs); - if (ret) - return ERR_PTR(ret); + new_xattr->parent = xattrs; + err = rhashtable_insert_fast(ht, &new_xattr->hash_node, simple_xattr_params); + if (err) + return err; - return no_free_ptr(xattrs); + list_add_tail_rcu(&new_xattr->node, xattrs); + return 0; } /** - * simple_xattrs_lazy_alloc - get or allocate xattrs for a set operation - * @xattrsp: pointer to the xattrs pointer (may point to NULL) - * @value: value being set (NULL means remove) - * @flags: xattr set flags - * - * For lazily-allocated xattrs on the write path. If no xattrs exist yet - * and this is a remove operation, returns the appropriate result without - * allocating. Otherwise ensures xattrs is allocated and published with - * store-release semantics. + * simple_xattrs_free - free xattrs + * @cache: anchor for the hash table + * @xattrs: xattr header whose xattrs to destroy + * @freed_space: approximate number of bytes of memory freed from @xattrs * - * Return: On success a valid pointer to the xattrs is returned. On - * failure or early-exit an ERR_PTR or NULL is returned. Callers should - * check with IS_ERR_OR_NULL() and propagate with PTR_ERR() which - * correctly returns 0 for the NULL no-op case. + * Destroy all xattrs in @xattrs. When this is called no one can hold a + * reference to any of the xattrs anymore. */ -struct simple_xattrs *simple_xattrs_lazy_alloc(struct simple_xattrs **xattrsp, - const void *value, int flags) +void simple_xattrs_free(struct simple_xattr_cache *cache, struct list_head *xattrs, + size_t *freed_space) { - struct simple_xattrs *xattrs; - - xattrs = READ_ONCE(*xattrsp); - if (xattrs) - return xattrs; - - if (!value) - return (flags & XATTR_REPLACE) ? ERR_PTR(-ENODATA) : NULL; - - xattrs = simple_xattrs_alloc(); - if (!IS_ERR(xattrs)) - smp_store_release(xattrsp, xattrs); - return xattrs; -} + if (freed_space) + *freed_space = 0; -static void simple_xattr_ht_free(void *ptr, void *arg) -{ - struct simple_xattr *xattr = ptr; - size_t *freed_space = arg; + while (!list_empty(xattrs)) { + struct simple_xattr *xattr = list_first_entry(xattrs, typeof(*xattr), node); - if (freed_space) - *freed_space += simple_xattr_space(xattr->name, xattr->size); - simple_xattr_free(xattr); + rhashtable_remove_fast(cache->ht, &xattr->hash_node, simple_xattr_params); + list_del(&xattr->node); + if (freed_space) + *freed_space += simple_xattr_space(xattr->name, xattr->size); + /* + * Free with RCU, since the xattr might still get accessed by + * the hash compare function + */ + simple_xattr_free_rcu(xattr); + } } /** - * simple_xattrs_free - free xattrs - * @xattrs: xattr header whose xattrs to destroy - * @freed_space: approximate number of bytes of memory freed from @xattrs + * simple_xattr_cache_cleanup - free the cache + * @cache: anchor for the hash table * - * Destroy all xattrs in @xattr. When this is called no one can hold a - * reference to any of the xattrs anymore. + * Destroy the cache table, which was lazily allocated on adding the first xattr. */ -void simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space) +void simple_xattr_cache_cleanup(struct simple_xattr_cache *cache) { - might_sleep(); - - if (freed_space) - *freed_space = 0; - rhashtable_free_and_destroy(&xattrs->ht, simple_xattr_ht_free, - freed_space); + if (cache->ht) { + WARN_ON(atomic_read(&cache->ht->nelems)); + rhashtable_destroy(cache->ht); + kfree(cache->ht); + cache->ht = NULL; + } } diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c index dcd2f93b6a6c7..0c5f0548021f4 100644 --- a/fs/xfs/libxfs/xfs_ag.c +++ b/fs/xfs/libxfs/xfs_ag.c @@ -866,7 +866,7 @@ xfs_ag_shrink_space( void xfs_growfs_compute_deltas( struct xfs_mount *mp, - xfs_rfsblock_t nb, + xfs_rfsblock_t *nb, int64_t *deltap, xfs_agnumber_t *nagcountp) { @@ -874,19 +874,19 @@ xfs_growfs_compute_deltas( int64_t delta; xfs_agnumber_t nagcount; - nb_div = nb; + nb_div = *nb; nb_mod = do_div(nb_div, mp->m_sb.sb_agblocks); if (nb_mod && nb_mod >= XFS_MIN_AG_BLOCKS) nb_div++; else if (nb_mod) - nb = nb_div * mp->m_sb.sb_agblocks; + *nb = nb_div * mp->m_sb.sb_agblocks; if (nb_div > XFS_MAX_AGNUMBER + 1) { nb_div = XFS_MAX_AGNUMBER + 1; - nb = nb_div * mp->m_sb.sb_agblocks; + *nb = nb_div * mp->m_sb.sb_agblocks; } nagcount = nb_div; - delta = nb - mp->m_sb.sb_dblocks; + delta = *nb - mp->m_sb.sb_dblocks; *deltap = delta; *nagcountp = nagcount; } diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h index 16a9b43a3c270..8aa4266c55714 100644 --- a/fs/xfs/libxfs/xfs_ag.h +++ b/fs/xfs/libxfs/xfs_ag.h @@ -330,7 +330,7 @@ int xfs_ag_init_headers(struct xfs_mount *mp, struct aghdr_init_data *id); int xfs_ag_shrink_space(struct xfs_perag *pag, struct xfs_trans **tpp, xfs_extlen_t delta); void -xfs_growfs_compute_deltas(struct xfs_mount *mp, xfs_rfsblock_t nb, +xfs_growfs_compute_deltas(struct xfs_mount *mp, xfs_rfsblock_t *nb, int64_t *deltap, xfs_agnumber_t *nagcountp); int xfs_ag_extend_space(struct xfs_perag *pag, struct xfs_trans *tp, xfs_extlen_t len); diff --git a/fs/xfs/libxfs/xfs_bmap_btree.h b/fs/xfs/libxfs/xfs_bmap_btree.h index b238d559ab036..e0c870beaf670 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.h +++ b/fs/xfs/libxfs/xfs_bmap_btree.h @@ -89,7 +89,7 @@ xfs_bmbt_key_addr( { return (struct xfs_bmbt_key *) ((char *)block + xfs_bmbt_block_len(mp) + - (index - 1) * sizeof(struct xfs_bmbt_key *)); + (index - 1) * sizeof(struct xfs_bmbt_key)); } static inline xfs_bmbt_ptr_t * diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index c6909716b0415..89501e8bd2f84 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -878,7 +878,7 @@ xfs_defer_add_barrier( if (dfp) return; - xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type); + dfp = xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type); trace_xfs_defer_add_item(tp->t_mountp, dfp, NULL); } diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c index ce767b40482f8..bbada0d3cc081 100644 --- a/fs/xfs/libxfs/xfs_dquot_buf.c +++ b/fs/xfs/libxfs/xfs_dquot_buf.c @@ -436,17 +436,27 @@ xfs_dqinode_metadir_create( error = xfs_metadir_create(&upd, S_IFREG); if (error) - return error; + goto out_cancel; xfs_trans_log_inode(upd.tp, upd.ip, XFS_ILOG_CORE); error = xfs_metadir_commit(&upd); if (error) - return error; + goto out_irele; xfs_finish_inode_setup(upd.ip); *ipp = upd.ip; return 0; + +out_cancel: + xfs_metadir_cancel(&upd, error); +out_irele: + /* Have to finish setting up the inode to ensure it's deleted. */ + if (upd.ip) { + xfs_finish_inode_setup(upd.ip); + xfs_irele(upd.ip); + } + return error; } #ifndef __KERNEL__ diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 5d28f4eac527b..541e33f331672 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -711,7 +711,7 @@ xfs_exchmaps_estimate_overhead( return -ENOSPC; /* Can't actually reserve more than UINT_MAX blocks. */ - if (req->resblks > UINT_MAX) + if (resblks > UINT_MAX) return -ENOSPC; req->resblks = resblks; diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 143114718d75c..19f014e2ece6a 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -383,6 +383,14 @@ xchk_dirpath_step_up( goto out_scanlock; } + /* The handle encoded in the parent pointer must match. */ + if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) { + trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + /* We've reached the root directory; the path is ok. */ if (parent_ino == dl->root_ino) { xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_OK); @@ -411,14 +419,6 @@ xchk_dirpath_step_up( goto out_scanlock; } - /* The handle encoded in the parent pointer must match. */ - if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) { - trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr, - path->nr_steps, &dl->xname, &dl->pptr_rec); - error = -EFSCORRUPTED; - goto out_scanlock; - } - /* Parent pointer must point up to a directory. */ if (!S_ISDIR(VFS_I(dp)->i_mode)) { trace_xchk_dirpath_nondir_parent(dl->sc, dp, path->path_nr, diff --git a/fs/xfs/scrub/dqiterate.c b/fs/xfs/scrub/dqiterate.c index 10950e4bd4c3c..079dc4e691a01 100644 --- a/fs/xfs/scrub/dqiterate.c +++ b/fs/xfs/scrub/dqiterate.c @@ -205,7 +205,7 @@ xchk_dquot_iter( if (error) return error; - cursor->id = dq->q_id + 1; + cursor->id = (uint64_t)dq->q_id + 1; *dqpp = dq; return 1; } diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 9738b9ce3f2df..2ac66163263b5 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -921,7 +921,7 @@ xrep_dinode_bad_bmbt_fork( if (nrecs == 0 || xfs_bmdr_space_calc(nrecs) > dfork_size) return true; - if (level == 0 || level >= XFS_BM_MAXLEVELS(sc->mp, whichfork)) + if (level == 0 || level > XFS_BM_MAXLEVELS(sc->mp, whichfork)) return true; dmxr = xfs_bmdr_maxrecs(dfork_size, 0); @@ -1753,7 +1753,7 @@ xrep_clamp_timestamp( struct xfs_inode *ip, struct timespec64 *ts) { - ts->tv_nsec = clamp_t(long, ts->tv_nsec, 0, NSEC_PER_SEC); + ts->tv_nsec = clamp_t(long, ts->tv_nsec, 0, NSEC_PER_SEC - 1); *ts = timestamp_truncate(*ts, VFS_I(ip)); } diff --git a/fs/xfs/scrub/rgsuper.c b/fs/xfs/scrub/rgsuper.c index 482f899a518a8..2bd2c0351b35f 100644 --- a/fs/xfs/scrub/rgsuper.c +++ b/fs/xfs/scrub/rgsuper.c @@ -23,6 +23,8 @@ int xchk_setup_rgsuperblock( struct xfs_scrub *sc) { + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); return xchk_trans_alloc(sc, 0); } @@ -43,6 +45,7 @@ xchk_rgsuperblock( struct xfs_scrub *sc) { xfs_rgnumber_t rgno = sc->sm->sm_agno; + unsigned int flags; int error; /* @@ -63,7 +66,12 @@ xchk_rgsuperblock( if (!xchk_xref_process_error(sc, 0, 0, &error)) return error; - error = xchk_rtgroup_lock(sc, &sc->sr, XFS_RTGLOCK_BITMAP_SHARED); + if (xfs_has_rtrmapbt(sc->mp)) + flags = XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP; + else + flags = XFS_RTGLOCK_BITMAP_SHARED; + + error = xchk_rtgroup_lock(sc, &sc->sr, flags); if (error) return error; @@ -80,9 +88,13 @@ int xrep_rgsuperblock( struct xfs_scrub *sc) { + struct xfs_buf *sb_bp; + ASSERT(rtg_rgno(sc->sr.rtg) == 0); + sb_bp = xfs_trans_getsb(sc->tp); xfs_log_sb(sc->tp); + xfs_log_rtsb(sc->tp, sb_bp); return 0; } #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/rtbitmap.c b/fs/xfs/scrub/rtbitmap.c index b3a4972d423e2..88b438d8fd89e 100644 --- a/fs/xfs/scrub/rtbitmap.c +++ b/fs/xfs/scrub/rtbitmap.c @@ -258,7 +258,7 @@ xchk_rtbitmap( * the last free extent we saw and the last possible extent in the rt * group. */ - last_rgbno = rtg->rtg_extents * mp->m_sb.sb_rextsize - 1; + last_rgbno = rtg->rtg_extents * mp->m_sb.sb_rextsize; if (rtb->next_free_rgbno < last_rgbno) xchk_xref_has_rt_owner(sc, rtb->next_free_rgbno, last_rgbno - rtb->next_free_rgbno); diff --git a/fs/xfs/scrub/rtrefcount.c b/fs/xfs/scrub/rtrefcount.c index 4951cc2717885..5f1d5ce7c268e 100644 --- a/fs/xfs/scrub/rtrefcount.c +++ b/fs/xfs/scrub/rtrefcount.c @@ -607,7 +607,7 @@ xchk_xref_is_rt_cow_staging( /* CoW lookup returned a shared extent record? */ if (rc.rc_domain != XFS_REFC_DOMAIN_COW) - xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); /* Must be at least as long as what was passed in */ if (rc.rc_blockcount < len) diff --git a/fs/xfs/scrub/rtrmap.c b/fs/xfs/scrub/rtrmap.c index 8b1a8389d32f0..8c9247923a624 100644 --- a/fs/xfs/scrub/rtrmap.c +++ b/fs/xfs/scrub/rtrmap.c @@ -87,6 +87,9 @@ xchk_rtrmapbt_is_shareable( return false; if (irec->rm_flags & XFS_RMAP_UNWRITTEN) return false; + if (irec->rm_owner == XFS_RMAP_OWN_COW || + irec->rm_owner == XFS_RMAP_OWN_FS) + return false; return true; } @@ -146,6 +149,9 @@ xchk_rtrmap_mergeable( return false; if (r1->rm_flags != r2->rm_flags) return false; + if (r1->rm_owner == XFS_RMAP_OWN_COW || + r1->rm_owner == XFS_RMAP_OWN_FS) + return true; return r1->rm_offset + r1->rm_blockcount == r2->rm_offset; } @@ -209,7 +215,7 @@ xchk_rtrmapbt_xref( xfs_rgbno_to_rtb(sc->sr.rtg, irec->rm_startblock), irec->rm_blockcount); if (irec->rm_owner == XFS_RMAP_OWN_COW) - xchk_xref_is_cow_staging(sc, irec->rm_startblock, + xchk_xref_is_rt_cow_staging(sc, irec->rm_startblock, irec->rm_blockcount); else xchk_rtrmapbt_xref_rtrefc(sc, irec); diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index a3f1abc913903..6d7d3523b71f2 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -11,7 +11,7 @@ struct xfs_scrub; struct xchk_relax { unsigned long next_resched; unsigned int resched_nr; - bool interruptible; + bool killable; }; /* Yield to the scheduler at most 10x per second. */ @@ -21,7 +21,7 @@ struct xchk_relax { (struct xchk_relax){ \ .next_resched = XCHK_RELAX_NEXT, \ .resched_nr = 0, \ - .interruptible = true, \ + .killable = true, \ } /* @@ -45,7 +45,7 @@ static inline int xchk_maybe_relax(struct xchk_relax *widget) widget->next_resched = XCHK_RELAX_NEXT; } - if (widget->interruptible && fatal_signal_pending(current)) + if (widget->killable && fatal_signal_pending(current)) return -EINTR; return 0; diff --git a/fs/xfs/scrub/xfarray.c b/fs/xfs/scrub/xfarray.c index c7c4a71b6fa7c..2ce24bfe4c0fa 100644 --- a/fs/xfs/scrub/xfarray.c +++ b/fs/xfs/scrub/xfarray.c @@ -487,8 +487,7 @@ xfarray_sortinfo_alloc( xfarray_sortinfo_lo(si)[0] = 0; xfarray_sortinfo_hi(si)[0] = array->nr - 1; si->relax = INIT_XCHK_RELAX; - if (flags & XFARRAY_SORT_KILLABLE) - si->relax.interruptible = false; + si->relax.killable = !!(flags & XFARRAY_SORT_KILLABLE); trace_xfarray_sort(si, nr_bytes); *infop = si; diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index f279055fcea03..a93d8e2cef402 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -753,8 +753,7 @@ xfs_bio_submit_read( /* defer read completions to the ioend workqueue */ iomap_init_ioend(iter->inode, bio, ctx->read_ctx_file_offset, 0); - bio->bi_end_io = xfs_end_bio; - submit_bio(bio); + iomap_bio_submit_read_endio(iter, ctx, xfs_end_bio); } static const struct iomap_read_ops xfs_iomap_read_ops = { diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 69e9bc588c8b6..c311f61d95541 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1216,6 +1216,14 @@ xfs_qm_dqflush_check( type != XFS_DQTYPE_PROJ) return __this_address; + /* bigtime flag should never be set on root dquots */ + if (dqp->q_type & XFS_DQTYPE_BIGTIME) { + if (!xfs_has_bigtime(dqp->q_mount)) + return __this_address; + if (dqp->q_id == 0) + return __this_address; + } + if (dqp->q_id == 0) return NULL; @@ -1231,14 +1239,6 @@ xfs_qm_dqflush_check( !dqp->q_rtb.timer) return __this_address; - /* bigtime flag should never be set on root dquots */ - if (dqp->q_type & XFS_DQTYPE_BIGTIME) { - if (!xfs_has_bigtime(dqp->q_mount)) - return __this_address; - if (dqp->q_id == 0) - return __this_address; - } - return NULL; } diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 8d64d904d73c1..436857356a0a1 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -124,7 +124,7 @@ xfs_growfs_data_private( mp->m_sb.sb_rextsize); if (error) return error; - xfs_growfs_compute_deltas(mp, nb, &delta, &nagcount); + xfs_growfs_compute_deltas(mp, &nb, &delta, &nagcount); /* * Reject filesystems with a single AG because they are not diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index f20a02f49ed96..1403d568a4425 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -113,6 +113,7 @@ xfs_bmbt_to_iomap( return xfs_alert_fsblock_zero(ip, imap); } + iomap->flags = iomap_flags; if (imap->br_startblock == HOLESTARTBLOCK) { iomap->addr = IOMAP_NULL_ADDR; iomap->type = IOMAP_HOLE; @@ -143,7 +144,6 @@ xfs_bmbt_to_iomap( } iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff); iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount); - iomap->flags = iomap_flags; if (mapping_flags & IOMAP_DAX) { iomap->dax_dev = target->bt_daxdev; } else { diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 09e6678ca4878..7f60d33f5d49f 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1028,7 +1028,7 @@ xlog_verify_head( { struct xlog_rec_header *tmp_rhead; char *tmp_buffer; - xfs_daddr_t first_bad; + xfs_daddr_t first_bad = XFS_BUF_DADDR_NULL; xfs_daddr_t tmp_rhead_blk; int found; int error; @@ -1057,7 +1057,8 @@ xlog_verify_head( */ error = xlog_do_recovery_pass(log, *head_blk, tmp_rhead_blk, XLOG_RECOVER_CRCPASS, &first_bad); - if ((error == -EFSBADCRC || error == -EFSCORRUPTED) && first_bad) { + if ((error == -EFSBADCRC || error == -EFSCORRUPTED) && + first_bad != XFS_BUF_DADDR_NULL) { /* * We've hit a potential torn write. Reset the error and warn * about it. @@ -1906,6 +1907,15 @@ xlog_recover_reorder_trans( list_for_each_entry_safe(item, n, &sort_list, ri_list) { enum xlog_recover_reorder fate = XLOG_REORDER_ITEM_LIST; + /* a committed item with no regions has a NULL ri_buf[0] */ + if (!item->ri_cnt || !item->ri_buf) { + xfs_warn(log->l_mp, + "%s: committed log item has no regions", + __func__); + error = -EFSCORRUPTED; + break; + } + item->ri_ops = xlog_find_item_ops(item); if (!item->ri_ops) { xfs_warn(log->l_mp, @@ -3575,4 +3585,3 @@ xlog_recover_cancel( if (xlog_recovery_needed(log)) xlog_recover_cancel_intents(log); } - diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 7bd15d9e7fba0..02d23226611ee 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -166,10 +166,9 @@ xfs_qm_dqpurge( * does it on success. */ error = xfs_qm_dqflush(dqp, bp); - if (!error) { + if (!error) error = xfs_bwrite(bp); - xfs_buf_relse(bp); - } + xfs_buf_relse(bp); xfs_dqflock(dqp); } xfs_dquot_detach_buf(dqp); diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 251dec48f0e38..e7db02579f288 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -440,6 +440,7 @@ xfs_reflink_fill_cow_hole( struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; xfs_filblks_t resaligned; + unsigned int seq_before = READ_ONCE(ip->i_df.if_seq); unsigned int dblocks = 0, rblocks = 0; int nimaps; int error; @@ -465,6 +466,22 @@ xfs_reflink_fill_cow_hole( *lockmode = XFS_ILOCK_EXCL; + /* + * The data fork mapping may have changed while we dropped the ILOCK + * (a racing O_DIRECT writer under IOLOCK_SHARED can complete a full + * CoW cycle including xfs_reflink_end_cow(), which remaps this offset + * and drops the refcount of the old shared block). Re-read it so the + * shared-status recheck below and the caller's in-place iomap both + * operate on the current mapping rather than a stale physical block. + */ + if (seq_before != READ_ONCE(ip->i_df.if_seq)) { + nimaps = 1; + error = xfs_bmapi_read(ip, imap->br_startoff, + imap->br_blockcount, imap, &nimaps, 0); + if (error) + goto out_trans_cancel; + } + error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); if (error || !*shared) goto out_trans_cancel; @@ -511,6 +528,8 @@ xfs_reflink_fill_delalloc( bool found; do { + unsigned int seq_before = READ_ONCE(ip->i_df.if_seq); + xfs_iunlock(ip, *lockmode); *lockmode = 0; @@ -521,6 +540,23 @@ xfs_reflink_fill_delalloc( *lockmode = XFS_ILOCK_EXCL; + /* + * The data fork mapping may have changed while we dropped the + * ILOCK (a racing O_DIRECT writer under IOLOCK_SHARED can + * complete a full CoW cycle including xfs_reflink_end_cow(), + * which remaps this offset and drops the refcount of the old + * shared block). Re-read it so the shared-status recheck + * below and the caller's in-place iomap both operate on the + * current mapping rather than a stale physical block. + */ + if (seq_before != READ_ONCE(ip->i_df.if_seq)) { + nimaps = 1; + error = xfs_bmapi_read(ip, imap->br_startoff, + imap->br_blockcount, imap, &nimaps, 0); + if (error) + goto out_trans_cancel; + } + error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); if (error || !*shared) diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index 153f3c378f9fb..7a3f97686989d 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -890,8 +890,7 @@ xfs_growfs_rt_sb_fields( static int xfs_growfs_rt_zoned( - struct xfs_rtgroup *rtg, - xfs_rfsblock_t nrblocks) + struct xfs_rtgroup *rtg) { struct xfs_mount *mp = rtg_mount(rtg); struct xfs_mount *nmp; @@ -903,7 +902,8 @@ xfs_growfs_rt_zoned( * Calculate new sb and mount fields for this round. Also ensure the * rtg_extents value is uptodate as the rtbitmap code relies on it. */ - nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks, + nmp = xfs_growfs_rt_alloc_fake_mount(mp, + xfs_rtgs_to_rfsbs(mp, rtg_rgno(rtg) + 1), mp->m_sb.sb_rextsize); if (!nmp) return -ENOMEM; @@ -933,6 +933,14 @@ xfs_growfs_rt_zoned( mp->m_features |= XFS_FEAT_REALTIME; xfs_rtrmapbt_compute_maxlevels(mp); xfs_rtrefcountbt_compute_maxlevels(mp); + + /* + * Finally add the newly added zone to the freelist and add the space + * to the available counter. The order is important here: only add + * the available space after the zones, as available space guarantees + * that zones to back it are available. + */ + xfs_zone_mark_free(rtg); xfs_zoned_add_available(mp, freed_rtx); out_free: kfree(nmp); @@ -1218,7 +1226,7 @@ xfs_growfs_rtg( } if (xfs_has_zoned(mp)) { - error = xfs_growfs_rt_zoned(rtg, nrblocks); + error = xfs_growfs_rt_zoned(rtg); goto out_rele; } diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c index 5e297b75a85f6..08d8b34f467e7 100644 --- a/fs/xfs/xfs_zone_alloc.c +++ b/fs/xfs/xfs_zone_alloc.c @@ -944,6 +944,14 @@ xfs_zone_rgbno_is_valid( rtg_rgno(rtg), XFS_RTG_FREE); } +void +xfs_zone_mark_free( + struct xfs_rtgroup *rtg) +{ + xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); + atomic_inc(&rtg_mount(rtg)->m_zone_info->zi_nr_free_zones); +} + static void xfs_free_open_zones( struct xfs_zone_info *zi) @@ -1082,8 +1090,7 @@ xfs_init_zone( if (write_pointer == 0) { /* zone is empty */ - atomic_inc(&zi->zi_nr_free_zones); - xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); + xfs_zone_mark_free(rtg); iz->available += rtg_blocks(rtg); } else if (write_pointer < rtg_blocks(rtg)) { /* zone is open */ diff --git a/fs/xfs/xfs_zone_alloc.h b/fs/xfs/xfs_zone_alloc.h index 8b2ef98c81eff..abf8358bb2662 100644 --- a/fs/xfs/xfs_zone_alloc.h +++ b/fs/xfs/xfs_zone_alloc.h @@ -42,6 +42,7 @@ void xfs_zoned_wake_all(struct xfs_mount *mp); bool xfs_zone_rgbno_is_valid(struct xfs_rtgroup *rtg, xfs_rgnumber_t rgbno); void xfs_mark_rtg_boundary(struct iomap_ioend *ioend); +void xfs_zone_mark_free(struct xfs_rtgroup *rtg); uint64_t xfs_zoned_default_resblks(struct xfs_mount *mp, enum xfs_free_counter ctr); void xfs_zoned_show_stats(struct seq_file *m, struct xfs_mount *mp); diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c index f03211e4354ad..f76a091308528 100644 --- a/fs/xfs/xfs_zone_gc.c +++ b/fs/xfs/xfs_zone_gc.c @@ -924,9 +924,7 @@ xfs_zone_gc_finish_reset( goto out; } - xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); - atomic_inc(&zi->zi_nr_free_zones); - + xfs_zone_mark_free(rtg); xfs_zoned_add_available(mp, rtg_blocks(rtg)); wake_up_all(&zi->zi_zone_wait); diff --git a/include/acpi/platform/aclinuxex.h b/include/acpi/platform/aclinuxex.h index 73265650f46ba..578833c9169f4 100644 --- a/include/acpi/platform/aclinuxex.h +++ b/include/acpi/platform/aclinuxex.h @@ -134,6 +134,7 @@ static inline void acpi_os_terminate_debugger(void) /* * OSL interfaces added by Linux */ +#define acpi_ut_safe_strncpy strscpy_pad #endif /* __KERNEL__ */ diff --git a/include/asm-generic/bitops/lock.h b/include/asm-generic/bitops/lock.h index 14d4ec8c5152d..ffb73b6129e73 100644 --- a/include/asm-generic/bitops/lock.h +++ b/include/asm-generic/bitops/lock.h @@ -16,16 +16,16 @@ * It can be used to implement bit locks. */ static __always_inline int -arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p) +arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *addr) { long old; unsigned long mask = BIT_MASK(nr); - p += BIT_WORD(nr); - if (READ_ONCE(*p) & mask) + addr += BIT_WORD(nr); + if (READ_ONCE(*addr) & mask) return 1; - old = raw_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p); + old = raw_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)addr); return !!(old & mask); } @@ -38,10 +38,10 @@ arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p) * This operation is atomic and provides release barrier semantics. */ static __always_inline void -arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p) +arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *addr) { - p += BIT_WORD(nr); - raw_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p); + addr += BIT_WORD(nr); + raw_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)addr); } /** @@ -56,14 +56,14 @@ arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p) * See for example x86's implementation. */ static inline void -arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *p) +arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *addr) { unsigned long old; - p += BIT_WORD(nr); - old = READ_ONCE(*p); + addr += BIT_WORD(nr); + old = READ_ONCE(*addr); old &= ~BIT_MASK(nr); - raw_atomic_long_set_release((atomic_long_t *)p, old); + raw_atomic_long_set_release((atomic_long_t *)addr, old); } #ifndef arch_xor_unlock_is_negative_byte diff --git a/include/asm-generic/rqspinlock.h b/include/asm-generic/rqspinlock.h index 151d267a496b3..4d46643f46cbc 100644 --- a/include/asm-generic/rqspinlock.h +++ b/include/asm-generic/rqspinlock.h @@ -243,12 +243,20 @@ static __always_inline void res_spin_unlock(rqspinlock_t *lock) ({ \ int __ret; \ local_irq_save(flags); \ - __ret = raw_res_spin_lock(lock); \ - if (__ret) \ + preempt_disable(); \ + __ret = res_spin_lock(lock); \ + if (__ret) { \ local_irq_restore(flags); \ + preempt_enable(); \ + } \ __ret; \ }) -#define raw_res_spin_unlock_irqrestore(lock, flags) ({ raw_res_spin_unlock(lock); local_irq_restore(flags); }) +#define raw_res_spin_unlock_irqrestore(lock, flags) \ + ({ \ + res_spin_unlock(lock); \ + local_irq_restore(flags); \ + preempt_enable(); \ + }) #endif /* __ASM_GENERIC_RQSPINLOCK_H */ diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index b276f783494c4..4569045e7139f 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -91,8 +91,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) memcpy((void __force *)to, from, n); return 0; } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER #endif /* CONFIG_UACCESS_MEMCPY */ /* diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index 2d42518cbdce8..c11eaf757ed02 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -148,19 +148,15 @@ static inline size_t drbg_max_request_bytes(struct drbg_state *drbg) return (1 << 16); } +/* + * SP800-90A allows implementations to support additional info / personalization + * strings of up to 2**35 bits. Implementations can have a smaller maximum. We + * use 2**35 - 16 bits == U32_MAX - 1 bytes so that the max + 1 always fits in a + * size_t, allowing drbg_healthcheck_sanity() to verify its enforcement. + */ static inline size_t drbg_max_addtl(struct drbg_state *drbg) { - /* SP800-90A requires 2**35 bytes additional info str / pers str */ -#if (__BITS_PER_LONG == 32) - /* - * SP800-90A allows smaller maximum numbers to be returned -- we - * return SIZE_MAX - 1 to allow the verification of the enforcement - * of this value in drbg_healthcheck_sanity. - */ - return (SIZE_MAX - 1); -#else - return (1UL<<35); -#endif + return U32_MAX - 1; } static inline size_t drbg_max_requests(struct drbg_state *drbg) diff --git a/include/crypto/internal/drbg.h b/include/crypto/internal/drbg.h index 371e52dcee6c5..b4e5ef0be602f 100644 --- a/include/crypto/internal/drbg.h +++ b/include/crypto/internal/drbg.h @@ -9,24 +9,6 @@ #ifndef _INTERNAL_DRBG_H #define _INTERNAL_DRBG_H -/* - * Convert an integer into a byte representation of this integer. - * The byte representation is big-endian - * - * @val value to be converted - * @buf buffer holding the converted integer -- caller must ensure that - * buffer size is at least 32 bit - */ -static inline void drbg_cpu_to_be32(__u32 val, unsigned char *buf) -{ - struct s { - __be32 conv; - }; - struct s *conversion = (struct s *)buf; - - conversion->conv = cpu_to_be32(val); -} - /* * Concatenation Helper and string operation helper * diff --git a/include/dt-bindings/clock/qcom,x1e80100-camcc.h b/include/dt-bindings/clock/qcom,x1e80100-camcc.h index d72fdfb06a7c7..06c316022fb0d 100644 --- a/include/dt-bindings/clock/qcom,x1e80100-camcc.h +++ b/include/dt-bindings/clock/qcom,x1e80100-camcc.h @@ -115,6 +115,9 @@ #define CAM_CC_SLEEP_CLK_SRC 105 #define CAM_CC_SLOW_AHB_CLK_SRC 106 #define CAM_CC_XO_CLK_SRC 107 +#define CAM_CC_QDSS_DEBUG_CLK 108 +#define CAM_CC_QDSS_DEBUG_CLK_SRC 109 +#define CAM_CC_QDSS_DEBUG_XO_CLK 110 /* CAM_CC power domains */ #define CAM_CC_BPS_GDSC 0 diff --git a/include/dt-bindings/power/fsl,imx93-power.h b/include/dt-bindings/power/fsl,imx93-power.h index 17f9f015bf7df..071221fe5c57d 100644 --- a/include/dt-bindings/power/fsl,imx93-power.h +++ b/include/dt-bindings/power/fsl,imx93-power.h @@ -11,5 +11,6 @@ #define IMX93_MEDIABLK_PD_PXP 2 #define IMX93_MEDIABLK_PD_LCDIF 3 #define IMX93_MEDIABLK_PD_ISI 4 +#define IMX93_MEDIABLK_PD_MIPI_PHY 5 #endif diff --git a/include/keys/request_key_auth-type.h b/include/keys/request_key_auth-type.h index 36b89a933310f..01e42ee5f4099 100644 --- a/include/keys/request_key_auth-type.h +++ b/include/keys/request_key_auth-type.h @@ -9,12 +9,14 @@ #define _KEYS_REQUEST_KEY_AUTH_TYPE_H #include +#include /* * Authorisation record for request_key(). */ struct request_key_auth { struct rcu_head rcu; + refcount_t usage; struct key *target_key; struct key *dest_keyring; const struct cred *cred; diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index 9946276aff737..6c54d5c0d21f7 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -71,11 +71,6 @@ struct amba_device { unsigned int cid; struct amba_cs_uci_id uci; unsigned int irq[AMBA_NR_IRQS]; - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; }; struct amba_driver { diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index e7195750d21bb..4de81848fe2ee 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -90,6 +90,11 @@ ARM_SMCCC_SMC_32, \ 0, 2) +#define ARM_SMCCC_ARCH_SOC_ID64 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + 0, 2) + #define ARM_SMCCC_ARCH_WORKAROUND_1 \ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ ARM_SMCCC_SMC_32, \ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 890128cdea1ce..dda80803740fb 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1214,16 +1214,12 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async) __blk_flush_plug(plug, async); } -/* - * tsk == current here - */ -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +static __always_inline void blk_plug_invalidate_ts(void) { - struct blk_plug *plug = tsk->plug; - - if (plug) - plug->cur_ktime = 0; - current->flags &= ~PF_BLOCK_TS; + if (unlikely(current->flags & PF_BLOCK_TS)) { + current->plug->cur_ktime = 0; + current->flags &= ~PF_BLOCK_TS; + } } int blkdev_issue_flush(struct block_device *bdev); @@ -1249,7 +1245,7 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async) { } -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +static inline void blk_plug_invalidate_ts(void) { } diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index b2e79c2b41d51..4d0cc65976a14 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -421,7 +421,7 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); int cgroup_bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr); + union bpf_attr __user *uattr, u32 uattr_size); const struct bpf_func_proto * cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); @@ -452,7 +452,8 @@ static inline int cgroup_bpf_link_attach(const union bpf_attr *attr, } static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, + u32 uattr_size) { return -EINVAL; } diff --git a/include/linux/bpf.h b/include/linux/bpf.h index cd191c5fdb0a5..a2cf24d168802 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -295,6 +295,7 @@ struct bpf_map_owner { struct bpf_map { u8 sha[SHA256_DIGEST_SIZE]; + u32 excl; const struct bpf_map_ops *ops; struct bpf_map *inner_map_meta; #ifdef CONFIG_SECURITY @@ -2595,6 +2596,7 @@ bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *r void bpf_obj_free_timer(const struct btf_record *rec, void *obj); void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj); void bpf_obj_free_task_work(const struct btf_record *rec, void *obj); +void bpf_obj_cancel_fields(struct bpf_map *map, void *obj); void bpf_obj_free_fields(const struct btf_record *rec, void *obj); void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu); diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index 643809cc78c33..dda272d78f013 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -14,6 +14,8 @@ #ifdef CONFIG_BPF_LSM +extern bool bpf_lsm_initialized __ro_after_init; + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ RET bpf_lsm_##NAME(__VA_ARGS__); #include @@ -52,9 +54,12 @@ int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str, const struct bpf_dynptr *value_p, int flags); int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str); bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog); +bool bpf_lsm_hook_returns_errno(u32 btf_id); #else /* !CONFIG_BPF_LSM */ +#define bpf_lsm_initialized false + static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id) { return false; @@ -104,6 +109,11 @@ static inline bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog) { return false; } + +static inline bool bpf_lsm_hook_returns_errno(u32 btf_id) +{ + return true; +} #endif /* CONFIG_BPF_LSM */ #endif /* _LINUX_BPF_LSM_H */ diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h index b1ba97f6c9ad6..f54770f110bc8 100644 --- a/include/linux/cdx/cdx_bus.h +++ b/include/linux/cdx/cdx_bus.h @@ -137,9 +137,6 @@ struct cdx_controller { * @enabled: is this bus enabled * @msi_dev_id: MSI Device ID associated with CDX device * @num_msi: Number of MSI's supported by the device - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * @irqchip_lock: lock to synchronize irq/msi configuration * @msi_write_pending: MSI write pending for this device */ @@ -165,7 +162,6 @@ struct cdx_device { bool enabled; u32 msi_dev_id; u32 num_msi; - const char *driver_override; struct mutex irqchip_lock; bool msi_write_pending; }; diff --git a/include/linux/damon.h b/include/linux/damon.h index f2cdb7c3f5e6c..18c19d41c58fa 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -979,9 +979,13 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx) static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) { - /* {aggr,sample}_interval are unsigned long, hence could overflow */ - return min(attrs->aggr_interval / attrs->sample_interval, + unsigned long sample_interval; + unsigned long max_nr_accesses; + + sample_interval = attrs->sample_interval ? : 1; + max_nr_accesses = min(attrs->aggr_interval / sample_interval, (unsigned long)UINT_MAX); + return max_nr_accesses ? : 1; } diff --git a/include/linux/device.h b/include/linux/device.h index 9c8fde6a3d866..79a4a6549be8e 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1065,9 +1065,12 @@ static inline void device_lock_assert(struct device *dev) static inline bool dev_has_sync_state(struct device *dev) { + struct device_driver *drv; + if (!dev) return false; - if (dev->driver && dev->driver->sync_state) + drv = READ_ONCE(dev->driver); + if (drv && drv->sync_state) return true; if (dev->bus && dev->bus->sync_state) return true; diff --git a/include/linux/edac.h b/include/linux/edac.h index deba46b3ee25d..e6b4e51130e5f 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -184,6 +184,7 @@ static inline char *mc_event_error_type(const unsigned int err_type) * @MEM_DDR5: Unbuffered DDR5 RAM * @MEM_RDDR5: Registered DDR5 RAM * @MEM_LRDDR5: Load-Reduced DDR5 memory. + * @MEM_LPDDR5: Low-Power DDR5 memory. * @MEM_NVDIMM: Non-volatile RAM * @MEM_WIO2: Wide I/O 2. * @MEM_HBM2: High bandwidth Memory Gen 2. @@ -216,6 +217,7 @@ enum mem_type { MEM_DDR5, MEM_RDDR5, MEM_LRDDR5, + MEM_LPDDR5, MEM_NVDIMM, MEM_WIO2, MEM_HBM2, @@ -247,6 +249,7 @@ enum mem_type { #define MEM_FLAG_DDR5 BIT(MEM_DDR5) #define MEM_FLAG_RDDR5 BIT(MEM_RDDR5) #define MEM_FLAG_LRDDR5 BIT(MEM_LRDDR5) +#define MEM_FLAG_LPDDR5 BIT(MEM_LPDDR5) #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) #define MEM_FLAG_WIO2 BIT(MEM_WIO2) #define MEM_FLAG_HBM2 BIT(MEM_HBM2) diff --git a/include/linux/err.h b/include/linux/err.h index 8c37be0620abf..d3e38d5b3a985 100644 --- a/include/linux/err.h +++ b/include/linux/err.h @@ -36,7 +36,7 @@ * * Return: A pointer with @error encoded within its value. */ -static inline void * __must_check ERR_PTR(long error) +static __always_inline void * __must_check ERR_PTR(long error) { return (void *) error; } @@ -60,7 +60,7 @@ static inline void * __must_check ERR_PTR(long error) * @ptr: An error pointer. * Return: The error code within @ptr. */ -static inline long __must_check PTR_ERR(__force const void *ptr) +static __always_inline long __must_check PTR_ERR(__force const void *ptr) { return (long) ptr; } @@ -73,7 +73,7 @@ static inline long __must_check PTR_ERR(__force const void *ptr) * @ptr: The pointer to check. * Return: true if @ptr is an error pointer, false otherwise. */ -static inline bool __must_check IS_ERR(__force const void *ptr) +static __always_inline bool __must_check IS_ERR(__force const void *ptr) { return IS_ERR_VALUE((unsigned long)ptr); } @@ -87,7 +87,7 @@ static inline bool __must_check IS_ERR(__force const void *ptr) * * Like IS_ERR(), but also returns true for a null pointer. */ -static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) +static __always_inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) { return unlikely(!ptr) || IS_ERR_VALUE((unsigned long)ptr); } @@ -99,7 +99,7 @@ static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) * Explicitly cast an error-valued pointer to another pointer type in such a * way as to make it clear that's what's going on. */ -static inline void * __must_check ERR_CAST(__force const void *ptr) +static __always_inline void * __must_check ERR_CAST(__force const void *ptr) { /* cast away the const */ return (void *) ptr; @@ -122,7 +122,7 @@ static inline void * __must_check ERR_CAST(__force const void *ptr) * * Return: The error code within @ptr if it is an error pointer; 0 otherwise. */ -static inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr) +static __always_inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr) { if (IS_ERR(ptr)) return PTR_ERR(ptr); diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 829a59399dacf..bb2b6cd5d5070 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -107,6 +107,7 @@ enum f2fs_error { ERROR_CORRUPTED_XATTR, ERROR_INVALID_NODE_REFERENCE, ERROR_INCONSISTENT_NAT, + ERROR_INCONSISTENT_ORPHAN, ERROR_MAX, }; diff --git a/include/linux/filelock.h b/include/linux/filelock.h index 5f0a2fb314506..77e1cc4afbaa8 100644 --- a/include/linux/filelock.h +++ b/include/linux/filelock.h @@ -564,7 +564,7 @@ static inline bool is_delegated(struct delegated_inode *di) return false; } -static inline int break_lease(struct inode *inode, bool wait) +static inline int break_lease(struct inode *inode, unsigned int mode) { return 0; } diff --git a/include/linux/filter.h b/include/linux/filter.h index 88a241aac36a2..ea654453e43a3 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1291,6 +1292,15 @@ extern long bpf_jit_limit_max; typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); +/* + * Flush the indirect branch predictors before reusing JIT memory, so that + * indirect jumps into a newly written program don't reuse predictions left + * behind by an old program that occupied the same space. + */ +void bpf_arch_pred_flush(void); +DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush); +DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + void bpf_jit_fill_hole_with_zero(void *area, unsigned int size); struct bpf_binary_header * @@ -1305,7 +1315,7 @@ void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic); void bpf_prog_pack_free(void *ptr, u32 size); static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) @@ -1319,7 +1329,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, unsigned int alignment, struct bpf_binary_header **rw_hdr, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns); + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic); int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header, struct bpf_binary_header *rw_header); void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, diff --git a/include/linux/firmware.h b/include/linux/firmware.h index aae1b85ffc10e..0fa3b027f02f1 100644 --- a/include/linux/firmware.h +++ b/include/linux/firmware.h @@ -110,6 +110,9 @@ int request_firmware_nowait( struct module *module, bool uevent, const char *name, struct device *device, gfp_t gfp, void *context, void (*cont)(const struct firmware *fw, void *context)); +void request_firmware_nowait_cancel(struct device *device, void *context, + void (*cont)(const struct firmware *fw, + void *context)); int request_firmware_direct(const struct firmware **fw, const char *name, struct device *device); int request_firmware_into_buf(const struct firmware **firmware_p, @@ -157,6 +160,13 @@ static inline int request_firmware_nowait( return -EINVAL; } +static inline void request_firmware_nowait_cancel(struct device *device, + void *context, + void (*cont)(const struct firmware *fw, + void *context)) +{ +} + static inline void release_firmware(const struct firmware *fw) { } diff --git a/include/linux/firmware/samsung/exynos-acpm-protocol.h b/include/linux/firmware/samsung/exynos-acpm-protocol.h index 13f17dc4443b8..24eacc7c16d25 100644 --- a/include/linux/firmware/samsung/exynos-acpm-protocol.h +++ b/include/linux/firmware/samsung/exynos-acpm-protocol.h @@ -49,16 +49,8 @@ struct acpm_handle { struct device; -#if IS_ENABLED(CONFIG_EXYNOS_ACPM_PROTOCOL) struct acpm_handle *devm_acpm_get_by_node(struct device *dev, struct device_node *np); -#else - -static inline struct acpm_handle *devm_acpm_get_by_node(struct device *dev, - struct device_node *np) -{ - return NULL; -} -#endif +struct acpm_handle *devm_acpm_get_by_phandle(struct device *dev); #endif /* __EXYNOS_ACPM_PROTOCOL_H */ diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index d70dcd462b441..7e27b0f7bf7ed 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -542,6 +542,18 @@ enum pm_gem_config_type { GEM_CONFIG_FIXED = 2, }; +/** + * enum pm_node_status - Device node status provided by xilpm fw + * @PM_NODE_UNUSED: Device is not used + * @PM_NODE_RUNNING: Device is power-on and out of reset + * @PM_NODE_HALT: Device is power-on but in the reset state + */ +enum pm_node_status { + PM_NODE_UNUSED = 0, + PM_NODE_RUNNING = 1, + PM_NODE_HALT = 12, +}; + /** * struct zynqmp_pm_query_data - PM query data * @qid: query ID @@ -630,6 +642,8 @@ int zynqmp_pm_set_rpu_mode(u32 node_id, enum rpu_oper_mode rpu_mode); int zynqmp_pm_set_tcm_config(u32 node_id, enum rpu_tcm_comb tcm_mode); int zynqmp_pm_get_node_status(const u32 node, u32 *const status, u32 *const requirements, u32 *const usage); +int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status, + u32 *const requirements, u32 *const usage); int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value); int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config, u32 value); @@ -939,6 +953,13 @@ static inline int zynqmp_pm_get_node_status(const u32 node, u32 *const status, return -ENODEV; } +static inline int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status, + u32 *const requirements, + u32 *const usage) +{ + return -ENODEV; +} + static inline int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value) diff --git a/include/linux/fs.h b/include/linux/fs.h index 11559c513dfbb..0f5f022e6e7bc 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2281,6 +2281,7 @@ struct file_system_type { #define FS_MGTIME 64 /* FS uses multigrain timestamps */ #define FS_LBS 128 /* FS supports LBS */ #define FS_POWER_FREEZE 256 /* Always freeze on suspend/hibernate */ +#define FS_USERNS_DELEGATABLE 1024 /* Can be mounted inside userns from outside */ #define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */ int (*init_fs_context)(struct fs_context *); const struct fs_parameter_spec *parameters; @@ -2438,6 +2439,11 @@ static inline struct mnt_idmap *file_mnt_idmap(const struct file *file) return mnt_idmap(file->f_path.mnt); } +static inline bool file_owner_or_capable(const struct file *file) +{ + return inode_owner_or_capable(file_mnt_idmap(file), file_inode(file)); +} + /** * is_idmapped_mnt - check whether a mount is mapped * @mnt: the mount to check diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 31df7608737e7..4e86e6990d284 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -210,8 +210,10 @@ static inline void fwnode_init(struct fwnode_handle *fwnode, { fwnode->secondary = NULL; fwnode->ops = ops; + fwnode->dev = NULL; INIT_LIST_HEAD(&fwnode->consumers); INIT_LIST_HEAD(&fwnode->suppliers); + fwnode->flags = 0; } static inline void fwnode_set_flag(struct fwnode_handle *fwnode, @@ -251,6 +253,7 @@ int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, u8 flags); void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); +void fw_devlink_refresh_fwnode(struct fwnode_handle *fwnode); bool fw_devlink_is_strict(void); #endif diff --git a/include/linux/glob.h b/include/linux/glob.h index 861327b33e41d..91595e750936e 100644 --- a/include/linux/glob.h +++ b/include/linux/glob.h @@ -6,5 +6,6 @@ #include /* For __pure */ bool __pure glob_match(char const *pat, char const *str); +bool __pure glob_match_len(char const *pat, char const *str, size_t len); #endif /* _LINUX_GLOB_H */ diff --git a/include/linux/hfs_common.h b/include/linux/hfs_common.h index 07dfc39630ab9..45fb4c9ff9f5e 100644 --- a/include/linux/hfs_common.h +++ b/include/linux/hfs_common.h @@ -513,6 +513,7 @@ struct hfs_btree_header_rec { /* HFS+ BTree misc info */ #define HFSPLUS_TREE_HEAD 0 #define HFSPLUS_NODE_MXSZ 32768 +#define HFSPLUS_NODE_MINSZ 512 #define HFSPLUS_ATTR_TREE_NODE_SIZE 8192 #define HFSPLUS_BTREE_HDR_NODE_RECS_COUNT 3 #define HFSPLUS_BTREE_HDR_MAP_REC_INDEX 2 /* Map (bitmap) record in Header node */ diff --git a/include/linux/hid-sensor-hub.h b/include/linux/hid-sensor-hub.h index e710565531085..ab5cc8db3fbb3 100644 --- a/include/linux/hid-sensor-hub.h +++ b/include/linux/hid-sensor-hub.h @@ -43,6 +43,8 @@ struct hid_sensor_hub_attribute_info { * @attr_usage_id: Usage Id of a field, e.g. X-axis for a gyro. * @raw_size: Response size for a read request. * @raw_data: Place holder for received response. + * @index: Current write index into raw_data for multi-byte reads. + * @max_raw_size: Total buffer size for multi-byte reads; 0 for single-value reads. */ struct sensor_hub_pending { bool status; @@ -51,6 +53,8 @@ struct sensor_hub_pending { u32 attr_usage_id; int raw_size; u8 *raw_data; + u32 index; + u32 max_raw_size; }; /** @@ -183,6 +187,27 @@ int sensor_hub_input_attr_get_raw_value(struct hid_sensor_hub_device *hsdev, bool is_signed ); +/** + * sensor_hub_input_attr_read_values() - Synchronous multi-byte read request + * @hsdev: Hub device instance. + * @usage_id: Attribute usage id of parent physical device as per spec + * @attr_usage_id: Attribute usage id as per spec + * @report_id: Report id to look for + * @flag: Synchronous or asynchronous read + * @buffer_size: Size of the buffer in bytes + * @buffer: Buffer to store the read data + * + * Issues a synchronous or asynchronous read request for an input attribute, + * accumulating data into the provided buffer until it is full. + * Return: 0 on success, -ETIMEDOUT if the device did not respond, or a + * negative error code. + */ +int sensor_hub_input_attr_read_values(struct hid_sensor_hub_device *hsdev, + u32 usage_id, u32 attr_usage_id, + u32 report_id, + enum sensor_hub_read_flags flag, + u32 buffer_size, u8 *buffer); + /** * sensor_hub_set_feature() - Feature set request * @hsdev: Hub device instance. diff --git a/include/linux/host1x.h b/include/linux/host1x.h index 1f5f55917d1cd..a7a6757831362 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -143,6 +143,12 @@ static inline struct host1x_bo_mapping *to_host1x_bo_mapping(struct kref *ref) return container_of(ref, struct host1x_bo_mapping, ref); } +/** + * struct host1x_bo_ops - operations implemented by a host1x_bo provider + * + * @pin: create a DMA mapping. Implementation must not touch the bo's refcount. + * @unpin: destroy a DMA mapping. Implementation must not touch the bo's refcount. + */ struct host1x_bo_ops { struct host1x_bo *(*get)(struct host1x_bo *bo); void (*put)(struct host1x_bo *bo); @@ -181,6 +187,7 @@ struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo enum dma_data_direction dir, struct host1x_bo_cache *cache); void host1x_bo_unpin(struct host1x_bo_mapping *map); +void host1x_bo_clear_cached_mappings(struct host1x_bo *bo); static inline void *host1x_bo_mmap(struct host1x_bo *bo) { diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 734b7ef98f4d2..9de2c8d6037aa 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1272,11 +1272,6 @@ struct hv_device { u16 device_id; struct device device; - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; struct vmbus_channel *channel; struct kset *channels_kset; diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 592b646f6134b..8cdd7be505d31 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -511,15 +511,23 @@ struct i3c_master_controller_ops { * @hotjoin: true if the master support hotjoin * @rpm_allowed: true if Runtime PM allowed * @rpm_ibi_allowed: true if IBI and Hot-Join allowed while runtime suspended + * @shutting_down: set to true when master begins shutdown or unregister * @boardinfo.i3c: list of I3C boardinfo objects * @boardinfo.i2c: list of I2C boardinfo objects * @boardinfo: board-level information attached to devices connected on the bus * @bus: I3C bus exposed by this master - * @wq: workqueue which can be used by master + * @wq: freezable workqueue which can be used by master * drivers if they need to postpone operations that need to take place * in a thread context. Typical examples are Hot Join processing which * requires taking the bus lock in maintenance, which in turn, can only * be done from a sleep-able context + * @hj_work: work item used to run DAA after a Hot-Join event is detected. + * Queued to @wq by i3c_master_queue_hotjoin() + * @reg_work: work item used to register newly discovered I3C devices with + * the driver model. Queued to @wq by i3c_master_do_daa_ext() so + * that device registration is deferred out of the DAA caller's + * context (notably the resume path), and is skipped if the + * controller is shutting down * @dev_nack_retry_count: retry count when slave device nack * * A &struct i3c_master_controller has to be registered to the I3C subsystem @@ -537,12 +545,15 @@ struct i3c_master_controller { unsigned int hotjoin: 1; unsigned int rpm_allowed: 1; unsigned int rpm_ibi_allowed: 1; + bool shutting_down; struct { struct list_head i3c; struct list_head i2c; } boardinfo; struct i3c_bus bus; struct workqueue_struct *wq; + struct work_struct hj_work; + struct work_struct reg_work; unsigned int dev_nack_retry_count; }; @@ -623,6 +634,7 @@ int i3c_master_register(struct i3c_master_controller *master, void i3c_master_unregister(struct i3c_master_controller *master); int i3c_master_enable_hotjoin(struct i3c_master_controller *master); int i3c_master_disable_hotjoin(struct i3c_master_controller *master); +void i3c_master_queue_hotjoin(struct i3c_master_controller *master); /** * i3c_dev_get_master_data() - get master private data attached to an I3C diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h index a97b1d01f3acf..05503950e0f1d 100644 --- a/include/linux/ieee80211-eht.h +++ b/include/linux/ieee80211-eht.h @@ -844,7 +844,7 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) const struct ieee80211_multi_link_elem *mle = (const void *)data; u8 fixed = sizeof(*mle); u8 common = 0; - bool check_common_len = false; + u8 common_len; u16 control; if (!data || len < fixed) @@ -855,7 +855,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) { case IEEE80211_ML_CONTROL_TYPE_BASIC: common += sizeof(struct ieee80211_mle_basic_common_info); - check_common_len = true; if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) common += 1; if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) @@ -875,9 +874,9 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) common += sizeof(struct ieee80211_mle_preq_common_info); if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID) common += 1; - check_common_len = true; break; case IEEE80211_ML_CONTROL_TYPE_RECONF: + common += 1; if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR) common += ETH_ALEN; if (control & IEEE80211_MLC_RECONF_PRES_EML_CAPA) @@ -889,7 +888,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) break; case IEEE80211_ML_CONTROL_TYPE_TDLS: common += sizeof(struct ieee80211_mle_tdls_common_info); - check_common_len = true; break; case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS: common = ETH_ALEN + 1; @@ -902,11 +900,9 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) if (len < fixed + common) return false; - if (!check_common_len) - return true; + common_len = mle->variable[0]; - /* if present, common length is the first octet there */ - return mle->variable[0] >= common; + return common_len >= common && common_len <= len - fixed; } /** diff --git a/include/linux/ieee80211-s1g.h b/include/linux/ieee80211-s1g.h index 22dde4cbc1b05..3f9626ad3d97c 100644 --- a/include/linux/ieee80211-s1g.h +++ b/include/linux/ieee80211-s1g.h @@ -556,7 +556,7 @@ static inline bool ieee80211_s1g_check_tim(const struct ieee80211_tim_ie *tim, */ err = ieee80211_s1g_find_target_block(&enc_blk, &target_aid, tim->virtual_map, - (const u8 *)tim + tim_len + 2); + (const u8 *)tim + tim_len); if (err) return false; diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index dccbeb25f7014..6032eea2539a6 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -293,6 +293,11 @@ static inline void in_dev_put(struct in_device *idev) #define __in_dev_put(idev) refcount_dec(&(idev)->refcnt) #define in_dev_hold(idev) refcount_inc(&(idev)->refcnt) +static inline bool in_dev_hold_safe(struct in_device *idev) +{ + return refcount_inc_not_zero(&idev->refcnt); +} + #endif /* __KERNEL__ */ static __inline__ __be32 inet_make_mask(int logmask) diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 2c5685adf3a97..45b6888af4c6e 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -615,6 +615,8 @@ extern struct bio_set iomap_ioend_bioset; #ifdef CONFIG_BLOCK int iomap_bio_read_folio_range(const struct iomap_iter *iter, struct iomap_read_folio_ctx *ctx, size_t plen); +void iomap_bio_submit_read_endio(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io); extern const struct iomap_read_ops iomap_bio_read_ops; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e587d4ac4d331..6957144263793 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -547,6 +547,7 @@ iommu_copy_struct_from_full_user_array(void *kdst, size_t kdst_entry_size, user_array->entry_num * user_array->entry_len)) return -EFAULT; + return 0; } /* Copy item by item */ diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index e21b2f7f4159f..351a5101c8628 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -76,20 +76,25 @@ struct kernfs_iattrs; * kernfs_open_file. * * kernfs_open_files are chained at kernfs_open_node->files, which is - * protected by kernfs_global_locks.open_file_mutex[i]. + * protected by kernfs_global_locks.node_mutex[i]. * * To reduce possible contention in sysfs access, arising due to single - * locks, use an array of locks (e.g. open_file_mutex) and use kernfs_node + * locks, use an array of locks (e.g. node_mutex) and use kernfs_node * object address as hash keys to get the index of these locks. * * Hashed mutexes are safe to use here because operations using these don't * rely on global exclusion. * + * The hashed mutex array protects per-node data: the kernfs_open_node for + * open file management, and kernfs_node xattr operations (necessary because + * multiple superblocks with different namespaces can share the same + * kernfs_node, making per-inode locking insufficient). + * * In future we intend to replace other global locks with hashed ones as well. * kernfs_global_locks acts as a holder for all such hash tables. */ struct kernfs_global_locks { - struct mutex open_file_mutex[NR_KERNFS_LOCKS]; + struct mutex node_mutex[NR_KERNFS_LOCKS]; }; enum kernfs_node_type { diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 4c14aee1fb063..5366f31e95b05 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1815,6 +1815,11 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm, struct kvm_irq_ack_notifier *kian); bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args); +static inline bool is_gfn_in_memslot(const struct kvm_memory_slot *slot, gfn_t gfn) +{ + return gfn >= slot->base_gfn && gfn < slot->base_gfn + slot->npages; +} + /* * Returns a pointer to the memslot if it contains gfn. * Otherwise returns NULL. @@ -1825,7 +1830,7 @@ try_get_memslot(struct kvm_memory_slot *slot, gfn_t gfn) if (!slot) return NULL; - if (gfn >= slot->base_gfn && gfn < slot->base_gfn + slot->npages) + if (is_gfn_in_memslot(slot, gfn)) return slot; else return NULL; @@ -2596,7 +2601,8 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, struct page *page, void *opaque); -long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages, +long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, + long npages, bool may_writeback_src, kvm_gmem_populate_cb post_populate, void *opaque); #endif diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index 30c5a39ff9e9c..88722e5caf020 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -175,7 +176,7 @@ struct liveupdate_flb_ops { * @retrieved: True once the FLB's retrieve() callback has run. */ struct luo_flb_private_state { - long count; + refcount_t count; u64 data; void *obj; struct mutex lock; @@ -239,6 +240,8 @@ void liveupdate_unregister_flb(struct liveupdate_file_handler *fh, struct liveupdate_flb *flb); int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb); + int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp); #else /* CONFIG_LIVEUPDATE */ @@ -279,6 +282,10 @@ static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) +{ +} + static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) { diff --git a/include/linux/mfd/rohm-bd72720.h b/include/linux/mfd/rohm-bd72720.h index ae7343bcab064..d8ddbf232bb3e 100644 --- a/include/linux/mfd/rohm-bd72720.h +++ b/include/linux/mfd/rohm-bd72720.h @@ -21,7 +21,6 @@ enum { BD72720_BUCK8, BD72720_BUCK9, BD72720_BUCK10, - BD72720_BUCK11, BD72720_LDO1, BD72720_LDO2, BD72720_LDO3, diff --git a/include/linux/mm.h b/include/linux/mm.h index fc2acedf0b763..80fce25159302 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -496,6 +496,21 @@ enum { #else #define VM_UFFD_MINOR VM_NONE #endif + +/* + * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on + * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits + * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose + * bitmap cannot hold it. + */ +#define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) +#define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR +#define VMA_UFFD_MINOR mk_vma_flags(VMA_UFFD_MINOR_BIT) +#else +#define VMA_UFFD_MINOR EMPTY_VMA_FLAGS +#endif + #ifdef CONFIG_64BIT #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) #define VM_SEALED INIT_VM_FLAG(SEALED) @@ -1238,6 +1253,30 @@ static __always_inline void vma_flags_set_mask(vma_flags_t *flags, #define vma_flags_set(flags, ...) \ vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__)) +static __always_inline vma_flags_t __mk_vma_flags_from_masks(size_t count, + const vma_flags_t *masks) +{ + vma_flags_t flags = EMPTY_VMA_FLAGS; + size_t i; + + for (i = 0; i < count; i++) + vma_flags_set_mask(&flags, masks[i]); + return flags; +} + +/* + * Combine pre-computed vma_flags_t masks into one value, e.g.: + * + * vma_flags_t flags = mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR); + * + * Unlike mk_vma_flags(), which takes bit numbers, this takes whole masks -- + * each of which may be EMPTY_VMA_FLAGS when its feature is unavailable -- so a + * bit that does not exist on the current build is never materialised. + */ +#define mk_vma_flags_from_masks(...) \ + __mk_vma_flags_from_masks(COUNT_ARGS(__VA_ARGS__), \ + (const vma_flags_t []){__VA_ARGS__}) + /* Clear all of the to-clear flags in flags, non-atomically. */ static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 0e1e581efc5ac..4a0e83709f29e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1980,6 +1980,8 @@ enum netdev_reg_state { * @qdisc_hash: qdisc hash table * @watchdog_timeo: Represents the timeout that is used by * the watchdog (see dev_watchdog()) + * @watchdog_lock: protect watchdog_ref_held + * @watchdog_ref_held: True if the watchdog device ref is taken. * @watchdog_timer: List of timers * * @proto_down_reason: reason a netdev interface is held down @@ -2392,6 +2394,8 @@ struct net_device { /* These may be needed for future network-power-down code. */ struct timer_list watchdog_timer; int watchdog_timeo; + spinlock_t watchdog_lock; + bool watchdog_ref_held; u32 proto_down_reason; diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index afe1d8f09d89f..4b9a35dbc062d 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -33,6 +33,7 @@ enum { PG_TEARDOWN, /* page group sync for destroy */ PG_UNLOCKPAGE, /* page group sync bit in read path */ PG_UPTODATE, /* page group sync bit in read path */ + PG_READ_FAILED, /* page group saw a read error */ PG_WB_END, /* page group sync bit in write path */ PG_REMOVE, /* page group sync bit in write path */ PG_CONTENDED1, /* Is someone waiting for a lock? */ diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index fcbd21b5685f4..40417e3a7f85a 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -580,7 +580,6 @@ struct nfs_lock_args { struct nfs_lowner lock_owner; unsigned char block : 1; unsigned char reclaim : 1; - unsigned char new_lock : 1; unsigned char new_lock_owner : 1; }; diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index 61e876e255e89..f23d4b218da07 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -120,14 +120,18 @@ struct page_ext_iter { * page_ext_iter_begin() - Prepare for iterating through page extensions. * @iter: page extension iterator. * @pfn: PFN of the page we're interested in. + * @count: maximum number of page extensions to return. * * Must be called with RCU read lock taken. * * Return: NULL if no page_ext exists for this page. */ static inline struct page_ext *page_ext_iter_begin(struct page_ext_iter *iter, - unsigned long pfn) + unsigned long pfn, unsigned long count) { + if (!count) + return NULL; + iter->index = 0; iter->start_pfn = pfn; iter->page_ext = page_ext_lookup(pfn); @@ -138,19 +142,22 @@ static inline struct page_ext *page_ext_iter_begin(struct page_ext_iter *iter, /** * page_ext_iter_next() - Get next page extension * @iter: page extension iterator. + * @count: maximum number of page extensions to return. * * Must be called with RCU read lock taken. * * Return: NULL if no next page_ext exists. */ -static inline struct page_ext *page_ext_iter_next(struct page_ext_iter *iter) +static inline struct page_ext *page_ext_iter_next(struct page_ext_iter *iter, + unsigned long count) { unsigned long pfn; if (WARN_ON_ONCE(!iter->page_ext)) return NULL; - iter->index++; + if (++iter->index >= count) + return NULL; pfn = iter->start_pfn + iter->index; if (page_ext_iter_next_fast_possible(pfn)) @@ -183,9 +190,9 @@ static inline struct page_ext *page_ext_iter_get(const struct page_ext_iter *ite * IMPORTANT: must be called with RCU read lock taken. */ #define for_each_page_ext(__page, __pgcount, __page_ext, __iter) \ - for (__page_ext = page_ext_iter_begin(&__iter, page_to_pfn(__page));\ - __page_ext && __iter.index < __pgcount; \ - __page_ext = page_ext_iter_next(&__iter)) + for (__page_ext = page_ext_iter_begin(&__iter, page_to_pfn(__page), __pgcount); \ + __page_ext; \ + __page_ext = page_ext_iter_next(&__iter, __pgcount)) #else /* !CONFIG_PAGE_EXTENSION */ struct page_ext; diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 24cb42f66e4b6..1c9d40e09107d 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2732,6 +2732,9 @@ #define PCI_DEVICE_ID_INTEL_82815_MC 0x1130 #define PCI_DEVICE_ID_INTEL_82815_CGC 0x1132 #define PCI_DEVICE_ID_INTEL_SST_TNG 0x119a +#define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb +#define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 +#define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 #define PCI_DEVICE_ID_INTEL_82092AA_0 0x1221 #define PCI_DEVICE_ID_INTEL_82437 0x122d #define PCI_DEVICE_ID_INTEL_82371FB_0 0x122e @@ -3052,6 +3055,11 @@ #define PCI_DEVICE_ID_INTEL_5400_FBD1 0x4036 #define PCI_DEVICE_ID_INTEL_HDA_TGL_H 0x43c8 #define PCI_DEVICE_ID_INTEL_HDA_DG1 0x490d +#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 +#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 +#define PCI_DEVICE_ID_INTEL_QAT_402XX 0x4944 +#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 #define PCI_DEVICE_ID_INTEL_HDA_EHL_0 0x4b55 #define PCI_DEVICE_ID_INTEL_HDA_EHL_3 0x4b58 #define PCI_DEVICE_ID_INTEL_HDA_WCL 0x4d28 diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h index fc2f596a6df1b..57c11ec9dc645 100644 --- a/include/linux/rhashtable-types.h +++ b/include/linux/rhashtable-types.h @@ -136,12 +136,26 @@ struct rhashtable_iter { bool end_of_table; }; -int rhashtable_init_noprof(struct rhashtable *ht, - const struct rhashtable_params *params); +int __rhashtable_init_noprof(struct rhashtable *ht, + const struct rhashtable_params *params, + struct lock_class_key *key); +#define rhashtable_init_noprof(ht, params) \ +({ \ + static struct lock_class_key __key; \ + \ + __rhashtable_init_noprof(ht, params, &__key); \ +}) #define rhashtable_init(...) alloc_hooks(rhashtable_init_noprof(__VA_ARGS__)) -int rhltable_init_noprof(struct rhltable *hlt, - const struct rhashtable_params *params); +int __rhltable_init_noprof(struct rhltable *hlt, + const struct rhashtable_params *params, + struct lock_class_key *key); +#define rhltable_init_noprof(hlt, params) \ +({ \ + static struct lock_class_key __key; \ + \ + __rhltable_init_noprof(hlt, params, &__key); \ +}) #define rhltable_init(...) alloc_hooks(rhltable_init_noprof(__VA_ARGS__)) #endif /* _LINUX_RHASHTABLE_TYPES_H */ diff --git a/include/linux/rpmsg.h b/include/linux/rpmsg.h index 83266ce146420..2e40eb54155e0 100644 --- a/include/linux/rpmsg.h +++ b/include/linux/rpmsg.h @@ -41,9 +41,6 @@ struct rpmsg_channel_info { * rpmsg_device - device that belong to the rpmsg bus * @dev: the device struct * @id: device id (used to match between rpmsg drivers and devices) - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * @src: local address * @dst: destination address * @ept: the rpmsg endpoint of this channel @@ -53,7 +50,6 @@ struct rpmsg_channel_info { struct rpmsg_device { struct device dev; struct rpmsg_device_id id; - const char *driver_override; u32 src; u32 dst; struct rpmsg_endpoint *ept; diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 93a0ba872ebe0..69b0177da1564 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -48,7 +48,7 @@ struct shmem_inode_info { }; struct timespec64 i_crtime; /* file creation time */ struct shared_policy policy; /* NUMA memory alloc policy */ - struct simple_xattrs *xattrs; /* list of xattrs */ + struct list_head xattrs; /* list of xattrs */ pgoff_t fallocend; /* highest fallocate endindex */ unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ @@ -89,6 +89,7 @@ struct shmem_sb_info { struct list_head shrinklist; /* List of shinkable inodes */ unsigned long shrinklist_len; /* Length of shrinklist */ struct shmem_quota_limits qlimits; /* Default quota limits */ + struct simple_xattr_cache xa_cache; }; static inline struct shmem_inode_info *SHMEM_I(struct inode *inode) diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 19f4f253b4f90..937823856de59 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -4,6 +4,7 @@ #ifndef _LINUX_SKMSG_H #define _LINUX_SKMSG_H +#include #include #include #include @@ -199,11 +200,14 @@ static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src, int which, u32 size) { dst->sg.data[which] = src->sg.data[which]; + __assign_bit(which, dst->sg.copy, test_bit(which, src->sg.copy)); dst->sg.data[which].length = size; dst->sg.size += size; src->sg.size -= size; src->sg.data[which].length -= size; src->sg.data[which].offset += size; + if (!src->sg.data[which].length) + __clear_bit(which, src->sg.copy); } static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src) @@ -273,16 +277,19 @@ static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) { do { - if (copy_state) - __set_bit(i, msg->sg.copy); - else - __clear_bit(i, msg->sg.copy); + __assign_bit(i, msg->sg.copy, copy_state); sk_msg_iter_var_next(i); if (i == msg->sg.end) break; } while (1); } +static inline void sk_msg_sg_copy_assign(struct sk_msg *dst, u32 dst_i, + const struct sk_msg *src, u32 src_i) +{ + __assign_bit(dst_i, dst->sg.copy, test_bit(src_i, src->sg.copy)); +} + static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start) { sk_msg_sg_copy(msg, start, true); diff --git a/include/linux/slab.h b/include/linux/slab.h index 2b5ab488e96b0..1a69255bb87f2 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -1000,7 +1000,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); /** * kmalloc_obj - Allocate a single instance of the given type * @VAR_OR_TYPE: Variable or type to allocate. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL * on failure. @@ -1012,7 +1012,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); * kmalloc_objs - Allocate an array of the given type * @VAR_OR_TYPE: Variable or type to allocate an array of. * @COUNT: How many elements in the array. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success, * or NULL on failure. @@ -1025,7 +1025,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); * @VAR_OR_TYPE: Variable or type to allocate (with its flex array). * @FAM: The name of the flexible array member of the structure. * @COUNT: How many flexible array member elements are desired. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on * failure. If @FAM has been annotated with __counted_by(), the allocation diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h index 3e6c8e9d67aef..ba88f4d78c1b1 100644 --- a/include/linux/sockptr.h +++ b/include/linux/sockptr.h @@ -91,7 +91,7 @@ static inline int copy_struct_from_sockptr(void *dst, size_t ksize, size_t rest = max(ksize, usize) - size; if (!sockptr_is_kernel(src)) - return copy_struct_from_user(dst, ksize, src.user, size); + return copy_struct_from_user(dst, ksize, src.user, usize); if (usize < ksize) { memset(dst + size, 0, rest); diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index f8b406b0a1af5..3c2b8c355ab3a 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -190,6 +190,7 @@ int rpc_switch_client_transport(struct rpc_clnt *, const struct rpc_timeout *); void rpc_shutdown_client(struct rpc_clnt *); +void rpc_hold_client(struct rpc_clnt *); void rpc_release_client(struct rpc_clnt *); void rpc_task_release_transport(struct rpc_task *); void rpc_task_release_client(struct rpc_task *); diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index df6e08aaad570..290ec5043aa9a 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -328,6 +328,7 @@ extern int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset, unsigned int length); /* svc_rdma_transport.c */ +extern void svc_rdma_xprt_deferred_close(struct svcxprt_rdma *rdma); extern struct svc_xprt_class svc_rdma_class; #ifdef CONFIG_SUNRPC_BACKCHANNEL extern struct svc_xprt_class svc_rdma_bc_class; diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 56328601218c5..9d39b640f0674 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -84,7 +84,7 @@ * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic()) * that are used instead. Out of those, __... ones are inlined. Plain * copy_{to,from}_user() might or might not be inlined. If you want them - * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER. + * inlined, have asm/uaccess.h define INLINE_COPY_USER. * * NOTE: only copy_from_user() zero-pads the destination in case of short copy. * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything @@ -157,7 +157,7 @@ __copy_to_user(void __user *to, const void *from, unsigned long n) } /* - * Architectures that #define INLINE_COPY_TO_USER use this function + * Architectures that #define INLINE_COPY_USER use this function * directly in the normal copy_to/from_user(), the other ones go * through an extern _copy_to/from_user(), which expands the same code * here. @@ -190,10 +190,6 @@ _inline_copy_from_user(void *to, const void __user *from, unsigned long n) memset(to + (n - res), 0, res); return res; } -#ifndef INLINE_COPY_FROM_USER -extern __must_check unsigned long -_copy_from_user(void *, const void __user *, unsigned long); -#endif static inline __must_check unsigned long _inline_copy_to_user(void __user *to, const void *from, unsigned long n) @@ -207,7 +203,13 @@ _inline_copy_to_user(void __user *to, const void *from, unsigned long n) } return n; } -#ifndef INLINE_COPY_TO_USER +#ifdef INLINE_COPY_USER +# define _copy_to_user _inline_copy_to_user +# define _copy_from_user _inline_copy_from_user +#else +extern __must_check unsigned long +_copy_from_user(void *, const void __user *, unsigned long); + extern __must_check unsigned long _copy_to_user(void __user *, const void *, unsigned long); #endif @@ -217,11 +219,7 @@ copy_from_user(void *to, const void __user *from, unsigned long n) { if (!check_copy_size(to, n, false)) return n; -#ifdef INLINE_COPY_FROM_USER - return _inline_copy_from_user(to, from, n); -#else return _copy_from_user(to, from, n); -#endif } static __always_inline unsigned long __must_check @@ -229,12 +227,7 @@ copy_to_user(void __user *to, const void *from, unsigned long n) { if (!check_copy_size(from, n, true)) return n; - -#ifdef INLINE_COPY_TO_USER - return _inline_copy_to_user(to, from, n); -#else return _copy_to_user(to, from, n); -#endif } #ifndef copy_mc_to_kernel @@ -510,7 +503,7 @@ copy_struct_to_user(void __user *dst, size_t usize, const void *src, return -EFAULT; } if (ignored_trailing) - *ignored_trailing = ksize < usize && + *ignored_trailing = usize < ksize && memchr_inv(src + size, 0, rest) != NULL; /* Copy the interoperable parts of the struct. */ if (copy_to_user(dst, src, size)) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index d2920f98ab869..8d2bfae6b4e67 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -23,8 +23,8 @@ /* The set of all possible UFFD-related VM flags. */ #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) -#define __VMA_UFFD_FLAGS mk_vma_flags(VMA_UFFD_MISSING_BIT, VMA_UFFD_WP_BIT, \ - VMA_UFFD_MINOR_BIT) +#define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \ + VMA_UFFD_MINOR) /* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 89165b769e5c1..6e21302db2f0e 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -101,6 +101,9 @@ struct vfio_pci_core_device { const struct vfio_pci_device_ops *pci_ops; void __iomem *barmap[PCI_STD_NUM_BARS]; bool bar_mmap_supported[PCI_STD_NUM_BARS]; + /* Flags modified at runtime - dedicated storage unit */ + bool virq_disabled; + bool bardirty; u8 *pci_config_map; u8 *vconfig; struct perm_bits *msi_perm; @@ -115,18 +118,20 @@ struct vfio_pci_core_device { u16 msix_size; u32 msix_offset; u32 rbar[7]; + /* Flags only modified on setup/release - bitfield ok */ bool has_dyn_msix:1; bool pci_2_3:1; - bool virq_disabled:1; bool reset_works:1; bool extended_caps:1; - bool bardirty:1; bool has_vga:1; - bool needs_reset:1; bool nointx:1; bool needs_pm_restore:1; - bool pm_intx_masked:1; - bool pm_runtime_engaged:1; + bool disable_idle_d3:1; + /* Flags modified at runtime - dedicated storage unit */ + bool needs_reset; + bool pm_intx_masked; + bool pm_runtime_engaged; + bool sriov_active; struct pci_saved_state *pci_saved_state; struct pci_saved_state *pm_save; int ioeventfds_nr; diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 3bbc4cb6a6727..bf089e51970ec 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -157,11 +157,13 @@ struct virtio_admin_cmd { * @id: the device type identification (used to match it with a driver). * @config: the configuration ops for this device. * @vringh_config: configuration ops for host vrings. + * @map: the map operations for mapping virtio device memory. * @vqs: the list of virtqueues for this device. * @features: the 64 lower features supported by both driver and device. * @features_array: the full features space supported by both driver and * device. * @priv: private pointer for the driver's use. + * @vmap: the map container with transport- or device-specific metadata. * @debugfs_dir: debugfs directory entry. * @debugfs_filter_features: features to be filtered set by debugfs. */ diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 3b02c0c6b3718..d87dc7f77f4e8 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -265,7 +265,9 @@ static inline bool is_vm_area_hugepages(const void *addr) * allocated in the vmalloc layer. */ #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC - return find_vm_area(addr)->page_order > 0; + struct vm_struct *area = find_vm_area(addr); + + return area && area->page_order > 0; #else return false; #endif diff --git a/include/linux/xattr.h b/include/linux/xattr.h index 8b6601367eae8..7aaaf4f8aff5b 100644 --- a/include/linux/xattr.h +++ b/include/linux/xattr.h @@ -106,12 +106,14 @@ static inline const char *xattr_prefix(const struct xattr_handler *handler) return handler->prefix ?: handler->name; } -struct simple_xattrs { - struct rhashtable ht; +struct simple_xattr_cache { + struct rhashtable *ht; }; struct simple_xattr { struct rhash_head hash_node; + struct list_head *parent; + struct list_head node; struct rcu_head rcu; char *name; size_t size; @@ -132,40 +134,35 @@ static inline void simple_xattr_limits_init(struct simple_xattr_limits *limits) atomic_set(&limits->xattr_size, 0); } -int simple_xattrs_init(struct simple_xattrs *xattrs); -struct simple_xattrs *simple_xattrs_alloc(void); -struct simple_xattrs *simple_xattrs_lazy_alloc(struct simple_xattrs **xattrsp, - const void *value, int flags); -void simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space); +void simple_xattrs_free(struct simple_xattr_cache *cache, struct list_head *xattrs, + size_t *freed_space); size_t simple_xattr_space(const char *name, size_t size); struct simple_xattr *simple_xattr_alloc(const void *value, size_t size); void simple_xattr_free(struct simple_xattr *xattr); void simple_xattr_free_rcu(struct simple_xattr *xattr); -int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, - void *buffer, size_t size); -struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs, +int simple_xattr_get(struct simple_xattr_cache *cache, struct list_head *xattrs, + const char *name, void *buffer, size_t size); +struct simple_xattr *simple_xattr_set(struct simple_xattr_cache *cache, + struct list_head *xattrs, const char *name, const void *value, size_t size, int flags); -int simple_xattr_set_limited(struct simple_xattrs *xattrs, +int simple_xattr_set_limited(struct simple_xattr_cache *cache, + struct list_head *xattrs, struct simple_xattr_limits *limits, const char *name, const void *value, size_t size, int flags); -ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, +ssize_t simple_xattr_list(struct inode *inode, struct list_head *xattrs, char *buffer, size_t size); -int simple_xattr_add(struct simple_xattrs *xattrs, +int simple_xattr_add(struct simple_xattr_cache *cache, struct list_head *xattrs, struct simple_xattr *new_xattr); int xattr_list_one(char **buffer, ssize_t *remaining_size, const char *name); +void simple_xattr_cache_cleanup(struct simple_xattr_cache *cache); + DEFINE_CLASS(simple_xattr, struct simple_xattr *, if (!IS_ERR_OR_NULL(_T)) simple_xattr_free(_T), simple_xattr_alloc(value, size), const void *value, size_t size) -DEFINE_CLASS(simple_xattrs, - struct simple_xattrs *, - if (!IS_ERR_OR_NULL(_T)) { simple_xattrs_free(_T, NULL); kfree(_T); }, - simple_xattrs_alloc(), - void) - #endif /* _LINUX_XATTR_H */ diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 9e96776945e5f..8ced27a8229b6 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -369,8 +369,11 @@ static inline struct inet6_dev *__in6_dev_get_rtnl_net(const struct net_device * static inline struct inet6_dev *__in6_dev_stats_get(const struct net_device *dev, const struct sk_buff *skb) { - if (netif_is_l3_master(dev)) + if (netif_is_l3_master(dev)) { dev = dev_get_by_index_rcu(dev_net(dev), inet6_iif(skb)); + if (!dev) + return NULL; + } return __in6_dev_get(dev); } @@ -443,6 +446,11 @@ static inline void in6_dev_hold(struct inet6_dev *idev) refcount_inc(&idev->refcnt); } +static inline bool in6_dev_hold_safe(struct inet6_dev *idev) +{ + return refcount_inc_not_zero(&idev->refcnt); +} + /* called with rcu_read_lock held */ static inline bool ip6_ignore_linkdown(const struct net_device *dev) { diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index 572b1c620c5d6..f5cf13b443427 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -3410,8 +3410,9 @@ static inline struct hci_iso_hdr *hci_iso_hdr(const struct sk_buff *skb) #define hci_iso_flags_pack(pb, ts) ((pb & 0x03) | ((ts & 0x01) << 2)) /* ISO data length and flags pack/unpack */ -#define hci_iso_data_len_pack(h, f) ((__u16) ((h) | ((f) << 14))) -#define hci_iso_data_len(h) ((h) & 0x3fff) +#define hci_iso_data_len_pack(h, f) ((__u16) (((h) & 0x0fff) | \ + (((f) & 0x3) << 14))) +#define hci_iso_data_len(h) ((h) & 0x0fff) #define hci_iso_data_flags(h) ((h) >> 14) /* codec transport types */ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index aa600fbf9a535..aa554c34f9ec3 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -988,6 +988,7 @@ enum { HCI_CONN_AUTH_FAILURE, HCI_CONN_PER_ADV, HCI_CONN_BIG_CREATED, + HCI_CONN_CREATE, HCI_CONN_CREATE_CIS, HCI_CONN_CREATE_BIG_SYNC, HCI_CONN_BIG_SYNC, diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index e0a1f2293679a..265afda2e13bc 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -620,7 +620,8 @@ struct l2cap_chan { struct l2cap_ops { char *name; - struct l2cap_chan *(*new_connection) (struct l2cap_chan *chan); + int (*new_connection)(struct l2cap_chan *chan, + struct l2cap_chan *new_chan); int (*recv) (struct l2cap_chan * chan, struct sk_buff *skb); void (*teardown) (struct l2cap_chan *chan, int err); @@ -748,6 +749,7 @@ enum { FLAG_ECRED_CONN_REQ_SENT, FLAG_PENDING_SECURITY, FLAG_HOLD_HCI_CONN, + FLAG_DEL, }; /* Lock nesting levels for L2CAP channels. We need these because lockdep @@ -884,9 +886,10 @@ static inline __u16 __next_seq(struct l2cap_chan *chan, __u16 seq) return (seq + 1) % (chan->tx_win_max + 1); } -static inline struct l2cap_chan *l2cap_chan_no_new_connection(struct l2cap_chan *chan) +static inline int l2cap_chan_no_new_connection(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - return NULL; + return -EOPNOTSUPP; } static inline int l2cap_chan_no_recv(struct l2cap_chan *chan, struct sk_buff *skb) @@ -963,7 +966,7 @@ int l2cap_chan_send(struct l2cap_chan *chan, struct msghdr *msg, size_t len, void l2cap_chan_busy(struct l2cap_chan *chan, int busy); void l2cap_chan_rx_avail(struct l2cap_chan *chan, ssize_t rx_avail); int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator); -void l2cap_chan_set_defaults(struct l2cap_chan *chan); +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan); int l2cap_ertm_init(struct l2cap_chan *chan); void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); diff --git a/include/net/bond_options.h b/include/net/bond_options.h index e6eedf23aea1a..52b966e927938 100644 --- a/include/net/bond_options.h +++ b/include/net/bond_options.h @@ -79,6 +79,7 @@ enum { BOND_OPT_COUPLED_CONTROL, BOND_OPT_BROADCAST_NEIGH, BOND_OPT_ACTOR_PORT_PRIO, + BOND_OPT_LACP_STRICT, BOND_OPT_LAST }; diff --git a/include/net/bonding.h b/include/net/bonding.h index edd1942dcd736..2c54a36a8477b 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -129,6 +129,7 @@ struct bond_params { int peer_notif_delay; int lacp_active; int lacp_fast; + int lacp_strict; unsigned int min_links; int ad_select; char primary[IFNAMSIZ]; diff --git a/include/net/dst_metadata.h b/include/net/dst_metadata.h index 1fc2fb03ce3f9..f45d1e3163f00 100644 --- a/include/net/dst_metadata.h +++ b/include/net/dst_metadata.h @@ -164,8 +164,11 @@ static inline struct metadata_dst *tun_dst_unclone(struct sk_buff *skb) if (!new_md) return ERR_PTR(-ENOMEM); - memcpy(&new_md->u.tun_info, &md_dst->u.tun_info, - sizeof(struct ip_tunnel_info) + md_size); + /* Copy in two stages to keep the __counted_by happy. */ + new_md->u.tun_info = md_dst->u.tun_info; + memcpy(ip_tunnel_info_opts(&new_md->u.tun_info), + ip_tunnel_info_opts(&md_dst->u.tun_info), md_size); + #ifdef CONFIG_DST_CACHE /* Unclone the dst cache if there is one */ if (new_md->u.tun_info.dst_cache.cache) { diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h index 6e68e359ad18c..7dee0ae616e32 100644 --- a/include/net/fib_rules.h +++ b/include/net/fib_rules.h @@ -111,6 +111,11 @@ static inline void fib_rule_get(struct fib_rule *rule) refcount_inc(&rule->refcnt); } +static inline bool fib_rule_get_safe(struct fib_rule *rule) +{ + return refcount_inc_not_zero(&rule->refcnt); +} + static inline void fib_rule_put(struct fib_rule *rule) { if (refcount_dec_and_test(&rule->refcnt)) diff --git a/include/net/gue.h b/include/net/gue.h index dfca298bec9cc..caefd6da86939 100644 --- a/include/net/gue.h +++ b/include/net/gue.h @@ -80,7 +80,7 @@ static inline size_t guehdr_flags_len(__be16 flags) static inline size_t guehdr_priv_flags_len(__be32 flags) { - return 0; + return (flags & GUE_PFLAG_REMCSUM) ? GUE_PLEN_REMCSUM : 0; } /* Validate standard and private flags. Returns non-zero (meaning invalid) diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index 318593743b6e1..040eed88fc672 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -375,7 +375,7 @@ static inline int fib_lookup(struct net *net, struct flowi4 *flp, struct fib_result *res, unsigned int flags) { struct fib_table *tb; - int err = -ENETUNREACH; + int err = -EAGAIN; flags |= FIB_LOOKUP_NOREF; if (net->ipv4.fib_has_custom_rules) @@ -389,17 +389,16 @@ static inline int fib_lookup(struct net *net, struct flowi4 *flp, if (tb) err = fib_table_lookup(tb, flp, res, flags); - if (!err) + if (err != -EAGAIN) goto out; tb = rcu_dereference_rtnl(net->ipv4.fib_default); if (tb) err = fib_table_lookup(tb, flp, res, flags); -out: if (err == -EAGAIN) err = -ENETUNREACH; - +out: rcu_read_unlock(); return err; @@ -629,6 +628,11 @@ static inline void fib_info_hold(struct fib_info *fi) refcount_inc(&fi->fib_clntref); } +static inline bool fib_info_hold_safe(struct fib_info *fi) +{ + return refcount_inc_not_zero(&fi->fib_clntref); +} + static inline void fib_info_put(struct fib_info *fi) { if (refcount_dec_and_test(&fi->fib_clntref)) diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index e517eaaa177b0..62433e48b1f9a 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -744,7 +744,8 @@ struct ip_vs_protocol { void (*state_transition)(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd); + struct ip_vs_proto_data *pd, + unsigned int iph_len); int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h index 8f721cd4e4a7c..4111b93169d2f 100644 --- a/include/net/mana/mana.h +++ b/include/net/mana/mana.h @@ -305,6 +305,14 @@ struct mana_recv_buf_oob { void *buf_va; bool from_pool; /* allocated from a page pool */ + /* head page of the page_pool fragment; valid only when + * from_pool && frag_count > 1. + */ + struct page *pp_page; + /* Fragment offset plus rxq->headroom, passed to + * page_pool_dma_sync_for_cpu(). + */ + u32 dma_sync_offset; /* SGL of the buffer going to be sent as part of the work request. */ u32 num_sge; diff --git a/include/net/netfilter/ipv4/nf_conntrack_ipv4.h b/include/net/netfilter/ipv4/nf_conntrack_ipv4.h index b39417ad955ec..0b07d5e69c159 100644 --- a/include/net/netfilter/ipv4/nf_conntrack_ipv4.h +++ b/include/net/netfilter/ipv4/nf_conntrack_ipv4.h @@ -20,4 +20,8 @@ extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_sctp; extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_gre; #endif +#if IS_ENABLED(CONFIG_NF_CONNTRACK_PPTP) +void gre_pptp_destroy_siblings(struct nf_conn *ct); +#endif + #endif /*_NF_CONNTRACK_IPV4_H*/ diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index 80f50fd0f7ad2..c024345c9bd86 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -26,6 +26,7 @@ struct nf_conntrack_expect { possible_net_t net; /* We expect this tuple, with the following mask */ + struct nf_conntrack_tuple master_tuple; struct nf_conntrack_tuple tuple; struct nf_conntrack_tuple_mask mask; @@ -54,8 +55,8 @@ struct nf_conntrack_expect { /* The conntrack of the master connection */ struct nf_conn *master; - /* Timer function; deletes the expectation. */ - struct timer_list timeout; + /* jiffies32 when this expectation expires */ + u32 timeout; #if IS_ENABLED(CONFIG_NF_NAT) union nf_inet_addr saved_addr; @@ -69,6 +70,14 @@ struct nf_conntrack_expect { struct rcu_head rcu; }; +static inline bool nf_ct_exp_is_expired(const struct nf_conntrack_expect *exp) +{ + if (READ_ONCE(exp->flags) & NF_CT_EXPECT_DEAD) + return true; + + return (__s32)(READ_ONCE(exp->timeout) - nfct_time_stamp) <= 0; +} + static inline struct net *nf_ct_exp_net(struct nf_conntrack_expect *exp) { return read_pnet(&exp->net); @@ -130,7 +139,6 @@ static inline void nf_ct_unlink_expect(struct nf_conntrack_expect *exp) void nf_ct_remove_expectations(struct nf_conn *ct); void nf_ct_unexpect_related(struct nf_conntrack_expect *exp); -bool nf_ct_remove_expect(struct nf_conntrack_expect *exp); void nf_ct_expect_iterate_destroy(bool (*iter)(struct nf_conntrack_expect *e, void *data), void *data); void nf_ct_expect_iterate_net(struct net *net, @@ -153,5 +161,8 @@ static inline int nf_ct_expect_related(struct nf_conntrack_expect *expect, return nf_ct_expect_related_report(expect, 0, 0, flags); } +struct nf_conn_help; +void nf_ct_expectation_gc(struct nf_conn_help *master_help); + #endif /*_NF_CONNTRACK_EXPECT_H*/ diff --git a/include/net/netfilter/nf_conntrack_extend.h b/include/net/netfilter/nf_conntrack_extend.h index 0b247248b0323..fd5c4dbf72caf 100644 --- a/include/net/netfilter/nf_conntrack_extend.h +++ b/include/net/netfilter/nf_conntrack_extend.h @@ -38,7 +38,6 @@ enum nf_ct_ext_id { struct nf_ct_ext { u8 offset[NF_CT_EXT_NUM]; u8 len; - unsigned int gen_id; char data[] __aligned(8); }; @@ -52,8 +51,6 @@ static inline bool nf_ct_ext_exist(const struct nf_conn *ct, u8 id) return (ct->ext && __nf_ct_ext_exist(ct->ext, id)); } -void *__nf_ct_ext_find(const struct nf_ct_ext *ext, u8 id); - static inline void *nf_ct_ext_find(const struct nf_conn *ct, u8 id) { struct nf_ct_ext *ext = ct->ext; @@ -61,19 +58,10 @@ static inline void *nf_ct_ext_find(const struct nf_conn *ct, u8 id) if (!ext || !__nf_ct_ext_exist(ext, id)) return NULL; - if (unlikely(ext->gen_id)) - return __nf_ct_ext_find(ext, id); - return (void *)ct->ext + ct->ext->offset[id]; } /* Add this type, returns pointer to data or NULL. */ void *nf_ct_ext_add(struct nf_conn *ct, enum nf_ct_ext_id id, gfp_t gfp); -/* ext genid. if ext->id != ext_genid, extensions cannot be used - * anymore unless conntrack has CONFIRMED bit set. - */ -extern atomic_t nf_conntrack_ext_genid; -void nf_ct_ext_bump_genid(void); - #endif /* _NF_CONNTRACK_EXTEND_H */ diff --git a/include/net/netfilter/nf_conntrack_helper.h b/include/net/netfilter/nf_conntrack_helper.h index 24cf3d2d97450..295c9d2a08907 100644 --- a/include/net/netfilter/nf_conntrack_helper.h +++ b/include/net/netfilter/nf_conntrack_helper.h @@ -29,13 +29,16 @@ enum nf_ct_helper_flags { #define NF_CT_HELPER_NAME_LEN 16 +/* Must be kept in sync with the classes defined by helpers */ +#define NF_CT_MAX_EXPECT_CLASSES 4 + struct nf_conntrack_helper { struct hlist_node hnode; /* Internal use. */ char name[NF_CT_HELPER_NAME_LEN]; /* name of the module */ refcount_t refcnt; struct module *me; /* pointer to self */ - const struct nf_conntrack_expect_policy *expect_policy; + struct nf_conntrack_expect_policy expect_policy[NF_CT_MAX_EXPECT_CLASSES]; /* Tuple of things we will help (compared against server response) */ struct nf_conntrack_tuple tuple; @@ -63,9 +66,6 @@ struct nf_conntrack_helper { char nat_mod_name[NF_CT_HELPER_NAME_LEN]; }; -/* Must be kept in sync with the classes defined by helpers */ -#define NF_CT_MAX_EXPECT_CLASSES 4 - /* nf_conn feature for connections that have a helper */ struct nf_conn_help { /* Helper. if any */ @@ -103,11 +103,13 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, struct nf_conn *ct), struct module *module); -int nf_conntrack_helper_register(struct nf_conntrack_helper *); +int nf_conntrack_helper_register(struct nf_conntrack_helper *, struct nf_conntrack_helper **); +int __nf_conntrack_helper_register(struct nf_conntrack_helper *); void nf_conntrack_helper_unregister(struct nf_conntrack_helper *); -int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int); -void nf_conntrack_helpers_unregister(struct nf_conntrack_helper *, +int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int, + struct nf_conntrack_helper **); +void nf_conntrack_helpers_unregister(struct nf_conntrack_helper **, unsigned int); struct nf_conn_help *nf_ct_helper_ext_add(struct nf_conn *ct, gfp_t gfp); @@ -132,6 +134,8 @@ static inline void *nfct_help_data(const struct nf_conn *ct) struct nf_conn_help *help; help = nf_ct_ext_find(ct, NF_CT_EXT_HELPER); + if (!help) + return NULL; return (void *)help->data; } diff --git a/include/net/netfilter/nf_conntrack_timeout.h b/include/net/netfilter/nf_conntrack_timeout.h index 3a66d4abb6d68..d60aa86be0194 100644 --- a/include/net/netfilter/nf_conntrack_timeout.h +++ b/include/net/netfilter/nf_conntrack_timeout.h @@ -12,6 +12,7 @@ #define CTNL_TIMEOUT_NAME_MAX 32 struct nf_ct_timeout { + refcount_t refcnt; __u16 l3num; const struct nf_conntrack_l4proto *l4proto; struct rcu_head rcu; @@ -22,6 +23,22 @@ struct nf_conn_timeout { struct nf_ct_timeout __rcu *timeout; }; +static inline void nf_ct_timeout_put(const struct nf_conn *ct) +{ +#ifdef CONFIG_NF_CONNTRACK_TIMEOUT + struct nf_conn_timeout *timeout_ext; + struct nf_ct_timeout *timeout; + + timeout_ext = nf_ct_ext_find(ct, NF_CT_EXT_TIMEOUT); + if (!timeout_ext) + return; + + timeout = rcu_dereference(timeout_ext->timeout); + if (timeout && refcount_dec_and_test(&timeout->refcnt)) + kfree_rcu(timeout, rcu); +#endif +} + static inline unsigned int * nf_ct_timeout_data(const struct nf_conn_timeout *t) { @@ -56,8 +73,14 @@ struct nf_conn_timeout *nf_ct_timeout_ext_add(struct nf_conn *ct, #ifdef CONFIG_NF_CONNTRACK_TIMEOUT struct nf_conn_timeout *timeout_ext; + if (!timeout) + return NULL; + timeout_ext = nf_ct_ext_add(ct, NF_CT_EXT_TIMEOUT, gfp); - if (timeout_ext == NULL) + if (!timeout_ext || timeout_ext->timeout) + return NULL; + + if (!refcount_inc_not_zero(&timeout->refcnt)) return NULL; rcu_assign_pointer(timeout_ext->timeout, timeout); @@ -75,7 +98,7 @@ static inline unsigned int *nf_ct_timeout_lookup(const struct nf_conn *ct) struct nf_conn_timeout *timeout_ext; timeout_ext = nf_ct_timeout_find(ct); - if (timeout_ext) + if (timeout_ext && rcu_access_pointer(timeout_ext->timeout)) timeouts = nf_ct_timeout_data(timeout_ext); #endif return timeouts; diff --git a/include/net/netfilter/nf_dup_netdev.h b/include/net/netfilter/nf_dup_netdev.h index 609bcf422a9b3..f6b05bd80c3fe 100644 --- a/include/net/netfilter/nf_dup_netdev.h +++ b/include/net/netfilter/nf_dup_netdev.h @@ -11,15 +11,39 @@ void nf_fwd_netdev_egress(const struct nft_pktinfo *pkt, int oif); #define NF_RECURSION_LIMIT 2 -static inline u8 *nf_get_nf_dup_skb_recursion(void) -{ #ifndef CONFIG_PREEMPT_RT - return this_cpu_ptr(&softnet_data.xmit.nf_dup_skb_recursion); +static inline bool nf_dev_xmit_recursion(void) +{ + return unlikely(__this_cpu_read(softnet_data.xmit.nf_dup_skb_recursion) > + NF_RECURSION_LIMIT); +} + +static inline void nf_dev_xmit_recursion_inc(void) +{ + __this_cpu_inc(softnet_data.xmit.nf_dup_skb_recursion); +} + +static inline void nf_dev_xmit_recursion_dec(void) +{ + __this_cpu_dec(softnet_data.xmit.nf_dup_skb_recursion); +} #else - return ¤t->net_xmit.nf_dup_skb_recursion; -#endif +static inline bool nf_dev_xmit_recursion(void) +{ + return unlikely(current->net_xmit.nf_dup_skb_recursion > NF_RECURSION_LIMIT); +} + +static inline void nf_dev_xmit_recursion_inc(void) +{ + current->net_xmit.nf_dup_skb_recursion++; } +static inline void nf_dev_xmit_recursion_dec(void) +{ + current->net_xmit.nf_dup_skb_recursion--; +} +#endif + struct nft_offload_ctx; struct nft_flow_rule; diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h index 7b23b245a5a86..ce414118962f5 100644 --- a/include/net/netfilter/nf_flow_table.h +++ b/include/net/netfilter/nf_flow_table.h @@ -155,11 +155,12 @@ struct flow_offload_tuple { tun_num:2, in_vlan_ingress:2; u16 mtu; + u32 dst_cookie; + struct dst_entry *dst_cache; + union { struct { - struct dst_entry *dst_cache; u32 ifidx; - u32 dst_cookie; }; struct { u32 ifidx; @@ -357,6 +358,8 @@ static inline int nf_flow_register_bpf(void) void nf_flow_offload_add(struct nf_flowtable *flowtable, struct flow_offload *flow); +void nf_flow_offload_refresh(struct nf_flowtable *flowtable, + struct flow_offload *flow); void nf_flow_offload_del(struct nf_flowtable *flowtable, struct flow_offload *flow); void nf_flow_offload_stats(struct nf_flowtable *flowtable, diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 3978c3174cdbe..fc3e81c07364f 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -18,6 +18,7 @@ struct nf_queue_entry { unsigned int id; unsigned int hook_index; /* index in hook_entries->hook[] */ #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) + struct net_device *bridge_dev; struct net_device *physin; struct net_device *physout; #endif diff --git a/include/net/netfilter/nft_meta.h b/include/net/netfilter/nft_meta.h index f74e63290603d..6cf1d910bbf8d 100644 --- a/include/net/netfilter/nft_meta.h +++ b/include/net/netfilter/nft_meta.h @@ -40,6 +40,8 @@ void nft_meta_set_eval(const struct nft_expr *expr, void nft_meta_set_destroy(const struct nft_ctx *ctx, const struct nft_expr *expr); +int nft_meta_get_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr); int nft_meta_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr); diff --git a/include/net/rtnetlink.h b/include/net/rtnetlink.h index ec65a8cebb992..2bff41aacc987 100644 --- a/include/net/rtnetlink.h +++ b/include/net/rtnetlink.h @@ -256,6 +256,8 @@ int rtnl_configure_link(struct net_device *dev, const struct ifinfomsg *ifm, int rtnl_nla_parse_ifinfomsg(struct nlattr **tb, const struct nlattr *nla_peer, struct netlink_ext_ack *exterr); struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid); +bool rtnl_dev_link_net_capable(const struct net_device *dev, + const struct net *link_net); #define MODULE_ALIAS_RTNL_LINK(kind) MODULE_ALIAS("rtnl-link-" kind) diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 11159a50d6a14..195e22a835662 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -542,6 +542,16 @@ static inline int qdisc_qlen(const struct Qdisc *q) return q->q.qlen; } +static inline void qdisc_qlen_inc(struct Qdisc *q) +{ + WRITE_ONCE(q->q.qlen, q->q.qlen + 1); +} + +static inline void qdisc_qlen_dec(struct Qdisc *q) +{ + WRITE_ONCE(q->q.qlen, q->q.qlen - 1); +} + static inline int qdisc_qlen_sum(const struct Qdisc *q) { __u32 qlen = q->qstats.qlen; @@ -549,9 +559,9 @@ static inline int qdisc_qlen_sum(const struct Qdisc *q) if (qdisc_is_percpu_stats(q)) { for_each_possible_cpu(i) - qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; + qlen += READ_ONCE(per_cpu_ptr(q->cpu_qstats, i)->qlen); } else { - qlen += q->q.qlen; + qlen += READ_ONCE(q->q.qlen); } return qlen; @@ -1063,7 +1073,7 @@ static inline void __qdisc_enqueue_tail(struct sk_buff *skb, qh->tail = skb; qh->head = skb; } - qh->qlen++; + WRITE_ONCE(qh->qlen, qh->qlen + 1); } static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) @@ -1081,7 +1091,7 @@ static inline void __qdisc_enqueue_head(struct sk_buff *skb, if (!qh->head) qh->tail = skb; qh->head = skb; - qh->qlen++; + WRITE_ONCE(qh->qlen, qh->qlen + 1); } static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) @@ -1090,7 +1100,7 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) if (likely(skb != NULL)) { qh->head = skb->next; - qh->qlen--; + WRITE_ONCE(qh->qlen, qh->qlen - 1); if (qh->head == NULL) qh->tail = NULL; skb->next = NULL; @@ -1105,7 +1115,7 @@ static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool dir skb = __skb_dequeue(&sch->gso_skb); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); return skb; } @@ -1261,7 +1271,7 @@ static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) __skb_queue_head(&sch->gso_skb, skb); /* it's still part of the queue */ qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); } } @@ -1278,7 +1288,7 @@ static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, } else { qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } @@ -1290,7 +1300,7 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, this_cpu_add(sch->cpu_qstats->backlog, pkt_len); } else { sch->qstats.backlog += pkt_len; - sch->q.qlen++; + qdisc_qlen_inc(sch); } } @@ -1306,7 +1316,7 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) qdisc_qstats_cpu_qlen_dec(sch); } else { qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } else { skb = sch->dequeue(sch); @@ -1327,7 +1337,7 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) qh->head = NULL; qh->tail = NULL; - qh->qlen = 0; + WRITE_ONCE(qh->qlen, 0); } } diff --git a/include/net/sctp/sctp.h b/include/net/sctp/sctp.h index 58242b37b47a1..cd82b05354a38 100644 --- a/include/net/sctp/sctp.h +++ b/include/net/sctp/sctp.h @@ -111,7 +111,8 @@ int sctp_transport_lookup_process(sctp_callback_t cb, struct net *net, const union sctp_addr *paddr, void *p, int dif); int sctp_transport_traverse_process(sctp_callback_t cb, sctp_callback_t cb_done, struct net *net, int *pos, void *p); -int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), void *p); +int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), + struct net *net, int *pos, void *p); int sctp_get_sctp_info(struct sock *sk, struct sctp_association *asoc, struct sctp_info *info); diff --git a/include/net/tc_act/tc_pedit.h b/include/net/tc_act/tc_pedit.h index cb7b82f2cbc7f..97754ea0a8279 100644 --- a/include/net/tc_act/tc_pedit.h +++ b/include/net/tc_act/tc_pedit.h @@ -37,17 +37,15 @@ static inline bool is_tcf_pedit(const struct tc_action *a) return false; } -static inline int tcf_pedit_nkeys(const struct tc_action *a) +/* Must be called with act->tcfa_lock held to ensure consistency of parallel + * reads of the same action's pedit keys (e.g. flow_offload count vs fill). + * Note, this is only used for pedit offload. + */ +static inline int tcf_pedit_nkeys_locked(const struct tc_action *a) { - struct tcf_pedit_parms *parms; - int nkeys; - - rcu_read_lock(); - parms = to_pedit_parms(a); - nkeys = parms->tcfp_nkeys; - rcu_read_unlock(); - - return nkeys; + lockdep_assert_held(&a->tcfa_lock); + return rcu_dereference_protected(to_pedit(a)->parms, + lockdep_is_held(&a->tcfa_lock))->tcfp_nkeys; } static inline u32 tcf_pedit_htype(const struct tc_action *a, int index) diff --git a/include/net/tcp.h b/include/net/tcp.h index 98848db62894a..607298501e123 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -2942,6 +2942,11 @@ static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, return tcp_call_bpf(sk, op, 3, args); } +static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) +{ + tcp_sk(sk)->bpf_sock_ops_cb_flags = 0; +} + #else static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) { @@ -2959,6 +2964,10 @@ static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, return -EPERM; } +static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) +{ +} + #endif static inline u32 tcp_timeout_init(struct sock *sk) diff --git a/include/net/tcp_ao.h b/include/net/tcp_ao.h index 1e9e27d6e06ba..df655ce6987d3 100644 --- a/include/net/tcp_ao.h +++ b/include/net/tcp_ao.h @@ -130,6 +130,7 @@ struct tcp_ao_info { u32 snd_sne; u32 rcv_sne; refcount_t refcnt; /* Protects twsk destruction */ + struct rcu_head rcu; }; #ifdef CONFIG_TCP_MD5SIG diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index 47c23d4a17408..49324e28ec275 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -94,7 +94,7 @@ struct udp_tunnel_sock_cfg { }; /* Setup the given (UDP) sock to receive UDP encapsulated packets */ -void setup_udp_tunnel_sock(struct net *net, struct socket *sock, +void setup_udp_tunnel_sock(struct net *net, struct sock *sk, struct udp_tunnel_sock_cfg *sock_cfg); /* -- List of parsable UDP tunnel types -- @@ -176,7 +176,7 @@ static inline void udp_tunnel_set_inner_protocol(struct sk_buff *skb, skb_set_inner_protocol(skb, inner_proto); } -void udp_tunnel_sock_release(struct socket *sock); +void udp_tunnel_sock_release(struct sock *sk); struct rtable *udp_tunnel_dst_lookup(struct sk_buff *skb, struct net_device *dev, diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 874409127e292..f8c909b0f0c34 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -943,6 +943,9 @@ static inline bool addr_match(const void *token1, const void *token2, unsigned int pdw; unsigned int pbi; + if (prefixlen > 128) + return false; + pdw = prefixlen >> 5; /* num of whole u32 in prefix */ pbi = prefixlen & 0x1f; /* num of bits in incomplete u32 in prefix */ @@ -967,6 +970,10 @@ static inline bool addr4_match(__be32 a1, __be32 a2, u8 prefixlen) /* C99 6.5.7 (3): u32 << 32 is undefined behaviour */ if (sizeof(long) == 4 && prefixlen == 0) return true; + + if (prefixlen > 32) + return false; + return !((a1 ^ a2) & htonl(~0UL << (32 - prefixlen))); } @@ -1250,8 +1257,8 @@ int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb, static inline bool __xfrm_check_nopolicy(struct net *net, struct sk_buff *skb, int dir) { - if (!net->xfrm.policy_count[dir] && !secpath_exists(skb)) - return net->xfrm.policy_default[dir] == XFRM_USERPOLICY_ACCEPT; + if (!READ_ONCE(net->xfrm.policy_count[dir]) && !secpath_exists(skb)) + return READ_ONCE(net->xfrm.policy_default[dir]) == XFRM_USERPOLICY_ACCEPT; return false; } @@ -1351,8 +1358,8 @@ static inline int xfrm_route_forward(struct sk_buff *skb, unsigned short family) { struct net *net = dev_net(skb->dev); - if (!net->xfrm.policy_count[XFRM_POLICY_OUT] && - net->xfrm.policy_default[XFRM_POLICY_OUT] == XFRM_USERPOLICY_ACCEPT) + if (!READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT]) && + READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT]) == XFRM_USERPOLICY_ACCEPT) return true; return (skb_dst(skb)->flags & DST_NOXFRM) || diff --git a/include/rdma/frmr_pools.h b/include/rdma/frmr_pools.h index af1b88801fa46..aed4d69d3841c 100644 --- a/include/rdma/frmr_pools.h +++ b/include/rdma/frmr_pools.h @@ -34,6 +34,7 @@ int ib_frmr_pools_init(struct ib_device *device, const struct ib_frmr_pool_ops *pool_ops); void ib_frmr_pools_cleanup(struct ib_device *device); int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr); -int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr); +void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr); +void ib_frmr_pool_drop(struct ib_mr *mr); #endif /* FRMR_POOLS_H */ diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 49172098a8de1..3e0c2c356d142 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -185,6 +185,10 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, { return 0; } +static inline bool ib_umem_is_contiguous(struct ib_umem *umem) +{ + return false; +} static inline struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, unsigned long offset, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 9dd76f489a0ba..46568a5221f40 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1977,6 +1977,11 @@ struct ib_dmah { struct ib_mr { struct ib_device *device; + /* + * Due to IB_MR_REREG_PD pd is not a fixed pointer and can change. For a + * user MR, this value should only be read from a system call that holds + * the uobject lock, or the driver should disable in-place REREG_PD. + */ struct ib_pd *pd; u32 lkey; u32 rkey; diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h index 7d8de561f71b5..7ffc83262a016 100644 --- a/include/rdma/rdma_vt.h +++ b/include/rdma/rdma_vt.h @@ -438,26 +438,6 @@ struct rvt_dev_info { struct rvt_wss *wss; }; -/** - * rvt_set_ibdev_name - Craft an IB device name from client info - * @rdi: pointer to the client rvt_dev_info structure - * @name: client specific name - * @unit: client specific unit number. - */ -static inline void rvt_set_ibdev_name(struct rvt_dev_info *rdi, - const char *fmt, const char *name, - const int unit) -{ - /* - * FIXME: rvt and its users want to touch the ibdev before - * registration and have things like the name work. We don't have the - * infrastructure in the core to support this directly today, hack it - * to work by setting the name manually here. - */ - dev_set_name(&rdi->ibdev.dev, fmt, name, unit); - strscpy(rdi->ibdev.name, dev_name(&rdi->ibdev.dev), IB_DEVICE_NAME_MAX); -} - /** * rvt_get_ibdev_name - return the IB name * @rdi: rdmavt device diff --git a/include/soc/spacemit/k3-syscon.h b/include/soc/spacemit/k3-syscon.h index 0299bea065a05..a68255dd641f3 100644 --- a/include/soc/spacemit/k3-syscon.h +++ b/include/soc/spacemit/k3-syscon.h @@ -168,8 +168,8 @@ #define APMU_CPU_C2_CLK_CTRL 0x394 #define APMU_CPU_C3_CLK_CTRL 0x208 #define APMU_PCIE_CLK_RES_CTRL_A 0x1f0 -#define APMU_PCIE_CLK_RES_CTRL_B 0x1c8 -#define APMU_PCIE_CLK_RES_CTRL_C 0x1d0 +#define APMU_PCIE_CLK_RES_CTRL_B 0x1d0 +#define APMU_PCIE_CLK_RES_CTRL_C 0x1c8 #define APMU_PCIE_CLK_RES_CTRL_D 0x1e0 #define APMU_PCIE_CLK_RES_CTRL_E 0x1e8 #define APMU_EMAC0_CLK_RES_CTRL 0x3e4 diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h index 24581080e26a0..1a1fe7a904c36 100644 --- a/include/sound/hda_codec.h +++ b/include/sound/hda_codec.h @@ -259,6 +259,7 @@ struct hda_codec { unsigned int forced_resume:1; /* forced resume for jack */ unsigned int no_stream_clean_at_suspend:1; /* do not clean streams at suspend */ unsigned int ctl_dev_id:1; /* old control element id build behaviour */ + unsigned int eld_jack_detect:1; /* Machine jack-detection by ELD */ unsigned long power_on_acct; unsigned long power_off_acct; diff --git a/include/sound/soc-dapm.h b/include/sound/soc-dapm.h index 4f8fb7622a139..c1e4f467efda7 100644 --- a/include/sound/soc-dapm.h +++ b/include/sound/soc-dapm.h @@ -685,6 +685,7 @@ int snd_soc_dapm_sync_unlocked(struct snd_soc_dapm_context *dapm); int snd_soc_dapm_force_enable_pin(struct snd_soc_dapm_context *dapm, const char *pin); int snd_soc_dapm_force_enable_pin_unlocked(struct snd_soc_dapm_context *dapm, const char *pin); int snd_soc_dapm_ignore_suspend(struct snd_soc_dapm_context *dapm, const char *pin); +bool snd_soc_dapm_pin_has_prefix(struct snd_soc_card *card, const char *pin); void snd_soc_dapm_mark_endpoints_dirty(struct snd_soc_card *card); /* dapm path query */ diff --git a/include/trace/events/dma_fence.h b/include/trace/events/dma_fence.h index 3abba45c0601a..5b10a9e06fb4e 100644 --- a/include/trace/events/dma_fence.h +++ b/include/trace/events/dma_fence.h @@ -9,12 +9,40 @@ struct dma_fence; +DECLARE_EVENT_CLASS(dma_fence, + + TP_PROTO(struct dma_fence *fence), + + TP_ARGS(fence), + + TP_STRUCT__entry( + __string(driver, dma_fence_driver_name(fence)) + __string(timeline, dma_fence_timeline_name(fence)) + __field(unsigned int, context) + __field(unsigned int, seqno) + ), + + TP_fast_assign( + __assign_str(driver); + __assign_str(timeline); + __entry->context = fence->context; + __entry->seqno = fence->seqno; + ), + + TP_printk("driver=%s timeline=%s context=%u seqno=%u", + __get_str(driver), __get_str(timeline), __entry->context, + __entry->seqno) +); + /* * Safe only for call sites which are guaranteed to not race with fence - * signaling,holding the fence->lock and having checked for not signaled, or the - * signaling path itself. + * signaling, holding the fence->lock and having checked for not signaled, or + * the signaling path itself. + * + * TODO: Remove the need for this event class when drivers switch to independent + * fences. */ -DECLARE_EVENT_CLASS(dma_fence, +DECLARE_EVENT_CLASS(dma_fence_ops, TP_PROTO(struct dma_fence *fence), @@ -46,7 +74,7 @@ DEFINE_EVENT(dma_fence, dma_fence_emit, TP_ARGS(fence) ); -DEFINE_EVENT(dma_fence, dma_fence_init, +DEFINE_EVENT(dma_fence_ops, dma_fence_init, TP_PROTO(struct dma_fence *fence), @@ -60,14 +88,14 @@ DEFINE_EVENT(dma_fence, dma_fence_destroy, TP_ARGS(fence) ); -DEFINE_EVENT(dma_fence, dma_fence_enable_signal, +DEFINE_EVENT(dma_fence_ops, dma_fence_enable_signal, TP_PROTO(struct dma_fence *fence), TP_ARGS(fence) ); -DEFINE_EVENT(dma_fence, dma_fence_signaled, +DEFINE_EVENT(dma_fence_ops, dma_fence_signaled, TP_PROTO(struct dma_fence *fence), diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 552bc5d9afbd1..daf7a1d3c1857 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -6698,6 +6698,7 @@ struct bpf_prog_info { __u32 verified_insns; __u32 attach_btf_obj_id; __u32 attach_btf_id; + __u32 :32; } __attribute__((aligned(8))); struct bpf_map_info { @@ -6719,6 +6720,7 @@ struct bpf_map_info { __u64 map_extra; __aligned_u64 hash; __u32 hash_size; + __u32 :32; } __attribute__((aligned(8))); struct bpf_btf_info { diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index 79ce4bc24cba6..9ef5784e78e83 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -1584,6 +1584,7 @@ enum { IFLA_BOND_NS_IP6_TARGET, IFLA_BOND_COUPLED_CONTROL, IFLA_BOND_BROADCAST_NEIGH, + IFLA_BOND_LACP_STRICT, __IFLA_BOND_MAX, }; diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index e998dfbd69603..0425d452d41ed 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -224,13 +224,17 @@ struct iommu_ioas_map { * @size: sizeof(struct iommu_ioas_map_file) * @flags: same as for iommu_ioas_map * @ioas_id: same as for iommu_ioas_map - * @fd: the memfd to map - * @start: byte offset from start of file to map from + * @fd: the memfd or supported dma-buf file to map + * @start: byte offset from start of the file to map from * @length: same as for iommu_ioas_map * @iova: same as for iommu_ioas_map * - * Set an IOVA mapping from a memfd file. All other arguments and semantics - * match those of IOMMU_IOAS_MAP. + * Set an IOVA mapping from a memfd file. On kernels with dma-buf support, + * supported dma-buf files may also be accepted. This is not a generic + * dma-buf import path; currently supported dma-bufs include single-range + * VFIO PCI dma-bufs exported through VFIO_DEVICE_FEATURE_DMA_BUF, and + * other dma-bufs may be rejected. All other arguments and semantics match + * those of IOMMU_IOAS_MAP. */ struct iommu_ioas_map_file { __u32 size; diff --git a/include/uapi/linux/netfilter/nf_conntrack_common.h b/include/uapi/linux/netfilter/nf_conntrack_common.h index 56b6b60a814f5..ee51045ae1d62 100644 --- a/include/uapi/linux/netfilter/nf_conntrack_common.h +++ b/include/uapi/linux/netfilter/nf_conntrack_common.h @@ -160,6 +160,7 @@ enum ip_conntrack_expect_events { #define NF_CT_EXPECT_USERSPACE 0x4 #ifdef __KERNEL__ +#define NF_CT_EXPECT_DEAD 0x8 #define NF_CT_EXPECT_MASK (NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE | \ NF_CT_EXPECT_USERSPACE) #endif diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index 40955eaba32e6..a4599d7b736aa 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -126,7 +126,7 @@ struct bnxt_re_resize_cq_req { }; enum bnxt_re_qp_mask { - BNXT_RE_QP_REQ_MASK_VAR_WQE_SQ_SLOTS = 0x1, + BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR = 0x1, }; struct bnxt_re_qp_req { @@ -135,6 +135,11 @@ struct bnxt_re_qp_req { __aligned_u64 qp_handle; __aligned_u64 comp_mask; __u32 sq_slots; + __u32 sq_npsn; +}; + +enum bnxt_re_create_qp_attrs { + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE = UVERBS_ID_DRIVER_NS_WITH_UHW, }; struct bnxt_re_qp_resp { diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index cfbc75d8df836..8280a95c00c74 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1020,8 +1020,6 @@ enum ufshcd_mcq_opr { * @caps: bitmask with information about UFS controller capabilities * @devfreq: frequency scaling information owned by the devfreq core * @clk_scaling: frequency scaling information owned by the UFS driver - * @system_suspending: system suspend has been started and system resume has - * not yet finished. * @is_sys_suspended: UFS device has been suspended because of system suspend * @urgent_bkops_lvl: keeps track of urgent bkops level for device * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for @@ -1197,7 +1195,6 @@ struct ufs_hba { struct devfreq *devfreq; struct ufs_clk_scaling clk_scaling; - bool system_suspending; bool is_sys_suspended; enum bkops_status urgent_bkops_lvl; diff --git a/init/Kconfig b/init/Kconfig index 2937c4d308aec..826a7d768ca36 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2198,6 +2198,7 @@ config RUST depends on !DEBUG_INFO_BTF || (PAHOLE_HAS_LANG_EXCLUDE && !LTO) depends on !CFI || HAVE_CFI_ICALL_NORMALIZE_INTEGERS_RUSTC select CFI_ICALL_NORMALIZE_INTEGERS if CFI + depends on !KASAN || CC_IS_CLANG depends on !KASAN_SW_TAGS help Enables Rust support in the kernel. diff --git a/init/initramfs_test.c b/init/initramfs_test.c index 2ce38d9a8fd0f..a4c06ec343689 100644 --- a/init/initramfs_test.c +++ b/init/initramfs_test.c @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include "initramfs_internal.h" @@ -510,8 +512,21 @@ static struct kunit_case __refdata initramfs_test_cases[] = { {}, }; -static struct kunit_suite initramfs_test_suite = { +static int __init initramfs_test_init(struct kunit_suite *suite) +{ + /* + * unpack_to_rootfs() uses module-static state (victim, byte_count, + * state, ...). The boot-time async do_populate_rootfs() may still be + * running, so wait for it to finish before we call unpack_to_rootfs() + * from the test thread, otherwise the two writers race and crash. + */ + wait_for_initramfs(); + return 0; +} + +static struct kunit_suite __refdata initramfs_test_suite = { .name = "initramfs", + .suite_init = initramfs_test_init, .test_cases = initramfs_test_cases, }; kunit_test_init_section_suites(&initramfs_test_suite); diff --git a/io_uring/bpf-ops.c b/io_uring/bpf-ops.c index 937e48bef40bc..bc27584609236 100644 --- a/io_uring/bpf-ops.c +++ b/io_uring/bpf-ops.c @@ -165,6 +165,8 @@ static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops) if (ctx->bpf_ops) return -EBUSY; + if (ops->priv) + return -EBUSY; if (WARN_ON_ONCE(!ops->loop_step)) return -EINVAL; diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 8cc7b47d30894..2e14880eef92a 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -602,7 +602,6 @@ static void io_worker_handle_work(struct io_wq_acct *acct, struct io_wq *wq = worker->wq; do { - bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); struct io_wq_work *work; /* @@ -638,6 +637,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct, /* handle a whole dependent link */ do { + bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); struct io_wq_work *next_hashed, *linked; unsigned int work_flags = atomic_read(&work->flags); unsigned int hash = __io_wq_is_hashed(work_flags) diff --git a/io_uring/net.c b/io_uring/net.c index ee848eb65ec99..bf60cd393b11a 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1733,7 +1733,7 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags) int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); - struct io_async_msghdr *io; + struct sockaddr_storage *addr; if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) return -EINVAL; @@ -1742,17 +1742,17 @@ int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) conn->addr_len = READ_ONCE(sqe->addr2); conn->in_progress = conn->seen_econnaborted = false; - io = io_msg_alloc_async(req); - if (unlikely(!io)) + addr = io_uring_alloc_async_data(NULL, req); + if (unlikely(!addr)) return -ENOMEM; - return move_addr_to_kernel(conn->addr, conn->addr_len, &io->addr); + return move_addr_to_kernel(conn->addr, conn->addr_len, addr); } int io_connect(struct io_kiocb *req, unsigned int issue_flags) { struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); - struct io_async_msghdr *io = req->async_data; + struct sockaddr_storage *addr = req->async_data; unsigned file_flags; int ret; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; @@ -1766,8 +1766,7 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) file_flags = force_nonblock ? O_NONBLOCK : 0; - ret = __sys_connect_file(req->file, &io->addr, connect->addr_len, - file_flags); + ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags); if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED) && force_nonblock) { if (ret == -EINPROGRESS) { @@ -1796,7 +1795,6 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) out: if (ret < 0) req_set_fail(req); - io_req_msg_cleanup(req, issue_flags); io_req_set_res(req, ret, 0); return IOU_COMPLETE; } @@ -1806,15 +1804,15 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) * which in turn end up in mnt_want_write() which will grab the fs * percpu start write sem. This can trigger a lockdep warning. */ -static int io_bind_file_create(const struct io_async_msghdr *io, int addr_len) +static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len) { const struct sockaddr_un *sun; - if (io->addr.ss_family != AF_UNIX) + if (addr->ss_family != AF_UNIX) return 0; if (addr_len <= offsetof(struct sockaddr_un, sun_path)) return 0; - sun = (const struct sockaddr_un *) &io->addr; + sun = (const struct sockaddr_un *) addr; return sun->sun_path[0] != '\0'; } @@ -1822,7 +1820,7 @@ int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); struct sockaddr __user *uaddr; - struct io_async_msghdr *io; + struct sockaddr_storage *addr; int ret; if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) @@ -1831,21 +1829,23 @@ int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); bind->addr_len = READ_ONCE(sqe->addr2); - io = io_msg_alloc_async(req); - if (unlikely(!io)) + addr = io_uring_alloc_async_data(NULL, req); + if (unlikely(!addr)) return -ENOMEM; - ret = move_addr_to_kernel(uaddr, bind->addr_len, &io->addr); + + ret = move_addr_to_kernel(uaddr, bind->addr_len, addr); if (unlikely(ret)) return ret; - if (io_bind_file_create(io, bind->addr_len)) + if (io_bind_file_create(addr, bind->addr_len)) req->flags |= REQ_F_FORCE_ASYNC; return 0; } + int io_bind(struct io_kiocb *req, unsigned int issue_flags) { struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); - struct io_async_msghdr *io = req->async_data; + struct sockaddr_storage *addr = req->async_data; struct socket *sock; int ret; @@ -1853,7 +1853,7 @@ int io_bind(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(!sock)) return -ENOTSOCK; - ret = __sys_bind_socket(sock, &io->addr, bind->addr_len); + ret = __sys_bind_socket(sock, addr, bind->addr_len); if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); diff --git a/io_uring/nop.c b/io_uring/nop.c index f5c9969e7f64a..2f9aa48832619 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -41,6 +41,8 @@ int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) nop->fd = READ_ONCE(sqe->fd); else nop->fd = -1; + if (nop->flags & IORING_NOP_FIXED_FILE) + req->flags |= REQ_F_FIXED_FILE; if (nop->flags & IORING_NOP_FIXED_BUFFER) req->buf_index = READ_ONCE(sqe->buf_index); if (nop->flags & IORING_NOP_CQE32) { @@ -60,12 +62,10 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) int ret = nop->result; if (nop->flags & IORING_NOP_FILE) { - if (nop->flags & IORING_NOP_FIXED_FILE) { + if (req->flags & REQ_F_FIXED_FILE) req->file = io_file_get_fixed(req, nop->fd, issue_flags); - req->flags |= REQ_F_FIXED_FILE; - } else { + else req->file = io_file_get_normal(req, nop->fd); - } if (!req->file) { ret = -EBADF; goto done; diff --git a/io_uring/opdef.c b/io_uring/opdef.c index c3ef52b708113..34cd320a427b8 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -203,7 +203,7 @@ const struct io_issue_def io_issue_defs[] = { .unbound_nonreg_file = 1, .pollout = 1, #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), + .async_size = sizeof(struct sockaddr_storage), .prep = io_connect_prep, .issue = io_connect, #else @@ -503,7 +503,7 @@ const struct io_issue_def io_issue_defs[] = { .needs_file = 1, .prep = io_bind_prep, .issue = io_bind, - .async_size = sizeof(struct io_async_msghdr), + .async_size = sizeof(struct sockaddr_storage), #else .prep = io_eopnotsupp_prep, #endif diff --git a/io_uring/rw.c b/io_uring/rw.c index 0c48346452797..63b6519e498cd 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -601,15 +601,15 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) { struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); struct io_kiocb *req = cmd_to_io_kiocb(rw); + int final_res = io_fixup_rw_res(req, res); if (kiocb->ki_flags & IOCB_WRITE) io_req_end_write(req); - if (unlikely(res != req->cqe.res)) { - if (res == -EAGAIN && io_rw_should_reissue(req)) - req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; - else - req->cqe.res = res; - } + + if (res == -EAGAIN && io_rw_should_reissue(req)) + req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; + else if (unlikely(final_res != req->cqe.res)) + req->cqe.res = final_res; /* order with io_iopoll_complete() checking ->iopoll_completed */ smp_store_release(&req->iopoll_completed, 1); diff --git a/kernel/audit.c b/kernel/audit.c index 34dc7cb246ff2..41a2de70fa43f 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -62,6 +62,7 @@ #include #include #include +#include #include "audit.h" @@ -950,7 +951,7 @@ static int kauditd_thread(void *dummy) * do the multicast send and rotate records from the * main queue to the retry/hold queues */ wait_event_freezable(kauditd_wait, - (skb_queue_len(&audit_queue) ? 1 : 0)); + (skb_queue_len_lockless(&audit_queue) ? 1 : 0)); } return 0; @@ -1283,7 +1284,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh, s.rate_limit = audit_rate_limit; s.backlog_limit = audit_backlog_limit; s.lost = atomic_read(&audit_lost); - s.backlog = skb_queue_len(&audit_queue); + s.backlog = skb_queue_len_lockless(&audit_queue); s.feature_bitmap = AUDIT_FEATURE_BITMAP_ALL; s.backlog_wait_time = audit_backlog_wait_time; s.backlog_wait_time_actual = atomic_read(&audit_backlog_wait_time_actual); @@ -1627,7 +1628,7 @@ static void audit_receive(struct sk_buff *skb) /* can't block with the ctrl lock, so penalize the sender now */ if (audit_backlog_limit && - (skb_queue_len(&audit_queue) > audit_backlog_limit)) { + (skb_queue_len_lockless(&audit_queue) > audit_backlog_limit)) { DECLARE_WAITQUEUE(wait, current); /* wake kauditd to try and flush the queue */ @@ -1933,7 +1934,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, long stime = audit_backlog_wait_time; while (audit_backlog_limit && - (skb_queue_len(&audit_queue) > audit_backlog_limit)) { + (skb_queue_len_lockless(&audit_queue) > audit_backlog_limit)) { /* wake kauditd to try and flush the queue */ wake_up_interruptible(&kauditd_wait); @@ -1953,7 +1954,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, } else { if (audit_rate_check() && printk_ratelimit()) pr_warn("audit_backlog=%d > audit_backlog_limit=%d\n", - skb_queue_len(&audit_queue), + skb_queue_len_lockless(&audit_queue), audit_backlog_limit); audit_log_lost("backlog limit exceeded"); return NULL; @@ -2080,7 +2081,8 @@ void audit_log_format(struct audit_buffer *ab, const char *fmt, ...) void audit_log_n_hex(struct audit_buffer *ab, const unsigned char *buf, size_t len) { - int i, avail, new_len; + int avail; + size_t i, new_len; unsigned char *ptr; struct sk_buff *skb; @@ -2090,7 +2092,12 @@ void audit_log_n_hex(struct audit_buffer *ab, const unsigned char *buf, BUG_ON(!ab->skb); skb = ab->skb; avail = skb_tailroom(skb); - new_len = len<<1; + + if (check_shl_overflow(len, 1, &new_len)) { + audit_log_format(ab, "?"); + return; + } + if (new_len >= avail) { /* Round the buffer request up to the next multiple */ new_len = AUDIT_BUFSIZ*(((new_len-avail)/AUDIT_BUFSIZ) + 1); diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c index 711454f9f7242..ae0e75403f768 100644 --- a/kernel/audit_fsnotify.c +++ b/kernel/audit_fsnotify.c @@ -84,10 +84,6 @@ struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pa dentry = kern_path_parent(pathname, &path); if (IS_ERR(dentry)) return ERR_CAST(dentry); /* returning an error */ - if (d_really_is_negative(dentry)) { - audit_mark = ERR_PTR(-ENOENT); - goto out; - } audit_mark = kzalloc_obj(*audit_mark); if (unlikely(!audit_mark)) { diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 49a8f7b1beef5..40c42e28a6168 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -59,6 +59,8 @@ struct bpf_arena { struct list_head vma_list; /* protects vma_list */ struct mutex lock; + u64 zap_gen; + struct mutex zap_mutex; struct irq_work free_irq; struct work_struct free_work; struct llist_head free_spans; @@ -228,6 +230,7 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) goto err; } mutex_init(&arena->lock); + mutex_init(&arena->zap_mutex); raw_res_spin_lock_init(&arena->spinlock); err = populate_pgtable_except_pte(arena); if (err) { @@ -318,6 +321,7 @@ struct vma_list { struct vm_area_struct *vma; struct list_head head; refcount_t mmap_count; + u64 zap_gen; }; static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma) @@ -330,6 +334,7 @@ static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma) refcount_set(&vml->mmap_count, 1); vma->vm_private_data = vml; vml->vma = vma; + vml->zap_gen = 0; list_add(&vml->head, &arena->vma_list); return 0; } @@ -668,12 +673,60 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt */ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) { + unsigned long size = (unsigned long)page_cnt << PAGE_SHIFT; + struct vm_area_struct *vma; + struct mm_struct *mm; struct vma_list *vml; + unsigned long vm_start; + u64 my_gen; - guard(mutex)(&arena->lock); - /* iterate link list under lock */ - list_for_each_entry(vml, &arena->vma_list, head) - zap_vma_range(vml->vma, uaddr, PAGE_SIZE * page_cnt); + /* + * Taking mmap_read_lock() under arena->lock would deadlock against + * arena_vm_close(), which runs with mmap_write_lock held and then + * acquires arena->lock. Drop arena->lock for mmap_read_lock(). + * + * Use per-call my_gen, recorded in vml->zap_gen, to remember which + * vmls this invocation has already processed across the lock drop. + * Hold zap_mutex around the whole walk so concurrent zap_pages() + * callers cannot overwrite each other's marks on shared vmls -- + * otherwise call B's mark would make call A skip a vml that A has + * not yet zapped for A's uaddr range. + */ + mutex_lock(&arena->zap_mutex); + mutex_lock(&arena->lock); + my_gen = ++arena->zap_gen; + for (;;) { + mm = NULL; + list_for_each_entry(vml, &arena->vma_list, head) { + if (vml->zap_gen >= my_gen) + continue; + vml->zap_gen = my_gen; + if (!mmget_not_zero(vml->vma->vm_mm)) + continue; + mm = vml->vma->vm_mm; + vm_start = vml->vma->vm_start; + break; + } + if (!mm) + break; + mutex_unlock(&arena->lock); + + mmap_read_lock(mm); + /* + * Re-resolve: while we waited the VMA could have been unmapped + * and a different mapping installed at the same address. + */ + vma = find_vma(mm, vm_start); + if (vma && vma->vm_start == vm_start && + vma->vm_file && vma->vm_file->private_data == &arena->map) + zap_vma_range(vma, uaddr, size); + mmap_read_unlock(mm); + mmput(mm); + + mutex_lock(&arena->lock); + } + mutex_unlock(&arena->lock); + mutex_unlock(&arena->zap_mutex); } static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index dfb2110ab7335..4b68d7a2b90ea 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -386,7 +386,7 @@ static long array_map_update_elem(struct bpf_map *map, void *key, void *value, if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { val = this_cpu_ptr(array->pptrs[index & array->index_mask]); copy_map_value(map, val, value); - bpf_obj_free_fields(array->map.record, val); + bpf_obj_cancel_fields(map, val); } else { val = array->value + (u64)array->elem_size * (index & array->index_mask); @@ -394,7 +394,7 @@ static long array_map_update_elem(struct bpf_map *map, void *key, void *value, copy_map_value_locked(map, val, value, false); else copy_map_value(map, val, value); - bpf_obj_free_fields(array->map.record, val); + bpf_obj_cancel_fields(map, val); } return 0; } @@ -434,14 +434,14 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, cpu = map_flags >> 32; ptr = per_cpu_ptr(pptr, cpu); copy_map_value(map, ptr, value); - bpf_obj_free_fields(array->map.record, ptr); + bpf_obj_cancel_fields(map, ptr); goto unlock; } for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; copy_map_value(map, ptr, val); - bpf_obj_free_fields(array->map.record, ptr); + bpf_obj_cancel_fields(map, ptr); } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c index 0da8d923e39d4..f9e81060c1f45 100644 --- a/kernel/bpf/bpf_inode_storage.c +++ b/kernel/bpf/bpf_inode_storage.c @@ -178,6 +178,15 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) { + /* + * Do not allow allocation of BPF_MAP_TYPE_INODE_STORAGE if the BPF LSM + * was not initialized by the LSM framework at boot. Without proper + * initialization, the BPF inode security blob offset remains unprepared, + * causing bpf_inode() to calculate an invalid memory offset and corrupt + * inode->i_security. + */ + if (!bpf_lsm_initialized) + return ERR_PTR(-EOPNOTSUPP); return bpf_local_storage_map_alloc(attr, &inode_cache); } diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c index e7a2fc60523f6..5ed7cb4b98c00 100644 --- a/kernel/bpf/bpf_lru_list.c +++ b/kernel/bpf/bpf_lru_list.c @@ -13,23 +13,8 @@ #define PERCPU_FREE_TARGET (4) #define PERCPU_NR_SCANS PERCPU_FREE_TARGET -/* Helpers to get the local list index */ -#define LOCAL_LIST_IDX(t) ((t) - BPF_LOCAL_LIST_T_OFFSET) -#define LOCAL_FREE_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_FREE) -#define LOCAL_PENDING_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_PENDING) #define IS_LOCAL_LIST_TYPE(t) ((t) >= BPF_LOCAL_LIST_T_OFFSET) -/* Local list helpers */ -static struct list_head *local_free_list(struct bpf_lru_locallist *loc_l) -{ - return &loc_l->lists[LOCAL_FREE_LIST_IDX]; -} - -static struct list_head *local_pending_list(struct bpf_lru_locallist *loc_l) -{ - return &loc_l->lists[LOCAL_PENDING_LIST_IDX]; -} - /* bpf_lru_node helpers */ static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node) { @@ -72,6 +57,7 @@ static void __bpf_lru_node_move_to_free(struct bpf_lru_list *l, bpf_lru_list_count_dec(l, node->type); node->type = tgt_free_type; + WRITE_ONCE(node->pending_free, 0); list_move(&node->list, free_list); } @@ -87,6 +73,9 @@ static void __bpf_lru_node_move_in(struct bpf_lru_list *l, bpf_lru_list_count_inc(l, tgt_type); node->type = tgt_type; bpf_lru_node_clear_ref(node); + /* Reset pending_free only when moving to the free list */ + if (tgt_type == BPF_LRU_LIST_T_FREE) + WRITE_ONCE(node->pending_free, 0); list_move(&node->list, &l->lists[tgt_type]); } @@ -212,9 +201,11 @@ __bpf_lru_list_shrink_inactive(struct bpf_lru *lru, unsigned int i = 0; list_for_each_entry_safe_reverse(node, tmp_node, inactive, list) { - if (bpf_lru_node_is_ref(node)) { + if (bpf_lru_node_is_ref(node) && + !READ_ONCE(node->pending_free)) { __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); - } else if (lru->del_from_htab(lru->del_arg, node)) { + } else if (READ_ONCE(node->pending_free) || + lru->del_from_htab(lru->del_arg, node)) { __bpf_lru_node_move_to_free(l, node, free_list, tgt_free_type); if (++nshrinked == tgt_nshrink) @@ -273,7 +264,8 @@ static unsigned int __bpf_lru_list_shrink(struct bpf_lru *lru, list_for_each_entry_safe_reverse(node, tmp_node, force_shrink_list, list) { - if (lru->del_from_htab(lru->del_arg, node)) { + if (READ_ONCE(node->pending_free) || + lru->del_from_htab(lru->del_arg, node)) { __bpf_lru_node_move_to_free(l, node, free_list, tgt_free_type); return 1; @@ -290,8 +282,10 @@ static void __local_list_flush(struct bpf_lru_list *l, struct bpf_lru_node *node, *tmp_node; list_for_each_entry_safe_reverse(node, tmp_node, - local_pending_list(loc_l), list) { - if (bpf_lru_node_is_ref(node)) + &loc_l->pending_list, list) { + if (READ_ONCE(node->pending_free)) + __bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_FREE); + else if (bpf_lru_node_is_ref(node)) __bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_ACTIVE); else __bpf_lru_node_move_in(l, node, @@ -307,9 +301,12 @@ static void bpf_lru_list_push_free(struct bpf_lru_list *l, if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) return; - raw_spin_lock_irqsave(&l->lock, flags); + if (raw_res_spin_lock_irqsave(&l->lock, flags)) { + WRITE_ONCE(node->pending_free, 1); + return; + } __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); - raw_spin_unlock_irqrestore(&l->lock, flags); + raw_res_spin_unlock_irqrestore(&l->lock, flags); } static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, @@ -318,8 +315,10 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, struct bpf_lru_list *l = &lru->common_lru.lru_list; struct bpf_lru_node *node, *tmp_node; unsigned int nfree = 0; + LIST_HEAD(tmp_free); - raw_spin_lock(&l->lock); + if (raw_res_spin_lock(&l->lock)) + return; __local_list_flush(l, loc_l); @@ -327,7 +326,7 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, list_for_each_entry_safe(node, tmp_node, &l->lists[BPF_LRU_LIST_T_FREE], list) { - __bpf_lru_node_move_to_free(l, node, local_free_list(loc_l), + __bpf_lru_node_move_to_free(l, node, &tmp_free, BPF_LRU_LOCAL_LIST_T_FREE); if (++nfree == lru->target_free) break; @@ -335,10 +334,19 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, if (nfree < lru->target_free) __bpf_lru_list_shrink(lru, l, lru->target_free - nfree, - local_free_list(loc_l), + &tmp_free, BPF_LRU_LOCAL_LIST_T_FREE); - raw_spin_unlock(&l->lock); + raw_res_spin_unlock(&l->lock); + + /* + * Transfer the harvested nodes from the temporary list_head into + * the lockless per-CPU free llist. + */ + list_for_each_entry_safe(node, tmp_node, &tmp_free, list) { + list_del(&node->list); + llist_add(&node->llist, &loc_l->free_llist); + } } static void __local_list_add_pending(struct bpf_lru *lru, @@ -350,22 +358,21 @@ static void __local_list_add_pending(struct bpf_lru *lru, *(u32 *)((void *)node + lru->hash_offset) = hash; node->cpu = cpu; node->type = BPF_LRU_LOCAL_LIST_T_PENDING; + WRITE_ONCE(node->pending_free, 0); bpf_lru_node_clear_ref(node); - list_add(&node->list, local_pending_list(loc_l)); + list_add(&node->list, &loc_l->pending_list); } static struct bpf_lru_node * __local_list_pop_free(struct bpf_lru_locallist *loc_l) { - struct bpf_lru_node *node; + struct llist_node *llnode; - node = list_first_entry_or_null(local_free_list(loc_l), - struct bpf_lru_node, - list); - if (node) - list_del(&node->list); + llnode = llist_del_first(&loc_l->free_llist); + if (!llnode) + return NULL; - return node; + return container_of(llnode, struct bpf_lru_node, llist); } static struct bpf_lru_node * @@ -376,10 +383,10 @@ __local_list_pop_pending(struct bpf_lru *lru, struct bpf_lru_locallist *loc_l) ignore_ref: /* Get from the tail (i.e. older element) of the pending list. */ - list_for_each_entry_reverse(node, local_pending_list(loc_l), - list) { + list_for_each_entry_reverse(node, &loc_l->pending_list, list) { if ((!bpf_lru_node_is_ref(node) || force) && - lru->del_from_htab(lru->del_arg, node)) { + (READ_ONCE(node->pending_free) || + lru->del_from_htab(lru->del_arg, node))) { list_del(&node->list); return node; } @@ -404,7 +411,8 @@ static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru, l = per_cpu_ptr(lru->percpu_lru, cpu); - raw_spin_lock_irqsave(&l->lock, flags); + if (raw_res_spin_lock_irqsave(&l->lock, flags)) + return NULL; __bpf_lru_list_rotate(lru, l); @@ -420,7 +428,7 @@ static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru, __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); } - raw_spin_unlock_irqrestore(&l->lock, flags); + raw_res_spin_unlock_irqrestore(&l->lock, flags); return node; } @@ -437,7 +445,8 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, loc_l = per_cpu_ptr(clru->local_list, cpu); - raw_spin_lock_irqsave(&loc_l->lock, flags); + if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) + return NULL; node = __local_list_pop_free(loc_l); if (!node) { @@ -448,17 +457,22 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, if (node) __local_list_add_pending(lru, loc_l, cpu, node, hash); - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); if (node) return node; - /* No free nodes found from the local free list and + /* + * No free nodes found from the local free list and * the global LRU list. * * Steal from the local free/pending list of the * current CPU and remote CPU in RR. It starts * with the loc_l->next_steal CPU. + * + * Acquire the victim's lock before touching either list. On + * acquisition failure (rqspinlock AA or timeout) skip the victim + * and try the next CPU. */ first_steal = loc_l->next_steal; @@ -466,24 +480,36 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, do { steal_loc_l = per_cpu_ptr(clru->local_list, steal); - raw_spin_lock_irqsave(&steal_loc_l->lock, flags); - - node = __local_list_pop_free(steal_loc_l); - if (!node) - node = __local_list_pop_pending(lru, steal_loc_l); - - raw_spin_unlock_irqrestore(&steal_loc_l->lock, flags); + if (!raw_res_spin_lock_irqsave(&steal_loc_l->lock, flags)) { + node = __local_list_pop_free(steal_loc_l); + if (!node) + node = __local_list_pop_pending(lru, steal_loc_l); + raw_res_spin_unlock_irqrestore(&steal_loc_l->lock, flags); + } steal = cpumask_next_wrap(steal, cpu_possible_mask); } while (!node && steal != first_steal); loc_l->next_steal = steal; - if (node) { - raw_spin_lock_irqsave(&loc_l->lock, flags); - __local_list_add_pending(lru, loc_l, cpu, node, hash); - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + if (!node) + return NULL; + + if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) { + /* + * The local pending lock can't be acquired (rqspinlock AA + * or timeout). Return the stolen node to the per-CPU + * free_llist instead of orphaning it; the next pop_free on + * this CPU will pick it up. + */ + node->type = BPF_LRU_LOCAL_LIST_T_FREE; + bpf_lru_node_clear_ref(node); + WRITE_ONCE(node->pending_free, 0); + llist_add(&node->llist, &loc_l->free_llist); + return NULL; } + __local_list_add_pending(lru, loc_l, cpu, node, hash); + raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); return node; } @@ -511,18 +537,24 @@ static void bpf_common_lru_push_free(struct bpf_lru *lru, loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu); - raw_spin_lock_irqsave(&loc_l->lock, flags); + if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) { + WRITE_ONCE(node->pending_free, 1); + return; + } if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + raw_res_spin_unlock_irqrestore(&loc_l->lock, + flags); goto check_lru_list; } node->type = BPF_LRU_LOCAL_LIST_T_FREE; bpf_lru_node_clear_ref(node); - list_move(&node->list, local_free_list(loc_l)); + list_del(&node->list); + + raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + llist_add(&node->llist, &loc_l->free_llist); return; } @@ -538,11 +570,14 @@ static void bpf_percpu_lru_push_free(struct bpf_lru *lru, l = per_cpu_ptr(lru->percpu_lru, node->cpu); - raw_spin_lock_irqsave(&l->lock, flags); + if (raw_res_spin_lock_irqsave(&l->lock, flags)) { + WRITE_ONCE(node->pending_free, 1); + return; + } __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); - raw_spin_unlock_irqrestore(&l->lock, flags); + raw_res_spin_unlock_irqrestore(&l->lock, flags); } void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node) @@ -565,6 +600,7 @@ static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf, node = (struct bpf_lru_node *)(buf + node_offset); node->type = BPF_LRU_LIST_T_FREE; + node->pending_free = 0; bpf_lru_node_clear_ref(node); list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); buf += elem_size; @@ -594,6 +630,7 @@ static void bpf_percpu_lru_populate(struct bpf_lru *lru, void *buf, node = (struct bpf_lru_node *)(buf + node_offset); node->cpu = cpu; node->type = BPF_LRU_LIST_T_FREE; + node->pending_free = 0; bpf_lru_node_clear_ref(node); list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); i++; @@ -618,14 +655,12 @@ void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset, static void bpf_lru_locallist_init(struct bpf_lru_locallist *loc_l, int cpu) { - int i; - - for (i = 0; i < NR_BPF_LRU_LOCAL_LIST_T; i++) - INIT_LIST_HEAD(&loc_l->lists[i]); + INIT_LIST_HEAD(&loc_l->pending_list); + init_llist_head(&loc_l->free_llist); loc_l->next_steal = cpu; - raw_spin_lock_init(&loc_l->lock); + raw_res_spin_lock_init(&loc_l->lock); } static void bpf_lru_list_init(struct bpf_lru_list *l) @@ -640,7 +675,7 @@ static void bpf_lru_list_init(struct bpf_lru_list *l) l->next_inactive_rotation = &l->lists[BPF_LRU_LIST_T_INACTIVE]; - raw_spin_lock_init(&l->lock); + raw_res_spin_lock_init(&l->lock); } int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset, diff --git a/kernel/bpf/bpf_lru_list.h b/kernel/bpf/bpf_lru_list.h index fe2661a58ea94..8d0ee61622af0 100644 --- a/kernel/bpf/bpf_lru_list.h +++ b/kernel/bpf/bpf_lru_list.h @@ -6,11 +6,11 @@ #include #include -#include +#include +#include #define NR_BPF_LRU_LIST_T (3) #define NR_BPF_LRU_LIST_COUNT (2) -#define NR_BPF_LRU_LOCAL_LIST_T (2) #define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T enum bpf_lru_list_type { @@ -22,10 +22,22 @@ enum bpf_lru_list_type { }; struct bpf_lru_node { - struct list_head list; + /* + * A node is in at most one list at a time. The free path on the + * per-CPU locallist uses an llist, so share storage via a union. + */ + union { + struct list_head list; + struct llist_node llist; + }; u16 cpu; u8 type; u8 ref; + /* + * Marks nodes whose *_push_free() lock acquire failed; reclaimed + * by flush/shrink which honor the flag instead of del_from_htab(). + */ + u8 pending_free; }; struct bpf_lru_list { @@ -34,13 +46,14 @@ struct bpf_lru_list { /* The next inactive list rotation starts from here */ struct list_head *next_inactive_rotation; - raw_spinlock_t lock ____cacheline_aligned_in_smp; + rqspinlock_t lock ____cacheline_aligned_in_smp; }; struct bpf_lru_locallist { - struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T]; + struct list_head pending_list; + struct llist_head free_llist; u16 next_steal; - raw_spinlock_t lock; + rqspinlock_t lock; }; struct bpf_common_lru { diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index c5c925f002021..1433809bb166a 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -51,6 +51,9 @@ BTF_ID(func, bpf_lsm_key_getsecurity) #ifdef CONFIG_AUDIT BTF_ID(func, bpf_lsm_audit_rule_match) #endif +#ifdef CONFIG_SECURITY_NETWORK_XFRM +BTF_ID(func, bpf_lsm_xfrm_decode_session) +#endif BTF_ID(func, bpf_lsm_ismaclabel) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_SET_END(bpf_lsm_disabled_hooks) @@ -427,6 +430,26 @@ BTF_ID(func, bpf_lsm_audit_rule_known) BTF_ID(func, bpf_lsm_inode_xattr_skipcap) BTF_SET_END(bool_lsm_hooks) +/* hooks returning void */ +#define LSM_HOOK_void(DEFAULT, NAME, ...) BTF_ID(func, bpf_lsm_##NAME) +#define LSM_HOOK_int(DEFAULT, NAME, ...) /* nothing */ +#define LSM_HOOK(RET, DEFAULT, NAME, ...) LSM_HOOK_##RET(DEFAULT, NAME, __VA_ARGS__) +BTF_SET_START(void_lsm_hooks) +#include +#undef LSM_HOOK +#undef LSM_HOOK_void +#undef LSM_HOOK_int +BTF_SET_END(void_lsm_hooks) + +bool bpf_lsm_hook_returns_errno(u32 btf_id) +{ + if (btf_id_set_contains(&bool_lsm_hooks, btf_id)) + return false; + if (btf_id_set_contains(&void_lsm_hooks, btf_id)) + return false; + return true; +} + int bpf_lsm_get_retval_range(const struct bpf_prog *prog, struct bpf_retval_range *retval_range) { diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a62d785812076..510aa32847da9 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3668,7 +3668,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, u32 field_cnt, u32 repeat_cnt, u32 elem_size) { - u32 i, j; + u32 i, j, total_cnt, total_repeats; u32 cur; /* Ensure not repeating fields that should not be repeated. */ @@ -3686,10 +3686,9 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, } } - /* The type of struct size or variable size is u32, - * so the multiplication will not overflow. - */ - if (field_cnt * (repeat_cnt + 1) > info_cnt) + if (check_add_overflow(repeat_cnt, 1, &total_repeats) || + check_mul_overflow(field_cnt, total_repeats, &total_cnt) || + total_cnt > (u32)info_cnt) return -E2BIG; cur = field_cnt; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 876f6a81a9b61..2f87c273e6429 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -55,6 +55,28 @@ void __init cgroup_bpf_lifetime_notifier_init(void) &cgroup_bpf_lifetime_nb)); } +#ifdef CONFIG_BPF_LSM +struct cgroup_lsm_atype { + u32 attach_btf_id; + int refcnt; + bool returns_errno; +}; + +static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM]; + +static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype) +{ + if (atype >= CGROUP_LSM_START && atype <= CGROUP_LSM_END) + return READ_ONCE(cgroup_lsm_atype[atype - CGROUP_LSM_START].returns_errno); + return true; +} +#else +static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype) +{ + return true; +} +#endif + /* __always_inline is necessary to prevent indirect call through run_prog * function pointer. */ @@ -83,7 +105,8 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp, *(ret_flags) |= (func_ret >> 1); func_ret &= 1; } - if (!func_ret && !IS_ERR_VALUE((long)run_ctx.retval)) + if (!func_ret && cgroup_bpf_hook_returns_errno(atype) && + !IS_ERR_VALUE((long)run_ctx.retval)) run_ctx.retval = -EPERM; item++; } @@ -156,13 +179,6 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx, } #ifdef CONFIG_BPF_LSM -struct cgroup_lsm_atype { - u32 attach_btf_id; - int refcnt; -}; - -static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM]; - static enum cgroup_bpf_attach_type bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id) { @@ -191,10 +207,13 @@ void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) lockdep_assert_held(&cgroup_mutex); - WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id && - cgroup_lsm_atype[i].attach_btf_id != attach_btf_id); - - cgroup_lsm_atype[i].attach_btf_id = attach_btf_id; + if (!cgroup_lsm_atype[i].attach_btf_id) { + cgroup_lsm_atype[i].attach_btf_id = attach_btf_id; + WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, + bpf_lsm_hook_returns_errno(attach_btf_id)); + } else { + WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id != attach_btf_id); + } cgroup_lsm_atype[i].refcnt++; } @@ -203,8 +222,10 @@ void bpf_cgroup_atype_put(int cgroup_atype) int i = cgroup_atype - CGROUP_LSM_START; cgroup_lock(); - if (--cgroup_lsm_atype[i].refcnt <= 0) + if (--cgroup_lsm_atype[i].refcnt <= 0) { + WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true); cgroup_lsm_atype[i].attach_btf_id = 0; + } WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0); cgroup_unlock(); } @@ -918,19 +939,65 @@ static int cgroup_bpf_attach(struct cgroup *cgrp, return ret; } +static int effective_prog_pos(struct cgroup *cgrp, + enum cgroup_bpf_attach_type atype, + struct bpf_prog_list *target_pl) +{ + int cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart, pos = -1; + struct bpf_prog_list *pl; + struct cgroup *p = cgrp; + + /* count effective programs to find where the preorder region ends */ + do { + if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) + cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt); + p = cgroup_parent(p); + } while (p); + + /* replay compute_effective_progs() placement and record target's slot */ + cnt = 0; + p = cgrp; + fstart = preorder_cnt; + bstart = preorder_cnt - 1; + do { + if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) + continue; + + init_bstart = bstart; + hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { + if (!prog_list_prog(pl)) + continue; + + if (pl->flags & BPF_F_PREORDER) { + if (pl == target_pl) + pos = bstart; + bstart--; + } else { + if (pl == target_pl) + pos = fstart; + fstart++; + } + cnt++; + } + + /* reverse pre-ordering progs at this cgroup level */ + if (pos >= bstart + 1 && pos <= init_bstart) + pos = bstart + 1 + init_bstart - pos; + } while ((p = cgroup_parent(p))); + + return pos; +} + /* Swap updated BPF program for given link in effective program arrays across * all descendant cgroups. This function is guaranteed to succeed. */ static void replace_effective_prog(struct cgroup *cgrp, enum cgroup_bpf_attach_type atype, - struct bpf_cgroup_link *link) + struct bpf_prog_list *pl) { struct bpf_prog_array_item *item; struct cgroup_subsys_state *css; struct bpf_prog_array *progs; - struct bpf_prog_list *pl; - struct hlist_head *head; - struct cgroup *cg; int pos; css_for_each_descendant_pre(css, &cgrp->self) { @@ -939,27 +1006,15 @@ static void replace_effective_prog(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; - /* find position of link in effective progs array */ - for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) { - if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) - continue; + pos = effective_prog_pos(desc, atype, pl); + if (WARN_ON_ONCE(pos < 0)) + continue; - head = &cg->bpf.progs[atype]; - hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) - continue; - if (pl->link == link) - goto found; - pos++; - } - } -found: - BUG_ON(!cg); progs = rcu_dereference_protected( desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); item = &progs->items[pos]; - WRITE_ONCE(item->prog, link->link.prog); + WRITE_ONCE(item->prog, pl->link->link.prog); } } @@ -1003,7 +1058,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); - replace_effective_prog(cgrp, atype, link); + replace_effective_prog(cgrp, atype, pl); bpf_prog_put(old_prog); return 0; } @@ -1070,19 +1125,14 @@ static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs, * recomputing the array in place. * * @cgrp: The cgroup which descendants to travers - * @prog: A program to detach or NULL - * @link: A link to detach or NULL + * @pl: The prog_list entry being detached * @atype: Type of detach operation */ -static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, - struct bpf_cgroup_link *link, +static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, enum cgroup_bpf_attach_type atype) { struct cgroup_subsys_state *css; struct bpf_prog_array *progs; - struct bpf_prog_list *pl; - struct hlist_head *head; - struct cgroup *cg; int pos; /* recompute effective prog array in place */ @@ -1092,24 +1142,11 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; - /* find position of link or prog in effective progs array */ - for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) { - if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) - continue; - - head = &cg->bpf.progs[atype]; - hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) - continue; - if (pl->prog == prog && pl->link == link) - goto found; - pos++; - } - } - + pos = effective_prog_pos(desc, atype, pl); /* no link or prog match, skip the cgroup of this layer */ - continue; -found: + if (pos < 0) + continue; + progs = rcu_dereference_protected( desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); @@ -1175,7 +1212,7 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, /* if update effective array failed replace the prog with a dummy prog*/ pl->prog = old_prog; pl->link = link; - purge_effective_progs(cgrp, old_prog, link, atype); + purge_effective_progs(cgrp, pl, atype); } /* now can actually delete it from this cgroup list */ @@ -1208,7 +1245,7 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, /* Must be called with cgroup_mutex held to avoid races. */ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { __u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags); bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE; @@ -1259,7 +1296,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, return -EFAULT; if (!effective_query && from_atype == to_atype) revision = cgrp->bpf.revisions[from_atype]; - if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) + if (uattr_size >= offsetofend(union bpf_attr, query.revision) && + copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) return -EFAULT; if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt) /* return early if user requested only program count + flags */ @@ -1312,12 +1350,12 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { int ret; cgroup_lock(); - ret = __cgroup_bpf_query(cgrp, attr, uattr); + ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size); cgroup_unlock(); return ret; } @@ -1520,7 +1558,7 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) } int cgroup_bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { struct cgroup *cgrp; int ret; @@ -1529,7 +1567,7 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, if (IS_ERR(cgrp)) return PTR_ERR(cgrp); - ret = cgroup_bpf_query(cgrp, attr, uattr); + ret = cgroup_bpf_query(cgrp, attr, uattr, uattr_size); cgroup_put(cgrp); return ret; @@ -1935,8 +1973,8 @@ int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, kfree(ctx.cur_val); - if (ret == 1 && ctx.new_updated) { - kfree(*buf); + if (!ret && ctx.new_updated) { + kvfree(*buf); *buf = ctx.new_val; *pcount = ctx.new_len; } else { diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6aa2a8b240306..f5d9c27e654d4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -875,6 +876,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, struct bpf_prog_pack { struct list_head list; void *ptr; + bool arch_flush_needed; unsigned long bitmap[]; }; @@ -883,6 +885,15 @@ void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) memset(area, 0, size); } +DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush); + +/* + * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the + * pack allocator test "is a predictor flush wired up at all" with a cheap + * static branch instead of repeatedly querying the static call target. + */ +DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) static DEFINE_MUTEX(pack_mutex); @@ -918,6 +929,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; set_vm_flush_reset_perms(pack->ptr); err = set_memory_rox((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); @@ -932,15 +945,23 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins return NULL; } -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); - struct bpf_prog_pack *pack; - unsigned long pos; + struct bpf_prog_pack *pack, *fallback_pack = NULL; + unsigned long pos, fallback_pos = 0; void *ptr = NULL; mutex_lock(&pack_mutex); if (size > BPF_PROG_PACK_SIZE) { + /* + * Allocations larger than a pack get their own pages, and + * predictors are not flushed for such allocation. This is only + * safe because cBPF programs (the unprivileged attack surface) + * are bounded well below a pack size. + */ + if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled)) + pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); if (ptr) { @@ -960,8 +981,29 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) list_for_each_entry(pack, &pack_list, list) { pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, nbits, 0); - if (pos < BPF_PROG_CHUNK_COUNT) + if (pos >= BPF_PROG_CHUNK_COUNT) + continue; + /* Flush not enabled, use any pack */ + if (!static_branch_unlikely(&bpf_pred_flush_enabled)) goto found_free_area; + /* + * cBPF reuse of a dirty pack triggers a flush, so prefer a + * clean pack for cBPF. eBPF never flushes, so steer it to a + * dirty pack and keep clean packs free for cBPF. + */ + if (was_classic ^ pack->arch_flush_needed) + goto found_free_area; + if (!fallback_pack) { + fallback_pack = pack; + fallback_pos = pos; + } + } + + /* No preferred pack found */ + if (fallback_pack) { + pack = fallback_pack; + pos = fallback_pos; + goto found_free_area; } pack = alloc_new_pack(bpf_fill_ill_insns); @@ -971,6 +1013,16 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: + /* Flush only for cBPF as it may contain a crafted gadget */ + if (static_branch_unlikely(&bpf_pred_flush_enabled) && + pack->arch_flush_needed && + was_classic) { + struct bpf_prog_pack *p; + + static_call_cond(bpf_arch_pred_flush)(); + list_for_each_entry(p, &pack_list, list) + p->arch_flush_needed = false; + } bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1008,6 +1060,9 @@ void bpf_prog_pack_free(void *ptr, u32 size) "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); + + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, BPF_PROG_CHUNK_COUNT, 0) == 0) { list_del(&pack->list); @@ -1130,7 +1185,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, struct bpf_binary_header **rw_header, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns) + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic) { struct bpf_binary_header *ro_header; u32 size, hole, start; @@ -1143,7 +1199,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, if (bpf_jit_charge_modmem(size)) return NULL; - ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic); if (!ro_header) { bpf_jit_uncharge_modmem(size); return NULL; @@ -2471,7 +2527,7 @@ static bool __bpf_prog_map_compatible(struct bpf_map *map, cookie = aux->cgroup_storage[i] ? aux->cgroup_storage[i]->cookie : 0; ret = map->owner->storage_cookie[i] == cookie || - !cookie; + (!cookie && !aux->tail_call_reachable); } if (ret && map->owner->attach_func_proto != aux->attach_func_proto) { diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index cc0a43ebab6b9..dc7b859e8bbfd 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -581,6 +581,10 @@ static int dev_map_enqueue_clone(struct bpf_dtab_netdev *obj, { struct xdp_frame *nxdpf; + /* Frags live outside the linear frame and cannot be cloned safely. */ + if (unlikely(xdp_frame_has_frags(xdpf))) + return -EOPNOTSUPP; + nxdpf = xdpf_clone(xdpf); if (!nxdpf) return -ENOMEM; @@ -706,6 +710,18 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, if (unlikely(err)) return err; + if (dst->xdp_prog && skb_cloned(skb)) { + struct sk_buff *nskb; + + nskb = skb_copy(skb, GFP_ATOMIC); + if (!nskb) + return -ENOMEM; + + nskb->mac_len = skb->mac_len; + consume_skb(skb); + skb = nskb; + } + /* Redirect has already succeeded semantically at this point, so we just * return 0 even if packet is dropped. Helper below takes care of * freeing skb. @@ -726,6 +742,9 @@ static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst, struct sk_buff *nskb; int err; + if (unlikely(skb_is_nonlinear(skb))) + return -EOPNOTSUPP; + nskb = skb_clone(skb, GFP_ATOMIC); if (!nskb) return -ENOMEM; diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index f8a3c7eb451e4..0391b3bc00734 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, */ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || - insn->src_reg == BPF_PSEUDO_MAP_VALUE; + insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE || + insn->src_reg == BPF_PSEUDO_BTF_ID; char tmp[64]; if (is_ptr && !allow_ptr_leaks) diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index b77db7413f8c7..ea2d60dc1feeb 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, mutex_lock(&d->mutex); if (!d->image) { - d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); + d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 3dd9b4924ae4f..74c5ced74032b 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -242,6 +242,10 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab) if (IS_ERR_OR_NULL(htab->map.record)) return; + /* + * Preallocated maps do not have a bpf_mem_alloc destructor, so fully + * destroy every element, including the extra elements. + */ if (htab_has_extra_elems(htab)) num_entries += num_possible_cpus(); for (i = 0; i < num_entries; i++) { @@ -834,8 +838,8 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, return insn - insn_buf; } -static void check_and_free_fields(struct bpf_htab *htab, - struct htab_elem *elem) +static void check_and_cancel_fields(struct bpf_htab *htab, + struct htab_elem *elem) { if (IS_ERR_OR_NULL(htab->map.record)) return; @@ -845,11 +849,11 @@ static void check_and_free_fields(struct bpf_htab *htab, int cpu; for_each_possible_cpu(cpu) - bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); + bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu)); } else { void *map_value = htab_elem_value(elem, htab->map.key_size); - bpf_obj_free_fields(htab->map.record, map_value); + bpf_obj_cancel_fields(&htab->map, map_value); } } @@ -884,7 +888,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) htab_unlock_bucket(b, flags); if (l == tgt_l) - check_and_free_fields(htab, l); + check_and_cancel_fields(htab, l); return l == tgt_l; } @@ -949,7 +953,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) { - check_and_free_fields(htab, l); + check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); @@ -1002,7 +1006,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) if (htab_is_prealloc(htab)) { bpf_map_dec_elem_count(&htab->map); - check_and_free_fields(htab, l); + check_and_cancel_fields(htab, l); pcpu_freelist_push(&htab->freelist, &l->fnode); } else { dec_elem_count(htab); @@ -1019,7 +1023,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, /* copy true value_size bytes */ ptr = this_cpu_ptr(pptr); copy_map_value(&htab->map, ptr, value); - bpf_obj_free_fields(htab->map.record, ptr); + bpf_obj_cancel_fields(&htab->map, ptr); } else { u32 size = round_up(htab->map.value_size, 8); void *val; @@ -1029,7 +1033,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, cpu = map_flags >> 32; ptr = per_cpu_ptr(pptr, cpu); copy_map_value(&htab->map, ptr, value); - bpf_obj_free_fields(htab->map.record, ptr); + bpf_obj_cancel_fields(&htab->map, ptr); return; } @@ -1037,7 +1041,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, ptr = per_cpu_ptr(pptr, cpu); val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; copy_map_value(&htab->map, ptr, val); - bpf_obj_free_fields(htab->map.record, ptr); + bpf_obj_cancel_fields(&htab->map, ptr); } } } @@ -1253,11 +1257,11 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (l_old) { hlist_nulls_del_rcu(&l_old->hash_node); - /* l_old has already been stashed in htab->extra_elems, free - * its special fields before it is available for reuse. + /* l_old has already been stashed in htab->extra_elems, cancel + * its reusable special fields before it is available for reuse. */ if (htab_is_prealloc(htab)) - check_and_free_fields(htab, l_old); + check_and_cancel_fields(htab, l_old); } htab_unlock_bucket(b, flags); if (l_old && !htab_is_prealloc(htab)) @@ -1270,7 +1274,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) { - check_and_free_fields(htab, elem); + check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); bpf_lru_push_free(&htab->lru, &elem->lru_node); } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b5314c9fed3cf..c2032af98e083 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2295,6 +2295,7 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root, struct bpf_spin_lock *spin_lock) { struct rb_root_cached orig_root, *root = rb_root; + struct bpf_rb_node_kern *node; struct rb_node *pos, *n; void *obj; @@ -2303,14 +2304,20 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root, __bpf_spin_lock_irqsave(spin_lock); orig_root = *root; + bpf_rbtree_postorder_for_each_entry_safe(pos, n, &orig_root.rb_root) { + node = rb_entry(pos, struct bpf_rb_node_kern, rb_node); + WRITE_ONCE(node->owner, BPF_PTR_POISON); + } *root = RB_ROOT_CACHED; __bpf_spin_unlock_irqrestore(spin_lock); bpf_rbtree_postorder_for_each_entry_safe(pos, n, &orig_root.rb_root) { obj = pos; obj -= field->graph_root.node_offset; - - + node = rb_entry(pos, struct bpf_rb_node_kern, rb_node); + RB_CLEAR_NODE(pos); + /* Ensure __bpf_rbtree_add() sees the node as unlinked. */ + smp_store_release(&node->owner, NULL); __bpf_obj_drop_impl(obj, field->graph_root.value_rec, false); } } @@ -2912,11 +2919,13 @@ __bpf_kfunc struct task_struct *bpf_task_from_vpid(s32 vpid) { struct task_struct *p; - rcu_read_lock(); + guard(rcu)(); + if (!task_active_pid_ns(current)) + return NULL; + p = find_task_by_vpid(vpid); if (p) p = bpf_task_acquire(p); - rcu_read_unlock(); return p; } diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 25c06a0118258..188c774a469ca 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -766,10 +766,18 @@ static void bpf_destroy_inode(struct inode *inode) { enum bpf_type type; - if (S_ISLNK(inode->i_mode)) - kfree(inode->i_link); if (!bpf_inode_type(inode, &type)) bpf_any_put(inode->i_private, type); +} + +/* + * Called after RCU grace period - safe to free inode and anything + * that might be accessed by RCU pathwalk (inode fields, i_link). + */ +static void bpf_free_inode(struct inode *inode) +{ + if (S_ISLNK(inode->i_mode)) + kfree(inode->i_link); free_inode_nonrcu(inode); } @@ -778,6 +786,7 @@ const struct super_operations bpf_super_ops = { .drop_inode = inode_just_drop, .show_options = bpf_show_options, .destroy_inode = bpf_destroy_inode, + .free_inode = bpf_free_inode, }; enum { diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c index 0f57608b385d4..4d6f25db9ba12 100644 --- a/kernel/bpf/lpm_trie.c +++ b/kernel/bpf/lpm_trie.c @@ -246,7 +246,7 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) /* Start walking the trie from the root node ... */ - for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held()); + for (node = rcu_dereference_check(trie->root, bpf_rcu_lock_held()); node;) { unsigned int next_bit; size_t matchlen; @@ -280,7 +280,7 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) */ next_bit = extract_bit(key->data, node->prefixlen); node = rcu_dereference_check(node->child[next_bit], - rcu_read_lock_bh_held()); + bpf_rcu_lock_held()); } if (!found) @@ -359,7 +359,7 @@ static long trie_update_elem(struct bpf_map *map, */ slot = &trie->root; - while ((node = rcu_dereference(*slot))) { + while ((node = rcu_dereference_protected(*slot, 1))) { matchlen = longest_prefix_match(trie, node, key); if (node->prefixlen != matchlen || @@ -482,7 +482,7 @@ static long trie_delete_elem(struct bpf_map *map, void *_key) trim = &trie->root; trim2 = trim; parent = NULL; - while ((node = rcu_dereference(*trim))) { + while ((node = rcu_dereference_protected(*trim, 1))) { matchlen = longest_prefix_match(trie, node, key); if (node->prefixlen != matchlen || diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index 645bd30bc9a9d..d2cbab4bdf644 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -20,7 +20,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) /* Does not support >1 level map-in-map */ if (inner_map->inner_map_meta) return ERR_PTR(-EINVAL); - + if (inner_map->excl_prog_sha) + return ERR_PTR(-ENOTSUPP); if (!inner_map->ops->map_meta_equal) return ERR_PTR(-ENOTSUPP); @@ -101,6 +102,8 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, inner_map = __bpf_map_get(f); if (IS_ERR(inner_map)) return inner_map; + if (inner_map->excl_prog_sha) + return ERR_PTR(-ENOTSUPP); inner_map_meta = map->inner_map_meta; if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index 261a03ea73d34..ae0741a09c6de 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -112,6 +112,10 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, map = bpf_map_get_with_uref(linfo->map.map_fd); if (IS_ERR(map)) return PTR_ERR(map); + if (map->excl_prog_sha) { + err = -EPERM; + goto put_map; + } if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 8478d2c6ed5b6..1dd99b3194d7c 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -437,12 +437,10 @@ static void __clean_func_state(struct bpf_verifier_env *env, continue; /* - * Only destroy spilled_ptr when hi half is dead. - * If hi half is still live with STACK_SPILL, the - * spilled_ptr metadata is needed for correct state - * comparison in stacksafe(). - * is_spilled_reg() is using slot_type[7], but - * is_spilled_scalar_after() check either slot_type[0] or [4] + * Only scalar spills can be degraded to raw stack bytes + * when their high half is dead. Pointer spills need the + * saved spilled_ptr metadata so partial fills keep + * rejecting as non-scalar register fills. */ if (!hi_live) { struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; @@ -450,6 +448,9 @@ static void __clean_func_state(struct bpf_verifier_env *env, if (lo_live && stype == STACK_SPILL) { u8 val = STACK_MISC; + if (spill->type != SCALAR_VALUE) + continue; + /* * 8 byte spill of scalar 0 where half slot is dead * should become STACK_ZERO in lo 4 bytes. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 630d530782fe8..96d1b797e1523 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -807,6 +807,11 @@ void bpf_obj_free_task_work(const struct btf_record *rec, void *obj) bpf_task_work_cancel_and_free(obj + rec->task_work_off); } +void bpf_obj_cancel_fields(struct bpf_map *map, void *obj) +{ + bpf_map_free_internal_structs(map, obj); +} + void bpf_obj_free_fields(const struct btf_record *rec, void *obj) { const struct btf_field *fields; @@ -1572,6 +1577,13 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) err = -EFAULT; goto free_map; } + + /* See libbpf: emit_signature_match() */ + BUILD_BUG_ON(offsetof(struct bpf_map, excl) != SHA256_DIGEST_SIZE); + BUILD_BUG_ON(!__same_type(map->excl, u32)); + BUILD_BUG_ON(offsetof(struct bpf_map, sha) != 0); + BUILD_BUG_ON(!__same_type(map->sha, u8[SHA256_DIGEST_SIZE])); + map->excl = 1; } else if (attr->excl_prog_hash_size) { err = -EINVAL; goto free_map; @@ -4281,6 +4293,11 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog, if (!btp) return -ENOENT; + if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) { + bpf_put_raw_tracepoint(btp); + return -EINVAL; + } + link = kzalloc_obj(*link, GFP_USER); if (!link) { err = -ENOMEM; @@ -4654,7 +4671,7 @@ static int bpf_prog_detach(const union bpf_attr *attr) #define BPF_PROG_QUERY_LAST_FIELD query.revision static int bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { if (!bpf_net_capable()) return -EPERM; @@ -4693,7 +4710,7 @@ static int bpf_prog_query(const union bpf_attr *attr, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: - return cgroup_bpf_prog_query(attr, uattr); + return cgroup_bpf_prog_query(attr, uattr, uattr_size); case BPF_LIRC_MODE2: return lirc_prog_query(attr, uattr); case BPF_FLOW_DISSECTOR: @@ -5045,10 +5062,11 @@ static int bpf_prog_get_info_by_fd(struct file *file, u32 info_len = attr->info.info_len; struct bpf_prog_kstats stats; char __user *uinsns; - u32 ulen; + u32 ulen, len; int err; - err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); + len = offsetofend(struct bpf_prog_info, attach_btf_id); + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len); if (err) return err; info_len = min_t(u32, sizeof(info), info_len); @@ -5330,10 +5348,11 @@ static int bpf_map_get_info_by_fd(struct file *file, { struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_map_info info; - u32 info_len = attr->info.info_len; + u32 info_len = attr->info.info_len, len; int err; - err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); + len = offsetofend(struct bpf_map_info, hash_size); + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len); if (err) return err; info_len = min_t(u32, sizeof(info), info_len); @@ -6189,7 +6208,7 @@ static int prog_stream_read(union bpf_attr *attr) return ret; } -#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.prog_fd +#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.flags static int prog_assoc_struct_ops(union bpf_attr *attr) { @@ -6286,7 +6305,7 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) err = bpf_prog_detach(&attr); break; case BPF_PROG_QUERY: - err = bpf_prog_query(&attr, uattr.user); + err = bpf_prog_query(&attr, uattr.user, size); break; case BPF_PROG_TEST_RUN: err = bpf_prog_test_run(&attr, uattr.user); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7fb88e1cd7c4d..523f00e609e2b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3836,7 +3836,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bool sanitize = reg && is_spillable_regtype(reg->type); for (i = 0; i < size; i++) { - u8 type = state->stack[spi].slot_type[i]; + u8 type = state->stack[spi].slot_type[(slot - i) % + BPF_REG_SIZE]; if (type != STACK_MISC && type != STACK_ZERO) { sanitize = true; @@ -5558,14 +5559,11 @@ static int check_max_stack_depth(struct bpf_verifier_env *env) static int __check_buffer_access(struct bpf_verifier_env *env, const char *buf_info, const struct bpf_reg_state *reg, - int regno, int off, int size) + int regno, int off, int size, + u32 *access_end) { - if (off < 0) { - verbose(env, - "R%d invalid %s buffer access: off=%d, size=%d\n", - regno, buf_info, off, size); - return -EACCES; - } + s64 start; + if (!tnum_is_const(reg->var_off)) { char tn_buf[48]; @@ -5576,6 +5574,15 @@ static int __check_buffer_access(struct bpf_verifier_env *env, return -EACCES; } + start = (s64)reg->var_off.value + off; + if (start < 0) { + verbose(env, + "R%d invalid negative %s buffer offset: off=%d, var_off=%lld\n", + regno, buf_info, off, (s64)reg->var_off.value); + return -EACCES; + } + + *access_end = start + size; return 0; } @@ -5583,14 +5590,14 @@ static int check_tp_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno, int off, int size) { + u32 access_end; int err; - err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); + err = __check_buffer_access(env, "tracepoint", reg, regno, off, size, &access_end); if (err) return err; - env->prog->aux->max_tp_access = max(reg->var_off.value + off + size, - env->prog->aux->max_tp_access); + env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access); return 0; } @@ -5602,13 +5609,14 @@ static int check_buffer_access(struct bpf_verifier_env *env, u32 *max_access) { const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; + u32 access_end; int err; - err = __check_buffer_access(env, buf_info, reg, regno, off, size); + err = __check_buffer_access(env, buf_info, reg, regno, off, size, &access_end); if (err) return err; - *max_access = max(reg->var_off.value + off + size, *max_access); + *max_access = max(access_end, *max_access); return 0; } @@ -6453,6 +6461,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn */ if (info.reg_type == SCALAR_VALUE) { if (info.is_retval && get_func_retval_range(env->prog, &range)) { + mark_reg_unknown(env, regs, value_regno); err = __mark_reg_s32_range(env, regs, value_regno, range.minval, range.maxval); if (err) @@ -8471,7 +8480,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, return 0; } -static bool can_elide_value_nullness(enum bpf_map_type type); +static bool can_elide_value_nullness(const struct bpf_map *map); static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, @@ -8621,7 +8630,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL); if (err) return err; - if (can_elide_value_nullness(meta->map.ptr->map_type)) { + if (can_elide_value_nullness(meta->map.ptr)) { err = get_constant_map_key(env, reg, key_size, &meta->const_map_key); if (err < 0) { meta->const_map_key = -1; @@ -10221,13 +10230,16 @@ static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno state->callback_subprogno == subprogno); } -/* Returns whether or not the given map type can potentially elide +/* Returns whether or not the given map can potentially elide * lookup return value nullness check. This is possible if the key * is statically known. */ -static bool can_elide_value_nullness(enum bpf_map_type type) +static bool can_elide_value_nullness(const struct bpf_map *map) { - switch (type) { + if (map->map_flags & BPF_F_INNER_MAP) + return false; + + switch (map->map_type) { case BPF_MAP_TYPE_ARRAY: case BPF_MAP_TYPE_PERCPU_ARRAY: return true; @@ -10589,7 +10601,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } if (func_id == BPF_FUNC_map_lookup_elem && - can_elide_value_nullness(meta.map.ptr->map_type) && + can_elide_value_nullness(meta.map.ptr) && meta.const_map_key >= 0 && meta.const_map_key < meta.map.ptr->max_entries) ret_flag &= ~PTR_MAYBE_NULL; @@ -19314,6 +19326,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, btp = bpf_get_raw_tracepoint(tname); if (!btp) return -EINVAL; + if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) { + bpf_log(log, "Sleepable program cannot attach to non-faultable tracepoint %s\n", + tname); + bpf_put_raw_tracepoint(btp); + return -EINVAL; + } fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL, trace_symbol); bpf_put_raw_tracepoint(btp); @@ -19530,14 +19548,19 @@ static bool can_be_sleepable(struct bpf_prog *prog) case BPF_MODIFY_RETURN: case BPF_TRACE_ITER: case BPF_TRACE_FSESSION: + case BPF_TRACE_RAW_TP: return true; default: return false; } } - return prog->type == BPF_PROG_TYPE_LSM || - prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || - prog->type == BPF_PROG_TYPE_STRUCT_OPS; + if (prog->type == BPF_PROG_TYPE_LSM) + return prog->expected_attach_type != BPF_LSM_CGROUP; + + return prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || + prog->type == BPF_PROG_TYPE_STRUCT_OPS || + prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || + prog->type == BPF_PROG_TYPE_TRACEPOINT; } static int check_attach_btf_id(struct bpf_verifier_env *env) @@ -19559,7 +19582,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) } if (prog->sleepable && !can_be_sleepable(prog)) { - verbose(env, "Only fentry/fexit/fsession/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n"); + verbose(env, "Program of this type cannot be sleepable\n"); return -EINVAL; } @@ -20206,13 +20229,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (!is_priv) mutex_unlock(&bpf_verifier_lock); bpf_clear_insn_aux_data(env, 0, env->prog->len); - vfree(env->insn_aux_data); err_free_env: bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + vfree(env->insn_aux_data); kvfree(env); return ret; } diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index c9e14fda3d6fd..8407bf44e807b 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -2650,7 +2650,12 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs) migrate = is_memory_migrate(cs); - mpol_rebind_mm(mm, &cs->mems_allowed); + /* + * For v1 we can have empty effective_mems, but we cannot + * attach any tasks (see cpuset_can_attach_check()). For v2, + * effective_mems is guaranteed to not be empty. + */ + mpol_rebind_mm(mm, &cs->effective_mems); if (migrate) cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); else diff --git a/kernel/cpu.c b/kernel/cpu.c index bc4f7a9ba64e6..b9acc261b1279 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -174,7 +174,7 @@ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, struct cpuhp_step *step = cpuhp_get_step(state); int (*cbm)(unsigned int cpu, struct hlist_node *node); int (*cb)(unsigned int cpu); - int ret, cnt; + int ret, cnt, rollback_ret; if (st->fail == state) { st->fail = CPUHP_INVALID; @@ -238,12 +238,12 @@ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, break; trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); - ret = cbm(cpu, node); - trace_cpuhp_exit(cpu, st->state, state, ret); + rollback_ret = cbm(cpu, node); + trace_cpuhp_exit(cpu, st->state, state, rollback_ret); /* * Rollback must not fail, */ - WARN_ON_ONCE(ret); + WARN_ON_ONCE(rollback_ret); } return ret; } @@ -2854,21 +2854,17 @@ static const struct attribute_group cpuhp_cpu_attr_group = { .name = "hotplug", }; -static ssize_t states_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t states_show(struct device *dev, struct device_attribute *attr, char *buf) { - ssize_t cur, res = 0; + ssize_t res = 0; int i; mutex_lock(&cpuhp_state_mutex); for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { struct cpuhp_step *sp = cpuhp_get_step(i); - if (sp->name) { - cur = sprintf(buf, "%3d: %s\n", i, sp->name); - buf += cur; - res += cur; - } + if (sp->name) + res += sysfs_emit_at(buf, res, "%3d: %s\n", i, sp->name); } mutex_unlock(&cpuhp_state_mutex); return res; diff --git a/kernel/events/core.c b/kernel/events/core.c index 7935d5663944e..240684f4b18c1 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4728,7 +4728,7 @@ static void perf_remove_from_owner(struct perf_event *event); static void perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx, struct task_struct *task, - bool revoke); + unsigned long detach_flags); /* * Removes all events from the current task that have been marked @@ -4755,7 +4755,7 @@ static void perf_event_remove_on_exec(struct perf_event_context *ctx) modified = true; - perf_event_exit_event(event, ctx, ctx->task, false); + perf_event_exit_event(event, ctx, ctx->task, DETACH_GROUP); } raw_spin_lock_irqsave(&ctx->lock, flags); @@ -7150,6 +7150,8 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) int err = 0; unsigned long pagenum; + guard(mutex)(&rb->aux_mutex); + /* * We map this as a VM_PFNMAP VMA. * @@ -12900,7 +12902,7 @@ static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event, /* * De-schedule the event and mark it REVOKED. */ - perf_event_exit_event(event, ctx, ctx->task, true); + perf_event_exit_event(event, ctx, ctx->task, DETACH_REVOKE); /* * All _free_event() bits that rely on event->pmu: @@ -14488,12 +14490,13 @@ static void perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx, struct task_struct *task, - bool revoke) + unsigned long detach_flags) { struct perf_event *parent_event = event->parent; - unsigned long detach_flags = DETACH_EXIT; unsigned int attach_state; + detach_flags |= DETACH_EXIT; + if (parent_event) { /* * Do not destroy the 'original' grouping; because of the @@ -14516,8 +14519,8 @@ perf_event_exit_event(struct perf_event *event, sync_child_event(event, task); } - if (revoke) - detach_flags |= DETACH_GROUP | DETACH_REVOKE; + if (detach_flags & DETACH_REVOKE) + detach_flags |= DETACH_GROUP; perf_remove_from_context(event, detach_flags); /* @@ -14605,7 +14608,7 @@ static void perf_event_exit_task_context(struct task_struct *task, bool exit) perf_event_task(task, ctx, 0); list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry) - perf_event_exit_event(child_event, ctx, exit ? task : NULL, false); + perf_event_exit_event(child_event, ctx, exit ? task : NULL, 0); mutex_unlock(&ctx->mutex); diff --git a/kernel/exit.c b/kernel/exit.c index f50d73c272d6e..7ba7966a23b9a 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -212,7 +212,12 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts __unhash_process(post, tsk, group_dead); write_sequnlock(&sig->stats_lock); - tsk->sighand = NULL; + /* + * Ensure that all preceeding state is visible. Pairs with + * the smp_acquire__after_ctrl_dep() in the sighand == NULL + * path of lock_task_sighand(). + */ + smp_store_release(&tsk->sighand, NULL); spin_unlock(&sighand->siglock); __cleanup_sighand(sighand); diff --git a/kernel/fork.c b/kernel/fork.c index 8ac38beae360b..5a4623b058a20 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1004,6 +1004,11 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->mm_cid.active = 0; INIT_HLIST_NODE(&tsk->mm_cid.node); #endif + +#ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(tsk->bpf_storage, NULL); + tsk->bpf_ctx = NULL; +#endif return tsk; free_stack: @@ -2223,10 +2228,6 @@ __latent_entropy struct task_struct *copy_process( p->sequential_io = 0; p->sequential_io_avg = 0; #endif -#ifdef CONFIG_BPF_SYSCALL - RCU_INIT_POINTER(p->bpf_storage, NULL); - p->bpf_ctx = NULL; -#endif unwind_task_init(p); @@ -2314,6 +2315,7 @@ __latent_entropy struct task_struct *copy_process( #ifdef CONFIG_BLOCK p->plug = NULL; + p->flags &= ~PF_BLOCK_TS; #endif futex_init_task(p); diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index 1d99a84dc9adb..b597cb3d17fc1 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -643,12 +643,6 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1, continue; } - /* Self-deadlock: non-top waiter already owns the PI futex. */ - if (rt_mutex_owner(&pi_state->pi_mutex) == this->task) { - ret = -EDEADLK; - break; - } - ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, this->rt_waiter, this->task); diff --git a/kernel/kcov.c b/kernel/kcov.c index 0b369e88c7c9b..f3a5ae960ce51 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -1109,10 +1109,10 @@ static void __init selftest(void) * potentially traced functions in this region. */ start = jiffies; - current->kcov_mode = KCOV_MODE_TRACE_PC; + WRITE_ONCE(current->kcov_mode, KCOV_MODE_TRACE_PC); while ((jiffies - start) * MSEC_PER_SEC / HZ < 300) ; - current->kcov_mode = 0; + WRITE_ONCE(current->kcov_mode, 0); pr_err("done running self test\n"); } #endif diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index a43d2da0fe3e7..dc770b9a6d053 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1146,9 +1146,11 @@ int kernel_kexec(void) goto Unlock; } - error = liveupdate_reboot(); - if (error) - goto Unlock; + if (!kexec_image->preserve_context) { + error = liveupdate_reboot(); + if (error) + goto Unlock; + } #ifdef CONFIG_KEXEC_JUMP if (kexec_image->preserve_context) { diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 1b592d86dc489..48d98fa9621e3 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -593,20 +593,30 @@ early_param("kho_scratch", kho_parse_scratch_size); static void __init scratch_size_update(void) { - phys_addr_t size; + /* + * If fixed sizes are not provided via command line, calculate them + * now. + */ + if (scratch_scale) { + phys_addr_t size; - if (!scratch_scale) - return; + size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, + NUMA_NO_NODE); + size = size * scratch_scale / 100; + scratch_size_lowmem = size; - size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, - NUMA_NO_NODE); - size = size * scratch_scale / 100; - scratch_size_lowmem = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, + NUMA_NO_NODE); + size = size * scratch_scale / 100 - scratch_size_lowmem; + scratch_size_global = size; + } - size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, - NUMA_NO_NODE); - size = size * scratch_scale / 100 - scratch_size_lowmem; - scratch_size_global = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + /* + * Scratch areas are released as MIGRATE_CMA. Round them up to the right + * size. + */ + scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES); + scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES); } static phys_addr_t __init scratch_size_node(int nid) diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c index a0a419085e283..208987502f73a 100644 --- a/kernel/liveupdate/luo_file.c +++ b/kernel/liveupdate/luo_file.c @@ -385,10 +385,11 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set) args.private_data = luo_file->private_data; luo_file->fh->ops->unpreserve(&args); luo_flb_file_unpreserve(luo_file->fh); - module_put(luo_file->fh->ops->owner); xa_erase(&luo_preserved_files, luo_get_id(luo_file->fh, luo_file->file)); + module_put(luo_file->fh->ops->owner); + list_del(&luo_file->list); file_set->count--; @@ -677,7 +678,6 @@ static void luo_file_finish_one(struct luo_file_set *file_set, luo_file->fh->ops->finish(&args); luo_flb_file_finish(luo_file->fh); - module_put(luo_file->fh->ops->owner); } /** @@ -738,6 +738,7 @@ int luo_file_finish(struct luo_file_set *file_set) luo_get_id(luo_file->fh, luo_file->file)); fput(luo_file->file); } + module_put(luo_file->fh->ops->owner); list_del(&luo_file->list); file_set->count--; mutex_destroy(&luo_file->mutex); diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 00f5494812c4a..8f5c5dd01cd04 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -111,7 +111,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - if (!private->outgoing.count) { + if (!refcount_read(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; int err; @@ -126,8 +126,10 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) } private->outgoing.data = args.data; private->outgoing.obj = args.obj; + refcount_set(&private->outgoing.count, 1); + } else { + refcount_inc(&private->outgoing.count); } - private->outgoing.count++; } return 0; @@ -138,8 +140,7 @@ static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - private->outgoing.count--; - if (!private->outgoing.count) { + if (refcount_dec_and_test(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; args.flb = flb; @@ -164,7 +165,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) bool found = false; int err; - guard(mutex)(&private->incoming.lock); + lockdep_assert_held(&private->incoming.lock); if (private->incoming.finished) return -ENODATA; @@ -178,7 +179,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) for (int i = 0; i < fh->header_ser->count; i++) { if (!strcmp(fh->ser[i].name, flb->compatible)) { private->incoming.data = fh->ser[i].data; - private->incoming.count = fh->ser[i].count; + refcount_set(&private->incoming.count, fh->ser[i].count); found = true; break; } @@ -205,16 +206,14 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) return 0; } -static void luo_flb_file_finish_one(struct liveupdate_flb *flb) +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); - u64 count; - - scoped_guard(mutex, &private->incoming.lock) - count = --private->incoming.count; + struct liveupdate_flb_op_args args = {0}; - if (!count) { - struct liveupdate_flb_op_args args = {0}; + scoped_guard(mutex, &private->incoming.lock) { + if (!refcount_dec_and_test(&private->incoming.count)) + return; if (!private->incoming.retrieved) { int err = luo_flb_retrieve_one(flb); @@ -223,16 +222,14 @@ static void luo_flb_file_finish_one(struct liveupdate_flb *flb) return; } - scoped_guard(mutex, &private->incoming.lock) { - args.flb = flb; - args.obj = private->incoming.obj; - flb->ops->finish(&args); + args.flb = flb; + args.obj = private->incoming.obj; + flb->ops->finish(&args); - private->incoming.data = 0; - private->incoming.obj = NULL; - private->incoming.finished = true; - module_put(flb->ops->owner); - } + private->incoming.data = 0; + private->incoming.obj = NULL; + private->incoming.finished = true; + module_put(flb->ops->owner); } } @@ -315,7 +312,7 @@ void luo_flb_file_finish(struct liveupdate_file_handler *fh) guard(rwsem_read)(&luo_register_rwlock); list_for_each_entry_reverse(iter, flb_list, list) - luo_flb_file_finish_one(iter->flb); + liveupdate_flb_put_incoming(iter->flb); } static void luo_flb_unregister_one(struct liveupdate_file_handler *fh, @@ -512,6 +509,8 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) if (!liveupdate_enabled()) return -EOPNOTSUPP; + guard(mutex)(&private->incoming.lock); + if (!private->incoming.obj) { int err = luo_flb_retrieve_one(flb); @@ -519,7 +518,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) return err; } - guard(mutex)(&private->incoming.lock); + refcount_inc(&private->incoming.count); *objp = private->incoming.obj; return 0; @@ -652,12 +651,13 @@ void luo_flb_serialize(void) guard(rwsem_read)(&luo_register_rwlock); list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) { struct luo_flb_private *private = luo_flb_get_private(gflb); + long count = refcount_read(&private->outgoing.count); - if (private->outgoing.count > 0) { + if (count > 0) { strscpy(fh->ser[i].name, gflb->compatible, sizeof(fh->ser[i].name)); fh->ser[i].data = private->outgoing.data; - fh->ser[i].count = private->outgoing.count; + fh->ser[i].count = count; i++; } } diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 7a42385dabe27..174de2cc56a6b 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -46,6 +46,38 @@ * 4. Retrieval: A userspace agent in the new kernel can then call * `luo_session_retrieve()` with a session name to get a new file * descriptor and access the preserved state. + * + * Locking: + * + * The LUO session subsystem uses a three-tier locking hierarchy to ensure thread + * safety and prevent deadlocks during concurrent session mutations and kexec + * serialization: + * + * 1. `luo_session_serialize_rwsem` (global rwsem): + * Protects session mutations (creation, retrieval, release, and ioctls) + * against the serialization process during reboot. + * + * - Readers: Taken by any path modifying or accessing session state (e.g., + * `luo_session_create()`, `luo_session_retrieve()`, `luo_session_release()`, + * and `luo_session_ioctl()`). + * - Writer: Taken by the serialization process (`luo_session_serialize()`) + * during reboot. On success, the write lock is held indefinitely to freeze + * the subsystem. On failure, it is released to allow recovery. + * + * 2. `luo_session_header->rwsem` (per-list rwsem): + * Synchronizes list-level operations for the incoming and outgoing session headers. + * + * - Writer: Taken during list mutation operations (inserting or removing a + * session from the list). + * - Reader: Taken when traversing the list (e.g., retrieving a session by name). + * + * 3. `luo_session->mutex` (per-session mutex): + * Protects the internal state and file sets of an individual session. It is + * acquired during per-session operations such as preserving, retrieving, + * or freezing files. + * + * Lock Hierarchy: + * `luo_session_serialize_rwsem` -> `luo_session_header->rwsem` -> `luo_session->mutex` */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -75,6 +107,8 @@ sizeof(struct luo_session_header_ser)) / \ sizeof(struct luo_session_ser)) +static DECLARE_RWSEM(luo_session_serialize_rwsem); + /** * struct luo_session_header - Header struct for managing LUO sessions. * @count: The number of sessions currently tracked in the @list. @@ -205,6 +239,7 @@ static int luo_session_release(struct inode *inodep, struct file *filep) struct luo_session *session = filep->private_data; struct luo_session_header *sh; + guard(rwsem_read)(&luo_session_serialize_rwsem); /* If retrieved is set, it means this session is from incoming list */ if (session->retrieved) { int err = luo_session_finish_one(session); @@ -295,32 +330,58 @@ union ucmd_buffer { struct liveupdate_session_retrieve_fd retrieve; }; +/* Type of sessions the ioctl applies to. */ +enum luo_ioctl_type { + LUO_IOCTL_INCOMING, + LUO_IOCTL_OUTGOING, + LUO_IOCTL_ALL, +}; + struct luo_ioctl_op { unsigned int size; unsigned int min_size; unsigned int ioctl_num; + enum luo_ioctl_type type; int (*execute)(struct luo_session *session, struct luo_ucmd *ucmd); }; -#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ +#define IOCTL_OP(_ioctl, _fn, _struct, _last, _type) \ [_IOC_NR(_ioctl) - LIVEUPDATE_CMD_SESSION_BASE] = { \ .size = sizeof(_struct) + \ BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \ sizeof(_struct)), \ .min_size = offsetofend(_struct, _last), \ .ioctl_num = _ioctl, \ + .type = _type, \ .execute = _fn, \ } static const struct luo_ioctl_op luo_session_ioctl_ops[] = { IOCTL_OP(LIVEUPDATE_SESSION_FINISH, luo_session_finish, - struct liveupdate_session_finish, reserved), + struct liveupdate_session_finish, reserved, LUO_IOCTL_INCOMING), IOCTL_OP(LIVEUPDATE_SESSION_PRESERVE_FD, luo_session_preserve_fd, - struct liveupdate_session_preserve_fd, token), + struct liveupdate_session_preserve_fd, token, LUO_IOCTL_OUTGOING), IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd, - struct liveupdate_session_retrieve_fd, token), + struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_INCOMING), }; +static bool luo_ioctl_type_valid(struct luo_session *session, + const struct luo_ioctl_op *op) +{ + switch (op->type) { + case LUO_IOCTL_INCOMING: + /* Retrieved is only set on incoming sessions */ + return session->retrieved; + case LUO_IOCTL_OUTGOING: + return !session->retrieved; + case LUO_IOCTL_ALL: + return true; + } + + /* Catch-all. */ + return false; +} + static long luo_session_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { @@ -345,6 +406,8 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, op = &luo_session_ioctl_ops[nr - LIVEUPDATE_CMD_SESSION_BASE]; if (op->ioctl_num != cmd) return -ENOIOCTLCMD; + if (!luo_ioctl_type_valid(session, op)) + return -EINVAL; if (ucmd.user_size < op->min_size) return -EINVAL; @@ -354,6 +417,7 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, if (ret) return ret; + guard(rwsem_read)(&luo_session_serialize_rwsem); return op->execute(session, &ucmd); } @@ -382,21 +446,28 @@ static int luo_session_getfile(struct luo_session *session, struct file **filep) int luo_session_create(const char *name, struct file **filep) { + size_t len = strnlen(name, LIVEUPDATE_SESSION_NAME_LENGTH); struct luo_session *session; int err; + if (len == 0 || len > LIVEUPDATE_SESSION_NAME_LENGTH - 1) + return -EINVAL; + session = luo_session_alloc(name); if (IS_ERR(session)) return PTR_ERR(session); + down_read(&luo_session_serialize_rwsem); err = luo_session_insert(&luo_session_global.outgoing, session); if (err) goto err_free; - scoped_guard(mutex, &session->mutex) - err = luo_session_getfile(session, filep); + mutex_lock(&session->mutex); + err = luo_session_getfile(session, filep); + mutex_unlock(&session->mutex); if (err) goto err_remove; + up_read(&luo_session_serialize_rwsem); return 0; @@ -404,6 +475,7 @@ int luo_session_create(const char *name, struct file **filep) luo_session_remove(&luo_session_global.outgoing, session); err_free: luo_session_free(session); + up_read(&luo_session_serialize_rwsem); return err; } @@ -415,12 +487,12 @@ int luo_session_retrieve(const char *name, struct file **filep) struct luo_session *it; int err; - scoped_guard(rwsem_read, &sh->rwsem) { - list_for_each_entry(it, &sh->list, list) { - if (!strncmp(it->name, name, sizeof(it->name))) { - session = it; - break; - } + guard(rwsem_read)(&luo_session_serialize_rwsem); + guard(rwsem_read)(&sh->rwsem); + list_for_each_entry(it, &sh->list, list) { + if (!strncmp(it->name, name, sizeof(it->name))) { + session = it; + break; } } @@ -588,7 +660,8 @@ int luo_session_serialize(void) int i = 0; int err; - guard(rwsem_write)(&sh->rwsem); + down_write(&luo_session_serialize_rwsem); + down_write(&sh->rwsem); list_for_each_entry(session, &sh->list, list) { err = luo_session_freeze_one(session, &sh->ser[i]); if (err) @@ -599,6 +672,7 @@ int luo_session_serialize(void) i++; } sh->header_ser->count = sh->count; + up_write(&sh->rwsem); return 0; @@ -608,7 +682,8 @@ int luo_session_serialize(void) luo_session_unfreeze_one(session, &sh->ser[i]); memset(sh->ser[i].name, 0, sizeof(sh->ser[i].name)); } + up_write(&sh->rwsem); + up_write(&luo_session_serialize_rwsem); return err; } - diff --git a/kernel/locking/spinlock_rt.c b/kernel/locking/spinlock_rt.c index db1e11b45de67..1d5e1b3c60bfa 100644 --- a/kernel/locking/spinlock_rt.c +++ b/kernel/locking/spinlock_rt.c @@ -79,10 +79,27 @@ void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU) { spin_release(&lock->dep_map, _RET_IP_); migrate_enable(); - rcu_read_unlock(); if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL))) rt_mutex_slowunlock(&lock->lock); + + /* + * This must be last to prevent the following UAF: + * + * T1 T2 + * spin_lock(&p->lock); rcu_read_lock(); + * invalidate(p); p = rcu_dereference(ptr); + * rcu_assign_pointer(ptr, NULL); if (!p) return; + * spin_unlock(&p->lock); spin_lock(&p->lock); + * kfree_rcu(p); rcu_read_unlock(); + * .... + * spin_unlock(&p->lock) + * rcu_read_unlock(); // Ends grace period + * rcu_do_batch() + * kfree(p); + * UAF -> rt_mutex_cmpxchg_release(&p->lock.lock...) + */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_spin_unlock); @@ -262,17 +279,21 @@ void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU) { rwlock_release(&rwlock->dep_map, _RET_IP_); migrate_enable(); - rcu_read_unlock(); rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT); + + /* This must be last. See comment in rt_spin_unlock() */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_read_unlock); void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU) { rwlock_release(&rwlock->dep_map, _RET_IP_); - rcu_read_unlock(); migrate_enable(); rwbase_write_unlock(&rwlock->rwbase); + + /* This must be last. See comment in rt_spin_unlock() */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_write_unlock); diff --git a/kernel/module/decompress.c b/kernel/module/decompress.c index 36f52a232a120..cce098671be97 100644 --- a/kernel/module/decompress.c +++ b/kernel/module/decompress.c @@ -307,6 +307,8 @@ int module_decompress(struct load_info *info, const void *buf, size_t size) */ n_pages = DIV_ROUND_UP(size, PAGE_SIZE) * 2; error = module_extend_max_pages(info, n_pages); + if (error) + return error; data_size = MODULE_DECOMPRESS_FN(info, buf, size); if (data_size < 0) { diff --git a/kernel/power/qos.c b/kernel/power/qos.c index 398b994b73aab..1944dbeb0d4c9 100644 --- a/kernel/power/qos.c +++ b/kernel/power/qos.c @@ -519,18 +519,23 @@ static int __init cpu_latency_qos_init(void) int ret; ret = misc_register(&cpu_latency_qos_miscdev); - if (ret < 0) + if (ret < 0) { pr_err("%s: %s setup failed\n", __func__, cpu_latency_qos_miscdev.name); + return ret; + } #ifdef CONFIG_PM_QOS_CPU_SYSTEM_WAKEUP ret = misc_register(&cpu_wakeup_latency_qos_miscdev); - if (ret < 0) + if (ret < 0) { pr_err("%s: %s setup failed\n", __func__, cpu_wakeup_latency_qos_miscdev.name); + misc_deregister(&cpu_latency_qos_miscdev); + return ret; + } #endif - return ret; + return 0; } late_initcall(cpu_latency_qos_init); #endif /* CONFIG_CPU_IDLE */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b8871449d3c69..69c2aa8c62466 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4710,6 +4710,7 @@ int sched_fork(u64 clone_flags, struct task_struct *p) p->policy = SCHED_NORMAL; p->static_prio = NICE_TO_PRIO(0); p->rt_priority = 0; + p->timer_slack_ns = p->default_timer_slack_ns; } else if (PRIO_TO_NICE(p->static_prio) < 0) p->static_prio = NICE_TO_PRIO(0); @@ -5252,6 +5253,12 @@ static struct rq *finish_task_switch(struct task_struct *prev) */ kmap_local_sched_in(); + /* + * Any cached block-layer timestamp (plug->cur_ktime) is stale now, + * invalidate it. + */ + blk_plug_invalidate_ts(); + fire_sched_in_preempt_notifiers(current); /* * When switching through a kernel thread, the loop in @@ -7251,12 +7258,10 @@ static inline void sched_submit_work(struct task_struct *tsk) static void sched_update_worker(struct task_struct *tsk) { - if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { - if (tsk->flags & PF_BLOCK_TS) - blk_plug_invalidate_ts(tsk); + if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { if (tsk->flags & PF_WQ_WORKER) wq_worker_running(tsk); - else if (tsk->flags & PF_IO_WORKER) + else io_wq_worker_running(tsk); } } @@ -10876,8 +10881,19 @@ static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm) } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) { unsigned int cid = rq->curr->mm_cid.cid; - /* Ensure it has the transition bit set */ - if (!cid_in_transit(cid)) { + /* + * Set the transition bit only on a genuine task-owned + * CID. A running active task can legitimately have + * MM_CID_UNSET here: in per-CPU mode CIDs are assigned + * lazily on schedule-in, so the fork()/execve() window + * leaves the task active with no owned CID. Setting the + * transition bit on MM_CID_UNSET would later feed + * clear_bit() an out-of-bounds bit number via + * mm_cid_schedout(), so exclude it. A CPU-owned + * (MM_CID_ONCPU) CID is handled by the cid_on_cpu() + * branch above and never reaches here. + */ + if (cid != MM_CID_UNSET && !cid_in_transit(cid)) { cid = cid_to_transit_cid(cid); rq->curr->mm_cid.cid = cid; pcp->cid = cid; diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index ae9fd211cec1f..a4e689eefdfbd 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -486,6 +486,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, sg_cpu->util, max_cap); + sg_policy->need_freq_update = false; sg_policy->last_freq_update_time = time; } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 74c1617cf6523..6f74bde684376 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -373,6 +373,9 @@ static ssize_t sched_server_write_common(struct file *filp, const char __user *u return -EINVAL; } + if (!cpu_online(cpu_of(rq))) + return -EBUSY; + update_rq_clock(rq); dl_server_stop(dl_se); retval = dl_server_apply_params(dl_se, runtime, period, 0); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3ebec186f9823..abb76775aa728 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4959,13 +4959,86 @@ static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s trace_pelt_cfs_tp(cfs_rq); } +#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) + +static inline void util_est_update(struct sched_entity *se) +{ + unsigned int ewma, dequeued, last_ewma_diff; + + if (!sched_feat(UTIL_EST)) + return; + + /* Get current estimate of utilization */ + ewma = READ_ONCE(se->avg.util_est); + + /* + * If the PELT values haven't changed since enqueue time, + * skip the util_est update. + */ + if (ewma & UTIL_AVG_UNCHANGED) + return; + + /* Get utilization at dequeue */ + dequeued = READ_ONCE(se->avg.util_avg); + + /* + * Reset EWMA on utilization increases, the moving average is used only + * to smooth utilization decreases. + */ + if (ewma <= dequeued) { + ewma = dequeued; + goto done; + } + + /* + * Skip update of task's estimated utilization when its members are + * already ~1% close to its last activation value. + */ + last_ewma_diff = ewma - dequeued; + if (last_ewma_diff < UTIL_EST_MARGIN) + goto done; + + /* + * To avoid underestimate of task utilization, skip updates of EWMA if + * we cannot grant that thread got all CPU time it wanted. + */ + if ((dequeued + UTIL_EST_MARGIN) < READ_ONCE(se->avg.runnable_avg)) + goto done; + + /* + * Update Task's estimated utilization + * + * When *p completes an activation we can consolidate another sample + * of the task size. This is done by using this value to update the + * Exponential Weighted Moving Average (EWMA): + * + * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) + * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) + * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) + * = w * ( -last_ewma_diff ) + ewma(t-1) + * = w * (-last_ewma_diff + ewma(t-1) / w) + * + * Where 'w' is the weight of new samples, which is configured to be + * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) + */ + ewma <<= UTIL_EST_WEIGHT_SHIFT; + ewma -= last_ewma_diff; + ewma >>= UTIL_EST_WEIGHT_SHIFT; +done: + ewma |= UTIL_AVG_UNCHANGED; + WRITE_ONCE(se->avg.util_est, ewma); + + trace_sched_util_est_se_tp(se); +} + /* * Optional action to be done while updating the load average */ -#define UPDATE_TG 0x1 -#define SKIP_AGE_LOAD 0x2 -#define DO_ATTACH 0x4 -#define DO_DETACH 0x8 +#define UPDATE_TG 0x01 +#define SKIP_AGE_LOAD 0x02 +#define DO_ATTACH 0x04 +#define DO_DETACH 0x08 +#define UPDATE_UTIL_EST 0x10 /* Update task and its cfs_rq load average */ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) @@ -5008,6 +5081,9 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s if (flags & UPDATE_TG) update_tg_load_avg(cfs_rq); } + + if (flags & UPDATE_UTIL_EST) + util_est_update(se); } /* @@ -5066,11 +5142,6 @@ static inline unsigned long task_util(struct task_struct *p) return READ_ONCE(p->se.avg.util_avg); } -static inline unsigned long task_runnable(struct task_struct *p) -{ - return READ_ONCE(p->se.avg.runnable_avg); -} - static inline unsigned long _task_util_est(struct task_struct *p) { return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; @@ -5113,88 +5184,6 @@ static inline void util_est_dequeue(struct cfs_rq *cfs_rq, trace_sched_util_est_cfs_tp(cfs_rq); } -#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) - -static inline void util_est_update(struct cfs_rq *cfs_rq, - struct task_struct *p, - bool task_sleep) -{ - unsigned int ewma, dequeued, last_ewma_diff; - - if (!sched_feat(UTIL_EST)) - return; - - /* - * Skip update of task's estimated utilization when the task has not - * yet completed an activation, e.g. being migrated. - */ - if (!task_sleep) - return; - - /* Get current estimate of utilization */ - ewma = READ_ONCE(p->se.avg.util_est); - - /* - * If the PELT values haven't changed since enqueue time, - * skip the util_est update. - */ - if (ewma & UTIL_AVG_UNCHANGED) - return; - - /* Get utilization at dequeue */ - dequeued = task_util(p); - - /* - * Reset EWMA on utilization increases, the moving average is used only - * to smooth utilization decreases. - */ - if (ewma <= dequeued) { - ewma = dequeued; - goto done; - } - - /* - * Skip update of task's estimated utilization when its members are - * already ~1% close to its last activation value. - */ - last_ewma_diff = ewma - dequeued; - if (last_ewma_diff < UTIL_EST_MARGIN) - goto done; - - /* - * To avoid underestimate of task utilization, skip updates of EWMA if - * we cannot grant that thread got all CPU time it wanted. - */ - if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p)) - goto done; - - - /* - * Update Task's estimated utilization - * - * When *p completes an activation we can consolidate another sample - * of the task size. This is done by using this value to update the - * Exponential Weighted Moving Average (EWMA): - * - * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) - * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) - * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) - * = w * ( -last_ewma_diff ) + ewma(t-1) - * = w * (-last_ewma_diff + ewma(t-1) / w) - * - * Where 'w' is the weight of new samples, which is configured to be - * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) - */ - ewma <<= UTIL_EST_WEIGHT_SHIFT; - ewma -= last_ewma_diff; - ewma >>= UTIL_EST_WEIGHT_SHIFT; -done: - ewma |= UTIL_AVG_UNCHANGED; - WRITE_ONCE(p->se.avg.util_est, ewma); - - trace_sched_util_est_se_tp(&p->se); -} - static inline unsigned long get_actual_cpu_capacity(int cpu) { unsigned long capacity = arch_scale_cpu_capacity(cpu); @@ -5647,7 +5636,7 @@ static bool dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { bool sleep = flags & DEQUEUE_SLEEP; - int action = UPDATE_TG; + int action = 0; update_curr(cfs_rq); clear_buddies(cfs_rq, se); @@ -5667,15 +5656,23 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) if (sched_feat(DELAY_DEQUEUE) && delay && !entity_eligible(cfs_rq, se)) { - update_load_avg(cfs_rq, se, 0); + if (entity_is_task(se)) + action |= UPDATE_UTIL_EST; + update_load_avg(cfs_rq, se, action); update_entity_lag(cfs_rq, se); set_delayed(se); return false; } } - if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) - action |= DO_DETACH; + action = UPDATE_TG; + if (entity_is_task(se)) { + if (task_on_rq_migrating(task_of(se))) + action |= DO_DETACH; + + if (sleep && !(flags & DEQUEUE_DELAYED)) + action |= UPDATE_UTIL_EST; + } /* * When dequeuing a sched_entity, we must: @@ -7438,7 +7435,6 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!p->se.sched_delayed) util_est_dequeue(&rq->cfs, p); - util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP); if (dequeue_entities(rq, &p->se, flags) < 0) return false; @@ -8260,25 +8256,32 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) static unsigned long cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) { + bool add_task = p && task_cpu(p) != cpu && dst_cpu == cpu; + bool sub_task = p && task_cpu(p) == cpu && dst_cpu != cpu; struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); unsigned long runnable; - if (boost) { - runnable = READ_ONCE(cfs_rq->avg.runnable_avg); - util = max(util, runnable); - } - /* * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its * contribution. If @p migrates from another CPU to @cpu add its * contribution. In all the other cases @cpu is not impacted by the * migration so its util_avg is already correct. */ - if (p && task_cpu(p) == cpu && dst_cpu != cpu) - lsub_positive(&util, task_util(p)); - else if (p && task_cpu(p) != cpu && dst_cpu == cpu) + if (add_task) util += task_util(p); + else if (sub_task) + lsub_positive(&util, task_util(p)); + + if (boost) { + runnable = READ_ONCE(cfs_rq->avg.runnable_avg); + if (add_task) + runnable += READ_ONCE(p->se.avg.runnable_avg); + else if (sub_task) + lsub_positive(&runnable, + READ_ONCE(p->se.avg.runnable_avg)); + util = max(util, runnable); + } if (sched_feat(UTIL_EST)) { unsigned long util_est; diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 84c4fe3abd743..8f0dee8fc4755 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -110,8 +110,16 @@ SCHED_FEAT(WARN_DOUBLE_CLOCK, false) * rq lock and possibly create a large contention, sending an * IPI to that CPU and let that CPU push the RT task to where * it should go may be a better scenario. + * + * This is best for PREEMPT_RT, but for non-RT it can cause issues + * when preemption is disabled for long periods of time. Have + * it only default enabled for PREEMPT_RT. */ +# ifdef CONFIG_PREEMPT_RT SCHED_FEAT(RT_PUSH_IPI, true) +# else +SCHED_FEAT(RT_PUSH_IPI, false) +# endif #endif SCHED_FEAT(RT_RUNTIME_SHARE, false) diff --git a/kernel/signal.c b/kernel/signal.c index 9c2b32c4d7553..bbc0fd4cc4d7c 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1362,8 +1362,16 @@ struct sighand_struct *lock_task_sighand(struct task_struct *tsk, rcu_read_lock(); for (;;) { sighand = rcu_dereference(tsk->sighand); - if (unlikely(sighand == NULL)) + if (unlikely(sighand == NULL)) { + /* + * Pairs with the smp_store_release() in + * __exit_signal(). It ensures that all state + * modifications to the task preceeding the store are + * visible to the callers of lock_task_sighand(). + */ + smp_acquire__after_ctrl_dep(); break; + } /* * This sighand can be already freed and even reused, but diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 73bd6a6a78935..2cd0172d05160 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -210,13 +210,39 @@ static int fill_stats_for_pid(pid_t pid, struct taskstats *stats) return 0; } +static void tgid_stats_add_task(struct taskstats *stats, + struct task_struct *tsk, u64 now_ns) +{ + u64 delta, utime, stime; + + /* + * Each accounting subsystem calls its functions here to + * accumulate its per-task stats for tsk, into the per-tgid structure + * + * per-task-foo(stats, tsk); + */ + delayacct_add_tsk(stats, tsk); + + /* calculate task elapsed time in nsec */ + delta = now_ns - tsk->start_time; + /* Convert to micro seconds */ + do_div(delta, NSEC_PER_USEC); + stats->ac_etime += delta; + + task_cputime(tsk, &utime, &stime); + stats->ac_utime += div_u64(utime, NSEC_PER_USEC); + stats->ac_stime += div_u64(stime, NSEC_PER_USEC); + + stats->nvcsw += tsk->nvcsw; + stats->nivcsw += tsk->nivcsw; +} + static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) { struct task_struct *tsk, *first; unsigned long flags; int rc = -ESRCH; - u64 delta, utime, stime; - u64 start_time; + u64 now_ns; /* * Add additional stats from live tasks except zombie thread group @@ -233,30 +259,12 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) else memset(stats, 0, sizeof(*stats)); - start_time = ktime_get_ns(); + now_ns = ktime_get_ns(); for_each_thread(first, tsk) { if (tsk->exit_state) continue; - /* - * Accounting subsystem can call its functions here to - * fill in relevant parts of struct taskstsats as follows - * - * per-task-foo(stats, tsk); - */ - delayacct_add_tsk(stats, tsk); - - /* calculate task elapsed time in nsec */ - delta = start_time - tsk->start_time; - /* Convert to micro seconds */ - do_div(delta, NSEC_PER_USEC); - stats->ac_etime += delta; - task_cputime(tsk, &utime, &stime); - stats->ac_utime += div_u64(utime, NSEC_PER_USEC); - stats->ac_stime += div_u64(stime, NSEC_PER_USEC); - - stats->nvcsw += tsk->nvcsw; - stats->nivcsw += tsk->nivcsw; + tgid_stats_add_task(stats, tsk, now_ns); } unlock_task_sighand(first, &flags); @@ -275,18 +283,14 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) static void fill_tgid_exit(struct task_struct *tsk) { unsigned long flags; + u64 now_ns; spin_lock_irqsave(&tsk->sighand->siglock, flags); if (!tsk->signal->stats) goto ret; - /* - * Each accounting subsystem calls its functions here to - * accumalate its per-task stats for tsk, into the per-tgid structure - * - * per-task-foo(tsk->signal->stats, tsk); - */ - delayacct_add_tsk(tsk->signal->stats, tsk); + now_ns = ktime_get_ns(); + tgid_stats_add_task(tsk->signal->stats, tsk, now_ns); ret: spin_unlock_irqrestore(&tsk->sighand->siglock, flags); return; diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 6e173d70d8259..c9aac73ba97a3 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -512,8 +512,6 @@ static enum alarmtimer_type clock2alarm(clockid_t clockid) * @now: time at the timer expiration * * Posix timer callback for expired alarm timers. - * - * Return: whether the timer is to be restarted */ static void alarm_handle_timer(struct alarm *alarm, ktime_t now) { @@ -527,12 +525,13 @@ static void alarm_handle_timer(struct alarm *alarm, ktime_t now) * alarm_timer_rearm - Posix timer callback for rearming timer * @timr: Pointer to the posixtimer data struct */ -static void alarm_timer_rearm(struct k_itimer *timr) +static bool alarm_timer_rearm(struct k_itimer *timr) { struct alarm *alarm = &timr->it.alarm.alarmtimer; timr->it_overrun += alarm_forward_now(alarm, timr->it_interval); alarm_start(alarm, alarm->node.expires); + return true; } /** @@ -588,7 +587,7 @@ static void alarm_timer_wait_running(struct k_itimer *timr) * @absolute: Expiry value is absolute time * @sigev_none: Posix timer does not deliver signals */ -static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires, +static bool alarm_timer_arm(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none) { struct alarm *alarm = &timr->it.alarm.alarmtimer; @@ -600,6 +599,7 @@ static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires, alarm->node.expires = expires; else alarm_start(&timr->it.alarm.alarmtimer, expires); + return true; } /** diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 1c954f330dfe7..d51428867a339 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -60,15 +60,14 @@ EXPORT_SYMBOL(get_jiffies_64); EXPORT_SYMBOL(jiffies); -static int __init init_jiffies_clocksource(void) -{ - return __clocksource_register(&clocksource_jiffies); -} - -core_initcall(init_jiffies_clocksource); +static bool cs_jiffies_registered __initdata; struct clocksource * __init __weak clocksource_default_clock(void) { + if (!cs_jiffies_registered) { + __clocksource_register(&clocksource_jiffies); + cs_jiffies_registered = true; + } return &clocksource_jiffies; } diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 0de2bb7cbec01..a7d3e8229c4ba 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -19,7 +19,7 @@ #include "posix-timers.h" -static void posix_cpu_timer_rearm(struct k_itimer *timer); +static bool posix_cpu_timer_rearm(struct k_itimer *timer); void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) { @@ -41,7 +41,7 @@ void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) */ int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) { - u64 nsecs = rlim_new * NSEC_PER_SEC; + u64 nsecs = (u64)rlim_new * NSEC_PER_SEC; unsigned long irq_fl; if (!lock_task_sighand(task, &irq_fl)) @@ -461,6 +461,109 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p) trigger_base_recalc_expires(timer, p); } +/* + * Lookup the task via timer->it.cpu.pid and attempt to lock the task's sighand. + * + * This can race with the reaping of the task: + * + * CPU0 CPU1 + * + * // Finds task + * p = pid_task(pid, pid_type); __exit_signal(p) + * lock(p, sighand); + * posix_cpu_timers*_exit(); + * sighand = lock_task_sighand(p); unhash_task(p); + * p->sighand = NULL; + * unlock(sighand); + * + * In this case sighand is NULL, which means the task and the associated timer + * queue cannot be longer accessed safely. + * + * __exit_signal() invokes posix_cpu_timers_exit() and if the thread group is + * dead it also invokes posix_cpu_timers_group_exit(). These functions delete + * all pending timers from the related timer queues. The POSIX timers (k_itimer) + * themself are still accessible, but not longer connected to the task. + * + * exec() works slightly differently. The task which exec()'s terminates all + * other threads in the thread group and runs __exit_signal() on them. As the + * thread group is not dead they only clean up the per task timers via + * posix_cpu_timers_exit(). + * + * As the TGID on exec() stays the same per process timers stay queued, if they + * are armed. This works without a problem when exec() is done by the thread + * group leader. If a non-leader thread exec()'s this can end up in the + * following scenario: + * + * CPU0 CPU1 + * // Returns old leader + * p = pid_task(pid, pid_type); de_thread() + * switch_leader() + * release_task(old leader) + * __exit_signal() + * old_leader->sighand = NULL; + * // Returns NULL + * sighand = lock_task_sighand(p) + * + * That's problematic for several functions: + * + * - posix_cpu_timer_del(): If the timer is still enqueued on the task the + * underlying k_itimer will be freed which results in a UAF in + * run_posix_cpu_timers() or on timerqueue related add/delete operations. + * If the timer is not enqueued, the failure is harmless + * + * - posix_cpu_timer_set(): Independent of the enqueued state that results in a + * transient failure which is user space visible (-ESRCH) for regular posix + * timers. But for the use case in do_cpu_nanosleep() it's the same UAF + * problem just that the timer is allocated on the stack. + * + * - posix_cpu_timer_rearm(): Timer is not enqueued at that point, but this + * silently ignores the rearm request, which is a functional problem as the + * timer wont expire anymore. + */ +static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned long *flags) +{ + enum pid_type type = clock_pid_type(timer->it_clock); + struct cpu_timer *ctmr = &timer->it.cpu; + + guard(rcu)(); + + for (;;) { + struct task_struct *t = pid_task(timer->it.cpu.pid, type); + + /* Fail if the task cannot be found. */ + if (!t) + break; + + /* Try to lock the task's sighand */ + if (lock_task_sighand(t, flags)) + return t; + + /* + * The next PID lookup might either fail or return the new + * leader. This is correct for both exit() and exec(). + */ + } + + /* + * If the timer is still enqueued, warn. There is nothing safe to do + * here as there might be two timers in there which are removed in + * parallel and that will cause more damage than good. This should never + * happen! + * + * Ensure that the stores to the timer and timerqueue are visible: + * + * __exit_signal() + * posix_cpu_timers*_exit() + * write_seqlock(seqlock) + * smp_wmb(); <------- + * __unhash_process() | !pid_task() + * ----> smp_rmb(); + * WARN_ON_ONCE(...) + */ + smp_rmb(); + WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); + return NULL; +} /* * Clean up a CPU-clock timer that is about to be destroyed. @@ -470,29 +573,13 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p) */ static int posix_cpu_timer_del(struct k_itimer *timer) { - struct cpu_timer *ctmr = &timer->it.cpu; - struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; int ret = 0; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) - goto out; + p = timer_lock_sighand(timer, &flags); - /* - * Protect against sighand release/switch in exit/exec and process/ - * thread timer list entry concurrent read/writes. - */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) { - /* - * This raced with the reaping of the task. The exit cleanup - * should have removed this timer from the timer queue. - */ - WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); - } else { + if (likely(p)) { if (timer->it.cpu.firing) { /* * Prevent signal delivery. The timer cannot be dequeued @@ -508,11 +595,8 @@ static int posix_cpu_timer_del(struct k_itimer *timer) unlock_task_sighand(p, &flags); } -out: - rcu_read_unlock(); - if (!ret) { - put_pid(ctmr->pid); + put_pid(timer->it.cpu.pid); timer->it_status = POSIX_TIMER_DISARMED; } return ret; @@ -626,21 +710,17 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); struct cpu_timer *ctmr = &timer->it.cpu; u64 old_expires, new_expires, now; - struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; int ret = 0; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) { - /* - * If p has just been reaped, we can no - * longer get any information about it at all. - */ - rcu_read_unlock(); + p = timer_lock_sighand(timer, &flags); + /* + * If p has just been reaped, we can no longer get any information about + * it at all. + */ + if (!p) return -ESRCH; - } /* * Use the to_ktime conversion because that clamps the maximum @@ -648,20 +728,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, */ new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value)); - /* - * Protect against sighand release/switch in exit/exec and p->cpu_timers - * and p->signal->cpu_timers read/write in arm_timer() - */ - sighand = lock_task_sighand(p, &flags); - /* - * If p has just been reaped, we can no - * longer get any information about it at all. - */ - if (unlikely(sighand == NULL)) { - rcu_read_unlock(); - return -ESRCH; - } - /* Retrieve the current expiry time before disarming the timer */ old_expires = cpu_timer_getexpires(ctmr); @@ -698,7 +764,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, /* Retry if the timer expiry is running concurrently */ if (unlikely(ret)) { unlock_task_sighand(p, &flags); - goto out; + return ret; } /* Convert relative expiry time to absolute */ @@ -733,8 +799,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, */ if (!sigev_none && new_expires && now >= new_expires) cpu_timer_fire(timer); -out: - rcu_read_unlock(); return ret; } @@ -1011,24 +1075,20 @@ static void check_process_timers(struct task_struct *tsk, /* * This is called from the signal code (via posixtimer_rearm) * when the last timer signal was delivered and we have to reload the timer. + * + * Return true unconditionally so the core code assumes the timer to be + * armed. Otherwise it would requeue the signal. */ -static void posix_cpu_timer_rearm(struct k_itimer *timer) +static bool posix_cpu_timer_rearm(struct k_itimer *timer) { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); struct task_struct *p; - struct sighand_struct *sighand; unsigned long flags; u64 now; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) - goto out; - - /* Protect timer list r/w in arm_timer() */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) - goto out; + p = timer_lock_sighand(timer, &flags); + if (unlikely(!p)) + return true; /* * Fetch the current sample and update the timer's expiry time. @@ -1045,8 +1105,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer) */ arm_timer(timer, p); unlock_task_sighand(p, &flags); -out: - rcu_read_unlock(); + return true; } /** @@ -1504,6 +1563,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, spin_lock_irq(&timer.it_lock); error = posix_cpu_timer_set(&timer, flags, &it, NULL); if (error) { + posix_cpu_timer_del(&timer); spin_unlock_irq(&timer.it_lock); return error; } diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 9331e16141246..da04ed42bf824 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -288,12 +288,13 @@ static inline int timer_overrun_to_int(struct k_itimer *timr) return (int)timr->it_overrun_last; } -static void common_hrtimer_rearm(struct k_itimer *timr) +static bool common_hrtimer_rearm(struct k_itimer *timr) { struct hrtimer *timer = &timr->it.real.timer; timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval); hrtimer_restart(timer); + return true; } static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr) @@ -795,7 +796,7 @@ SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id) return timer_overrun_to_int(scoped_timer); } -static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, +static bool common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none) { struct hrtimer *timer = &timr->it.real.timer; @@ -822,6 +823,7 @@ static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, if (!sigev_none) hrtimer_start_expires(timer, HRTIMER_MODE_ABS); + return true; } static int common_hrtimer_try_to_cancel(struct k_itimer *timr) diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h index 7f259e845d240..4ea9611dd7165 100644 --- a/kernel/time/posix-timers.h +++ b/kernel/time/posix-timers.h @@ -27,11 +27,11 @@ struct k_clock { int (*timer_del)(struct k_itimer *timr); void (*timer_get)(struct k_itimer *timr, struct itimerspec64 *cur_setting); - void (*timer_rearm)(struct k_itimer *timr); + bool (*timer_rearm)(struct k_itimer *timr); s64 (*timer_forward)(struct k_itimer *timr, ktime_t now); ktime_t (*timer_remaining)(struct k_itimer *timr, ktime_t now); int (*timer_try_to_cancel)(struct k_itimer *timr); - void (*timer_arm)(struct k_itimer *timr, ktime_t expires, + bool (*timer_arm)(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none); void (*timer_wait_running)(struct k_itimer *timr); }; diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index cbbb87a0c6e7c..171393367b5c1 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -797,15 +797,16 @@ static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime, *last_update_time = ktime_to_us(now); do { + ktime_t delta = 0; + seq = read_seqcount_begin(&ts->idle_sleeptime_seq); if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) { - ktime_t delta = ktime_sub(now, ts->idle_entrytime); - - idle = ktime_add(*sleeptime, delta); - } else { - idle = *sleeptime; + if (now > ts->idle_entrytime) + delta = ktime_sub(now, ts->idle_entrytime); } + + idle = ktime_add(*sleeptime, delta); } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq)); return ktime_to_us(idle); diff --git a/kernel/time/time.c b/kernel/time/time.c index 771cef87ad3b0..0dd63a91e7c58 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -251,7 +251,7 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv, get_user(new_ts.tv_nsec, &tv->tv_usec)) return -EFAULT; - if (new_ts.tv_nsec > USEC_PER_SEC || new_ts.tv_nsec < 0) + if (new_ts.tv_nsec >= USEC_PER_SEC || new_ts.tv_nsec < 0) return -EINVAL; new_ts.tv_nsec *= NSEC_PER_USEC; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index c493a40103055..eea51450e4181 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1980,13 +1980,14 @@ void __init timekeeping_init(void) */ wall_to_mono = timespec64_sub(boot_offset, wall_time); + clock = clocksource_default_clock(); + if (clock->enable) + clock->enable(clock); + guard(raw_spinlock_irqsave)(&tk_core.lock); ntp_init(); - clock = clocksource_default_clock(); - if (clock->enable) - clock->enable(clock); tk_setup_internals(tks, clock); tk_set_xtime(tks, &wall_time); diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h index 70879cde6fdd0..4c0073f3d3212 100644 --- a/kernel/time/timer_migration.h +++ b/kernel/time/timer_migration.h @@ -75,15 +75,17 @@ struct tmigr_group { /** * struct tmigr_cpu - timer migration per CPU group * @lock: Lock protecting the tmigr_cpu group information - * @online: Indicates whether the CPU is online; In deactivate path - * it is required to know whether the migrator in the top - * level group is to be set offline, while a timer is - * pending. Then another online CPU needs to be notified to - * take over the migrator role. Furthermore the information - * is required in CPU hotplug path as the CPU is able to go - * idle before the timer migration hierarchy hotplug AP is - * reached. During this phase, the CPU has to handle the + * @available: Indicates whether the CPU is available for handling + * global timers. In the deactivate path it is required to + * know whether the migrator in the top level group is to + * be set offline, while a timer is pending. Then another + * available CPU needs to be notified to take over the + * migrator role. Furthermore the information is required + * in the CPU hotplug path as the CPU is able to go idle + * before the timer migration hierarchy hotplug callback is + * reached. During this phase, the CPU has to handle the * global timers on its own and must not act as a migrator. + * @idle: Indicates whether the CPU is idle in the timer migration * hierarchy * @remote: Is set when timers of the CPU are expired remotely diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index a02bd258677ee..91309df691840 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2331,9 +2331,12 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 int err = -ENOMEM; unsigned int i; + if (!access_ok(usyms, cnt * sizeof(*usyms))) + return -EFAULT; + syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); if (!syms) - goto error; + return -ENOMEM; buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); if (!buf) @@ -2358,10 +2361,8 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 return 0; error: - if (err) { - kvfree(syms); - kvfree(buf); - } + kvfree(syms); + kvfree(buf); return err; } @@ -3180,6 +3181,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr unsigned long __user *uoffsets; u64 __user *ucookies; void __user *upath; + unsigned long size; u32 flags, cnt, i; struct path path; char *name; @@ -3217,6 +3219,16 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets); ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies); + /* + * All uoffsets/uref_ctr_offsets/ucookies arrays have the same value + * size, we need to check their address range is safe for __get_user + * calls. + */ + size = sizeof(*uoffsets) * cnt; + if (!access_ok(uoffsets, size) || !access_ok(uref_ctr_offsets, size) || + !access_ok(ucookies, size)) + return -EFAULT; + name = strndup_user(upath, PATH_MAX); if (IS_ERR(name)) { err = PTR_ERR(name); diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index f378613ad1209..f215990b90613 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -613,6 +613,16 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops continue; data_size = fp->entry_data_size; + /* + * The list may have grown since it was sized, so this node + * may not fit. Skip it as missed rather than overrun the + * reservation. + */ + if (fp->exit_handler && + used + FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(data_size) > reserved_words) { + fp->nmissed++; + continue; + } if (data_size && fp->exit_handler) data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG; else diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 7b07d2004cc6f..da9339466bddd 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -266,7 +266,8 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event) if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX) return length; length -= RB_EVNT_HDR_SIZE; - if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0])) + if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]) || + RB_FORCE_8BYTE_ALIGNMENT) length -= sizeof(event->array[0]); return length; } @@ -2219,10 +2220,7 @@ static struct ring_buffer_desc *ring_buffer_desc(struct trace_buffer_desc *trace size_t len; int i; - if (!trace_desc) - return NULL; - - if (cpu >= trace_desc->nr_cpus) + if (!trace_desc || !trace_desc->nr_cpus) return NULL; end = (struct ring_buffer_desc *)((void *)trace_desc + trace_desc->struct_len); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 6eb4d3097a4d5..4c3729c8d5e20 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -87,7 +87,7 @@ void __init disable_tracing_selftest(const char *reason) /* Pipe tracepoints to printk */ static struct trace_iterator *tracepoint_print_iter; -int tracepoint_printk; +static int tracepoint_printk; static bool tracepoint_printk_stop_on_boot __initdata; static bool traceoff_after_boot __initdata; static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key); diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index b66d6196338d0..50518b0714144 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -315,7 +315,7 @@ get_event_field(struct fetch_insn *code, void *rec) val = (unsigned long)addr; break; case FILTER_PTR_STRING: - val = (unsigned long)(*(char *)addr); + val = *(unsigned long *)addr; break; default: WARN_ON_ONCE(1); diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 609325f579427..6385cd662d8d1 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -1056,11 +1056,9 @@ static int regex_match_end(char *str, struct regex *r, int len) return 0; } -static int regex_match_glob(char *str, struct regex *r, int len __maybe_unused) +static int regex_match_glob(char *str, struct regex *r, int len) { - if (glob_match(r->pattern, str)) - return 1; - return 0; + return glob_match_len(r->pattern, str, len) ? 1 : 0; } /** diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index eb2c2bc8bc3d5..9701650c89b2f 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -2967,13 +2968,22 @@ find_synthetic_field_var(struct hist_trigger_data *target_hist_data, { struct hist_field *event_var; char *synthetic_name; + struct seq_buf s; synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); if (!synthetic_name) return ERR_PTR(-ENOMEM); - strcpy(synthetic_name, "synthetic_"); - strcat(synthetic_name, field_name); + seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL); + seq_buf_printf(&s, "synthetic_%s", field_name); + + /* Terminate synthetic_name with a NUL. */ + seq_buf_str(&s); + + if (seq_buf_has_overflowed(&s)) { + kfree(synthetic_name); + return ERR_PTR(-E2BIG); + } event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); @@ -3019,6 +3029,7 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, struct hist_field *key_field; struct hist_field *event_var; char *saved_filter; + struct seq_buf s; char *cmd; int ret; @@ -3063,28 +3074,34 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, return ERR_PTR(-ENOMEM); } + seq_buf_init(&s, cmd, MAX_FILTER_STR_VAL); + /* Use the same keys as the compatible histogram */ - strcat(cmd, "keys="); + seq_buf_puts(&s, "keys="); for_each_hist_key_field(i, hist_data) { key_field = hist_data->fields[i]; if (!first) - strcat(cmd, ","); - strcat(cmd, key_field->field->name); + seq_buf_putc(&s, ','); + seq_buf_puts(&s, key_field->field->name); first = false; } /* Create the synthetic field variable specification */ - strcat(cmd, ":synthetic_"); - strcat(cmd, field_name); - strcat(cmd, "="); - strcat(cmd, field_name); + seq_buf_printf(&s, ":synthetic_%s=%s", field_name, field_name); /* Use the same filter as the compatible histogram */ saved_filter = find_trigger_filter(hist_data, file); - if (saved_filter) { - strcat(cmd, " if "); - strcat(cmd, saved_filter); + if (saved_filter) + seq_buf_printf(&s, " if %s", saved_filter); + + /* Terminate cmd with a NUL. */ + seq_buf_str(&s); + + if (seq_buf_has_overflowed(&s)) { + kfree(cmd); + kfree(var_hist); + return ERR_PTR(-E2BIG); } var_hist->cmd = kstrdup(cmd, GFP_KERNEL); diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index c4ba484f7b38b..8c82ecb735f41 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -109,6 +109,9 @@ struct user_event_enabler { /* Track enable bit, flags, etc. Aligned for bitops. */ unsigned long values; + + /* Defer the event put and enabler free past an RCU grace period. */ + struct rcu_work put_rwork; }; /* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */ @@ -396,17 +399,39 @@ static struct user_event_group *user_event_group_create(void) return NULL; }; -static void user_event_enabler_destroy(struct user_event_enabler *enabler, - bool locked) +static void delayed_user_event_enabler_put(struct work_struct *work) { - list_del_rcu(&enabler->mm_enablers_link); + struct user_event_enabler *enabler = container_of(to_rcu_work(work), + struct user_event_enabler, put_rwork); /* No longer tracking the event via the enabler */ - user_event_put(enabler->event, locked); + user_event_put(enabler->event, false); + /* Run from queue_rcu_work(), the RCU grace period has elapsed */ kfree(enabler); } +static void user_event_enabler_destroy(struct user_event_enabler *enabler) +{ + list_del_rcu(&enabler->mm_enablers_link); + + /* + * The enabler is removed from an RCU-traversed list + * (user_event_mm_dup() walks mm->enablers under rcu_read_lock() only), + * and readers there dereference enabler->event and take a new ref on + * it. Both the put of that event reference and the free of the enabler + * therefore have to wait for a grace period so no reader can be looking + * at the enabler or racing the last put of its event. + * + * The put itself must not run in RCU context: when it drops the last + * reference user_event_put() takes event_mutex, which cannot be taken + * from a softirq/RCU callback. Defer both to a work item scheduled + * after a grace period via queue_rcu_work(). + */ + INIT_RCU_WORK(&enabler->put_rwork, delayed_user_event_enabler_put); + queue_rcu_work(system_percpu_wq, &enabler->put_rwork); +} + static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr, int attempt) { @@ -464,7 +489,7 @@ static void user_event_enabler_fault_fixup(struct work_struct *work) /* User asked for enabler to be removed during fault */ if (test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))) { - user_event_enabler_destroy(enabler, true); + user_event_enabler_destroy(enabler); goto out; } @@ -764,7 +789,7 @@ static void user_event_mm_destroy(struct user_event_mm *mm) struct user_event_enabler *enabler, *next; list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link) - user_event_enabler_destroy(enabler, false); + user_event_enabler_destroy(enabler); mmdrop(mm->mm); kfree(mm); @@ -2645,7 +2670,7 @@ static long user_events_ioctl_unreg(unsigned long uarg) flags |= enabler->values & ENABLE_VAL_COMPAT_MASK; if (!test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler))) - user_event_enabler_destroy(enabler, true); + user_event_enabler_destroy(enabler); /* Removed at least one */ ret = 0; diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index f283391a4dc80..cd37f20137581 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -458,12 +458,12 @@ func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) ftrace_func_t func; u32 new_flags; - /* Do nothing if already set. */ - if (!!set == !!(tr->current_trace_flags->val & bit)) + /* We can change this flag only when current tracer is function. */ + if (tr->current_trace != &function_trace) return 0; - /* We can change this flag only when not running. */ - if (tr->current_trace != &function_trace) + /* Do nothing if already set. */ + if (!!set == !!(tr->current_trace_flags->val & bit)) return 0; new_flags = (tr->current_trace_flags->val & ~bit) | (set ? bit : 0); diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 75678053b21c5..7a244ef814382 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -162,7 +162,9 @@ static void osnoise_unregister_instance(struct trace_array *tr) if (!found) return; - kvfree_rcu_mightsleep(inst); + /* Do a full sync to ensure that tr remains valid, not just inst */ + synchronize_rcu(); + kvfree(inst); } /* diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 44c22d4e78818..733ce83e1e11c 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -332,6 +332,19 @@ static int parse_trace_event_arg(char *arg, struct fetch_insn *code, return -ENOENT; } +static int parse_trace_event(char *arg, struct fetch_insn *code, + struct traceprobe_parse_context *ctx) +{ + int ret; + + if (code->data) + return -EFAULT; + ret = parse_trace_event_arg(arg, code, ctx); + if (!ret) + return 0; + return -EINVAL; +} + #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS static u32 btf_type_int(const struct btf_type *t) @@ -376,11 +389,16 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) && BTF_INT_BITS(intdata) == 8; } +static struct btf *ctx_btf(struct traceprobe_parse_context *ctx) +{ + return ctx->struct_btf ? : ctx->btf; +} + static int check_prepare_btf_string_fetch(char *typename, struct fetch_insn **pcode, struct traceprobe_parse_context *ctx) { - struct btf *btf = ctx->btf; + struct btf *btf = ctx_btf(ctx); if (!btf || !ctx->last_type) return 0; @@ -506,6 +524,15 @@ static int query_btf_context(struct traceprobe_parse_context *ctx) return 0; } +static void clear_struct_btf(struct traceprobe_parse_context *ctx) +{ + if (ctx->struct_btf) { + btf_put(ctx->struct_btf); + ctx->struct_btf = NULL; + ctx->last_struct = NULL; + } +} + static void clear_btf_context(struct traceprobe_parse_context *ctx) { if (ctx->btf) { @@ -554,22 +581,29 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, struct fetch_insn *code = *pcode; const struct btf_member *field; u32 bitoffs, anon_offs; + bool is_struct = ctx->struct_btf != NULL; + struct btf *btf = ctx_btf(ctx); char *next; int is_ptr; s32 tid; do { - /* Outer loop for solving arrow operator ('->') */ - if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { - trace_probe_log_err(ctx->offset, NO_PTR_STRCT); - return -EINVAL; - } - /* Convert a struct pointer type to a struct type */ - type = btf_type_skip_modifiers(ctx->btf, type->type, &tid); - if (!type) { - trace_probe_log_err(ctx->offset, BAD_BTF_TID); - return -EINVAL; + if (!is_struct) { + /* Outer loop for solving arrow operator ('->') */ + if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { + trace_probe_log_err(ctx->offset, NO_PTR_STRCT); + return -EINVAL; + } + + /* Convert a struct pointer type to a struct type */ + type = btf_type_skip_modifiers(btf, type->type, &tid); + if (!type) { + trace_probe_log_err(ctx->offset, BAD_BTF_TID); + return -EINVAL; + } } + /* Only the first type can skip being a pointer */ + is_struct = false; bitoffs = 0; do { @@ -580,7 +614,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, return is_ptr; anon_offs = 0; - field = btf_find_struct_member(ctx->btf, type, fieldname, + field = btf_find_struct_member(btf, type, fieldname, &anon_offs); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); @@ -602,7 +636,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, ctx->last_bitsize = 0; } - type = btf_type_skip_modifiers(ctx->btf, field->type, &tid); + type = btf_type_skip_modifiers(btf, field->type, &tid); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -640,7 +674,7 @@ static int parse_btf_arg(char *varname, int i, is_ptr, ret; u32 tid; - if (WARN_ON_ONCE(!ctx->funcname)) + if (!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT)) return -EINVAL; is_ptr = split_next_field(varname, &field, ctx); @@ -653,6 +687,19 @@ static int parse_btf_arg(char *varname, return -EOPNOTSUPP; } + if (ctx->flags & TPARG_FL_TEVENT) { + ret = parse_trace_event(varname, code, ctx); + if (ret < 0) { + trace_probe_log_err(ctx->offset, BAD_ATTACH_ARG); + return ret; + } + /* TEVENT is only here via a typecast */ + if (WARN_ON_ONCE(ctx->struct_btf == NULL)) + return -EINVAL; + type = ctx->last_struct; + goto found_type; + } + if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) { code->op = FETCH_OP_RETVAL; /* Check whether the function return type is not void */ @@ -709,6 +756,7 @@ static int parse_btf_arg(char *varname, found: type = btf_type_skip_modifiers(ctx->btf, tid, &tid); +found_type: if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -727,7 +775,7 @@ static int parse_btf_arg(char *varname, static const struct fetch_type *find_fetch_type_from_btf_type( struct traceprobe_parse_context *ctx) { - struct btf *btf = ctx->btf; + struct btf *btf = ctx_btf(ctx); const char *typestr = NULL; if (btf && ctx->last_type) @@ -758,7 +806,67 @@ static int parse_btf_bitfield(struct fetch_insn **pcode, return 0; } -#else +static int query_btf_struct(const char *sname, struct traceprobe_parse_context *ctx) +{ + struct btf *btf = NULL; + int id; + + /* A struct_btf should only be used by a single argument */ + if (WARN_ON_ONCE(ctx->struct_btf)) { + btf_put(ctx->struct_btf); + ctx->struct_btf = NULL; + } + + id = bpf_find_btf_id(sname, BTF_KIND_STRUCT, &btf); + if (id < 0) + return id; + ctx->struct_btf = btf; + ctx->last_struct = btf_type_by_id(ctx->struct_btf, id); + return 0; +} + +static int handle_typecast(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, + struct traceprobe_parse_context *ctx) +{ + char *tmp; + int ret; + + /* Currently this only works for eprobes */ + if (!(ctx->flags & TPARG_FL_TEVENT)) { + trace_probe_log_err(ctx->offset, TYPECAST_NOT_EVENT); + return -EINVAL; + } + + tmp = strchr(arg, ')'); + if (!tmp) { + trace_probe_log_err(ctx->offset + strlen(arg), + DEREF_OPEN_BRACE); + return -EINVAL; + } + *tmp = '\0'; + ret = query_btf_struct(arg + 1, ctx); + *tmp = ')'; + + if (ret < 0) { + trace_probe_log_err(ctx->offset + 1, NO_PTR_STRCT); + return -EINVAL; + } + + tmp++; + + ctx->offset += tmp - arg; + ret = parse_btf_arg(tmp, pcode, end, ctx); + return ret; +} + +#else /* !CONFIG_PROBE_EVENTS_BTF_ARGS */ + +static void clear_struct_btf(struct traceprobe_parse_context *ctx) +{ + ctx->struct_btf = NULL; +} + static void clear_btf_context(struct traceprobe_parse_context *ctx) { ctx->btf = NULL; @@ -794,7 +902,15 @@ static int check_prepare_btf_string_fetch(char *typename, return 0; } -#endif +static int handle_typecast(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, + struct traceprobe_parse_context *ctx) +{ + trace_probe_log_err(ctx->offset, NOSUP_BTFARG); + return -EOPNOTSUPP; +} + +#endif /* CONFIG_PROBE_EVENTS_BTF_ARGS */ #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API @@ -953,16 +1069,15 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, int len; if (ctx->flags & TPARG_FL_TEVENT) { - if (code->data) - return -EFAULT; - ret = parse_trace_event_arg(arg, code, ctx); - if (!ret) - return 0; - if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { - code->op = FETCH_OP_COMM; - return 0; + if (parse_trace_event(arg, code, ctx) < 0) { + /* 'comm' should be checked after field parsing. */ + if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { + code->op = FETCH_OP_COMM; + return 0; + } + goto inval; } - goto inval; + return 0; } if (str_has_prefix(arg, "retval")) { @@ -1133,6 +1248,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_FOFFS; code->immediate = (unsigned long)offset; // imm64? + offset = 0; } else { /* uprobes don't support symbols */ if (!(ctx->flags & TPARG_FL_KERNEL)) { @@ -1229,6 +1345,9 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_IMM; } break; + case '(': + ret = handle_typecast(arg, pcode, end, ctx); + break; default: if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */ if (!tparg_is_function_entry(ctx->flags) && @@ -1561,6 +1680,9 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size, } kfree(tmp); + /* struct_btf should not be passed to other arguments */ + clear_struct_btf(ctx); + return ret; } diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 262d8707a3df7..de489f98b8a72 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -422,7 +422,9 @@ struct traceprobe_parse_context { const struct btf_param *params; /* Parameter of the function */ s32 nr_params; /* The number of the parameters */ struct btf *btf; /* The BTF to be used */ + struct btf *struct_btf; /* The BTF to be used for structs */ const struct btf_type *last_type; /* Saved type */ + const struct btf_type *last_struct; /* Saved structure */ u32 last_bitoffs; /* Saved bitoffs */ u32 last_bitsize; /* Saved bitsize */ struct trace_probe *tp; @@ -509,7 +511,7 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NO_RETVAL, "This function returns 'void' type"), \ C(BAD_STACK_NUM, "Invalid stack number"), \ C(BAD_ARG_NUM, "Invalid argument number"), \ - C(BAD_VAR, "Invalid $-valiable specified"), \ + C(BAD_VAR, "Invalid $-variable specified"), \ C(BAD_REG_NAME, "Invalid register name"), \ C(BAD_MEM_ADDR, "Invalid memory address"), \ C(BAD_IMM, "Invalid immediate value"), \ @@ -563,7 +565,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NEED_STRING_TYPE, "$comm and immediate-string only accepts string type"),\ C(TOO_MANY_ARGS, "Too many arguments are specified"), \ C(TOO_MANY_EARGS, "Too many entry arguments specified"), \ - C(EVENT_TOO_BIG, "Event too big (too many fields?)"), + C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \ + C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), #undef C #define C(a, b) TP_ERR_##a diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index d6c3f94d67cd9..6dde6bdcbde70 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -979,33 +979,30 @@ EXPORT_SYMBOL_GPL(trace_remote_free_buffer); int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, size_t buffer_size, const struct cpumask *cpumask) { + size_t min_desc_size = trace_buffer_desc_size(buffer_size, cpumask_weight(cpumask)); unsigned int nr_pages = max(DIV_ROUND_UP(buffer_size, PAGE_SIZE), 2UL) + 1; - void *desc_end = desc + desc_size; struct ring_buffer_desc *rb_desc; int cpu, ret = -ENOMEM; - if (desc_size < struct_size(desc, __data, 0)) + if (desc_size < min_desc_size) return -EINVAL; desc->nr_cpus = 0; - desc->struct_len = struct_size(desc, __data, 0); + desc->struct_len = min_desc_size; - rb_desc = (struct ring_buffer_desc *)&desc->__data[0]; + rb_desc = __first_ring_buffer_desc(desc); for_each_cpu(cpu, cpumask) { unsigned int id; - if ((void *)rb_desc + struct_size(rb_desc, page_va, nr_pages) > desc_end) { - ret = -EINVAL; - goto err; - } - rb_desc->cpu = cpu; rb_desc->nr_page_va = 0; rb_desc->meta_va = (unsigned long)__get_free_page(GFP_KERNEL); if (!rb_desc->meta_va) goto err; + desc->nr_cpus++; + for (id = 0; id < nr_pages; id++) { rb_desc->page_va[id] = (unsigned long)__get_free_page(GFP_KERNEL); if (!rb_desc->page_va[id]) @@ -1013,9 +1010,6 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, rb_desc->nr_page_va++; } - desc->nr_cpus++; - desc->struct_len += offsetof(struct ring_buffer_desc, page_va); - desc->struct_len += struct_size(rb_desc, page_va, rb_desc->nr_page_va); rb_desc = __next_ring_buffer_desc(rb_desc); } diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 33b721a9af022..bd79b3b5ca878 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -6310,7 +6310,7 @@ EXPORT_SYMBOL_GPL(set_worker_desc); */ void print_worker_info(const char *log_lvl, struct task_struct *task) { - work_func_t *fn = NULL; + work_func_t fn = NULL; char name[WQ_NAME_LEN] = { }; char desc[WORKER_DESC_LEN] = { }; struct pool_workqueue *pwq = NULL; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8ff5adcfe1e0a..040f4e077435b 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2649,12 +2649,17 @@ config TEST_VMALLOC config TEST_WORKQUEUE tristate "Test module for stress/performance analysis of workqueue" + depends on m default n help This builds the "test_workqueue" module for benchmarking workqueue throughput under contention. Useful for evaluating affinity scope changes (e.g., cache_shard vs cache). + The test drives sysfs to switch affinity scopes, so it must be + loaded after userspace has mounted sysfs; building it in (=y) + would run module_init before /sys is available. + If unsure, say N. config TEST_BPF diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index ed1bdcf1f8abf..a9ab88f416b9f 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -45,6 +45,7 @@ int alloc_tag_ref_offs; struct allocinfo_private { struct codetag_iterator iter; + struct codetag_iterator reported_iter; bool print_header; }; @@ -58,16 +59,20 @@ static void *allocinfo_start(struct seq_file *m, loff_t *pos) if (node == 0) { priv->print_header = true; priv->iter = codetag_get_ct_iter(alloc_tag_cttype); - codetag_next_ct(&priv->iter); + } else { + priv->iter = priv->reported_iter; } + codetag_next_ct(&priv->iter); return priv->iter.ct ? priv : NULL; } static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) { struct allocinfo_private *priv = (struct allocinfo_private *)arg; - struct codetag *ct = codetag_next_ct(&priv->iter); + struct codetag *ct; + priv->reported_iter = priv->iter; + ct = codetag_next_ct(&priv->iter); (*pos)++; if (!ct) return NULL; diff --git a/lib/base64.c b/lib/base64.c index 41961a444028b..20dacee25f650 100644 --- a/lib/base64.c +++ b/lib/base64.c @@ -168,15 +168,16 @@ int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base6 return -1; val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6); - *bp++ = val >> 10; if (srclen == 2) { if (val & 0x800003ff) return -1; + *bp++ = val >> 10; } else { val |= base64_rev_tables[s[2]]; if (val & 0x80000003) return -1; + *bp++ = val >> 10; *bp++ = val >> 2; } return bp - dst; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index d3904b72dae7a..1ee6cfadcd84f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -8,8 +8,7 @@ config CRYPTO_LIB_UTILS config CRYPTO_LIB_AES tristate - # Select dependencies of modes that are part of libaes. - select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_UTILS config CRYPTO_LIB_AES_ARCH bool diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 6fb00e08a4e29..877f7675ba26b 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -894,6 +894,14 @@ int debug_object_activate(void *addr, const struct debug_obj_descr *descr) } raw_spin_unlock_irqrestore(&db->lock, flags); + + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. + */ + if (!debug_objects_enabled) + return 0; + debug_print_object(&o, "activate"); switch (o.state) { @@ -1071,6 +1079,15 @@ void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) return; } + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. Don't run the fixup + * as it might turn a valid object useless. See for example + * hrtimer_fixup_assert_init(). + */ + if (!debug_objects_enabled) + return; + /* Object is neither tracked nor static. It's not initialized. */ debug_print_object(&o, "assert_init"); debug_object_fixup(descr->fixup_assert_init, addr, ODEBUG_STATE_NOTAVAILABLE); diff --git a/lib/fonts/font_acorn_8x8.c b/lib/fonts/font_acorn_8x8.c index 36c51016769d4..4ff52c79f8c4d 100644 --- a/lib/fonts/font_acorn_8x8.c +++ b/lib/fonts/font_acorn_8x8.c @@ -5,7 +5,12 @@ #define FONTDATAMAX 2048 +#ifdef BOOTLOADER +/* The acorndata_8x8 symbol is needed by the ARM bootloader too. */ +const struct font_data acorndata_8x8 = { +#else static const struct font_data acorndata_8x8 = { +#endif { 0, 0, FONTDATAMAX, 0 }, { /* 00 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* ^@ */ /* 01 */ 0x7e, 0x81, 0xa5, 0x81, 0xbd, 0x99, 0x81, 0x7e, /* ^A */ diff --git a/lib/glob.c b/lib/glob.c index 7aca76c25bcb2..c80d9dd736b4f 100644 --- a/lib/glob.c +++ b/lib/glob.c @@ -11,6 +11,9 @@ MODULE_DESCRIPTION("glob(7) matching"); MODULE_LICENSE("Dual MIT/GPL"); +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end); + /** * glob_match - Shell-style pattern matching, like !fnmatch(pat, str, 0) * @pat: Shell-style pattern to match, e.g. "*.[ch]". @@ -40,6 +43,29 @@ MODULE_LICENSE("Dual MIT/GPL"); * An opening bracket without a matching close is matched literally. */ bool __pure glob_match(char const *pat, char const *str) +{ + return glob_match_str(pat, str, NULL); +} +EXPORT_SYMBOL(glob_match); + +/** + * glob_match_len - glob match against a length-bounded string + * @pat: Shell-style pattern to match. + * @str: String to match. Need not be NUL-terminated. + * @len: Number of bytes of @str that may be read. + * + * Like glob_match(), but @str is only read up to @len bytes, so it can be + * used on buffers that are not NUL-terminated (e.g. trace event fields). + * A NUL byte within @len still terminates the string. + */ +bool __pure glob_match_len(char const *pat, char const *str, size_t len) +{ + return glob_match_str(pat, str, str + len); +} +EXPORT_SYMBOL(glob_match_len); + +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end) { /* * Backtrack to previous * on mismatch and retry starting one @@ -55,9 +81,11 @@ bool __pure glob_match(char const *pat, char const *str) * on mismatch, or true after matching the trailing nul bytes. */ for (;;) { - unsigned char c = *str++; + unsigned char c = (str_end && str >= str_end) ? '\0' : *str; unsigned char d = *pat++; + str++; + switch (d) { case '?': /* Wildcard: anything but nul */ if (c == '\0') @@ -125,4 +153,3 @@ bool __pure glob_match(char const *pat, char const *str) } } } -EXPORT_SYMBOL(glob_match); diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index d939403331b5a..bfafe1204c7b6 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1431,10 +1431,9 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { int saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP int saved_mgd_count = current->migration_disabled; -#endif int saved_rcu_count = current->rcu_read_lock_nesting; + int saved_sched_rt_mutex = current->sched_rt_mutex; #endif WARN_ON(irqs_disabled()); @@ -1471,10 +1470,10 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP + current->sched_rt_mutex = saved_sched_rt_mutex; + while (current->migration_disabled > saved_mgd_count) migrate_enable(); -#endif while (current->rcu_read_lock_nesting > saved_rcu_count) rcu_read_unlock(); diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 04b3a808fca9f..c0ba34eadb397 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -1057,8 +1057,9 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed) * .obj_hashfn = my_hash_fn, * }; */ -int rhashtable_init_noprof(struct rhashtable *ht, - const struct rhashtable_params *params) +int __rhashtable_init_noprof(struct rhashtable *ht, + const struct rhashtable_params *params, + struct lock_class_key *key) { struct bucket_table *tbl; size_t size; @@ -1068,7 +1069,7 @@ int rhashtable_init_noprof(struct rhashtable *ht, return -EINVAL; memset(ht, 0, sizeof(*ht)); - mutex_init(&ht->mutex); + mutex_init_with_key(&ht->mutex, key); spin_lock_init(&ht->lock); memcpy(&ht->p, params, sizeof(*params)); @@ -1120,7 +1121,7 @@ int rhashtable_init_noprof(struct rhashtable *ht, return 0; } -EXPORT_SYMBOL_GPL(rhashtable_init_noprof); +EXPORT_SYMBOL_GPL(__rhashtable_init_noprof); /** * rhltable_init - initialize a new hash list table @@ -1131,15 +1132,17 @@ EXPORT_SYMBOL_GPL(rhashtable_init_noprof); * * See documentation for rhashtable_init. */ -int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params) +int __rhltable_init_noprof(struct rhltable *hlt, + const struct rhashtable_params *params, + struct lock_class_key *key) { int err; - err = rhashtable_init_noprof(&hlt->ht, params); + err = __rhashtable_init_noprof(&hlt->ht, params, key); hlt->ht.rhlist = true; return err; } -EXPORT_SYMBOL_GPL(rhltable_init_noprof); +EXPORT_SYMBOL_GPL(__rhltable_init_noprof); static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, void (*free_fn)(void *ptr, void *arg), diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 2135049157372..1e2b76e79c74e 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -1111,9 +1111,6 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, unsigned long *src_pfns; unsigned long *dst_pfns; - src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); - dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); - start = cmd->addr; end = start + size; if (end < start) @@ -1123,6 +1120,9 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, if (!mmget_not_zero(mm)) return -EINVAL; + src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); + dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); + cmd->cpages = 0; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { @@ -1253,8 +1253,8 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror, mmap_read_unlock(mm); mmput(mm); free_mem: - kfree(src_pfns); - kfree(dst_pfns); + kvfree(src_pfns); + kvfree(dst_pfns); return ret; } diff --git a/lib/test_meminit.c b/lib/test_meminit.c index 6298f66c964bb..d028a6552cd61 100644 --- a/lib/test_meminit.c +++ b/lib/test_meminit.c @@ -387,7 +387,7 @@ static int __init test_kmemcache(int *total_failures) ctor = flags & 1; rcu = flags & 2; zero = flags & 4; - if (ctor & zero) + if (ctor && zero) continue; num_tests += do_kmem_cache_size(size, ctor, rcu, zero, &failures); diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c index f02f7b7aa7960..1e6fce9cb255f 100644 --- a/lib/tests/kunit_iov_iter.c +++ b/lib/tests/kunit_iov_iter.c @@ -53,7 +53,7 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, size_t npages) { struct page **pages; - unsigned long got; + unsigned long got, last; void *buffer; unsigned int i; @@ -61,7 +61,15 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); *ppages = pages; - got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + got = 0; + while (true) { + last = got; + got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + + if (last == got || got == npages) + break; + } + if (got != npages) { release_pages(pages, got); kvfree(pages); diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c index e4b0ecbee32fe..4c08a7c6fb788 100644 --- a/lib/tests/liveupdate.c +++ b/lib/tests/liveupdate.c @@ -105,6 +105,9 @@ static void liveupdate_test_init(void) pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n", flb->compatible, ERR_PTR(err)); } + + if (!err) + liveupdate_flb_put_incoming(flb); } initialized = true; } diff --git a/lib/usercopy.c b/lib/usercopy.c index b00a3a957de6b..e2f0bf104a591 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -12,15 +12,13 @@ /* out-of-line parts */ -#if !defined(INLINE_COPY_FROM_USER) +#if !defined(INLINE_COPY_USER) unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n) { return _inline_copy_from_user(to, from, n); } EXPORT_SYMBOL(_copy_from_user); -#endif -#if !defined(INLINE_COPY_TO_USER) unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { return _inline_copy_to_user(to, from, n); diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 9f359b31c8d1c..3c76cc5c7f9ce 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -2640,6 +2640,18 @@ static unsigned char spec_flag(unsigned char c) return (c < sizeof(spec_flag_array)) ? spec_flag_array[c] : 0; } +static void set_field_width(struct printf_spec *spec, int width) +{ + spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX); + WARN_ONCE(spec->field_width != width, "field width %d out of range", width); +} + +static void set_precision(struct printf_spec *spec, int prec) +{ + spec->precision = clamp(prec, 0, PRECISION_MAX); + WARN_ONCE(spec->precision < prec, "precision %d too large", prec); +} + /* * Helper function to decode printf style format. * Each call decode a token from the format and return the @@ -2710,7 +2722,7 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec) spec->field_width = -1; if (isdigit(*fmt.str)) - spec->field_width = skip_atoi(&fmt.str); + set_field_width(spec, skip_atoi(&fmt.str)); else if (unlikely(*fmt.str == '*')) { /* it's the next argument */ fmt.state = FORMAT_STATE_WIDTH; @@ -2724,9 +2736,7 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec) if (unlikely(*fmt.str == '.')) { fmt.str++; if (isdigit(*fmt.str)) { - spec->precision = skip_atoi(&fmt.str); - if (spec->precision < 0) - spec->precision = 0; + set_precision(spec, skip_atoi(&fmt.str)); } else if (*fmt.str == '*') { /* it's the next argument */ fmt.state = FORMAT_STATE_PRECISION; @@ -2799,24 +2809,6 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec) return fmt; } -static void -set_field_width(struct printf_spec *spec, int width) -{ - spec->field_width = width; - if (WARN_ONCE(spec->field_width != width, "field width %d too large", width)) { - spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX); - } -} - -static void -set_precision(struct printf_spec *spec, int prec) -{ - spec->precision = prec; - if (WARN_ONCE(spec->precision != prec, "precision %d too large", prec)) { - spec->precision = clamp(prec, 0, PRECISION_MAX); - } -} - /* * Turn a 1/2/4-byte value into a 64-bit one for printing: truncate * as necessary and deal with signedness. diff --git a/mm/compaction.c b/mm/compaction.c index 3648ce22c8072..42eec51c06043 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1875,15 +1875,14 @@ static void compaction_free(struct folio *dst, unsigned long data) int order = folio_order(dst); struct page *page = &dst->page; - if (folio_put_testzero(dst)) { - free_pages_prepare(page, order); + if (folio_put_testzero(dst) && free_pages_prepare(page, order)) { list_add(&dst->lru, &cc->freepages[order]); cc->nr_freepages += 1 << order; } cc->nr_migratepages += 1 << order; /* - * someone else has referenced the page, we cannot take it back to our - * free list. + * someone else has referenced the page or free_pages_prepare() fails, + * we cannot take it back to our free list. */ } diff --git a/mm/damon/core.c b/mm/damon/core.c index 3dbbbfdeff719..60f5f191e17a0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1257,10 +1257,36 @@ static int damon_commit_target( return 0; } +/* + * damon_revert_target_commits() - revert unsuccessful target commits. + * @dst: Commit destination context + * @failed: Commit failed destination target + * @src: Commit source context + * + * Revert target states that changed by damon_commit_target(), and cannot be + * cleaned up by the destination context's ops.cleanup_target(). + */ +static void damon_revert_target_commits(struct damon_ctx *dst, + struct damon_target *failed, struct damon_ctx *src) +{ + struct damon_target *target; + + if (!damon_target_has_pid(src)) + return; + if (dst->ops.cleanup_target) + return; + damon_for_each_target(target, dst) { + if (target == failed) + return; + put_pid(target->pid); + } +} + static int damon_commit_targets( struct damon_ctx *dst, struct damon_ctx *src) { struct damon_target *dst_target, *next, *src_target, *new_target; + struct damon_target *failed; int i = 0, j = 0, err; damon_for_each_target_safe(dst_target, next, dst) { @@ -1274,8 +1300,10 @@ static int damon_commit_targets( dst_target, damon_target_has_pid(dst), src_target, damon_target_has_pid(src), src->min_region_sz); - if (err) - return err; + if (err) { + failed = dst_target; + goto out; + } } else { struct damos *s; @@ -1289,25 +1317,34 @@ static int damon_commit_targets( } } + failed = NULL; damon_for_each_target_safe(src_target, next, src) { if (j++ < i) continue; /* target to remove has no matching dst */ - if (src_target->obsolete) - return -EINVAL; + if (src_target->obsolete) { + err = -EINVAL; + goto out; + } new_target = damon_new_target(); - if (!new_target) - return -ENOMEM; + if (!new_target) { + err = -ENOMEM; + goto out; + } err = damon_commit_target(new_target, false, src_target, damon_target_has_pid(src), src->min_region_sz); if (err) { damon_destroy_target(new_target, NULL); - return err; + goto out; } damon_add_target(dst, new_target); } return 0; + +out: + damon_revert_target_commits(dst, failed, src); + return err; } /** @@ -1346,8 +1383,10 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) */ if (!damon_attrs_equals(&dst->attrs, &src->attrs)) { err = damon_set_attrs(dst, &src->attrs); - if (err) + if (err) { + damon_revert_target_commits(dst, NULL, src); return err; + } } dst->ops = src->ops; dst->addr_unit = src->addr_unit; @@ -2106,7 +2145,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, if (damos_quota_is_set(quota) && quota->charged_sz >= quota->esz) { quota->charge_target_from = t; - quota->charge_addr_from = r->ar.end + 1; + quota->charge_addr_from = r->ar.end; } } if (s->action != DAMOS_STAT) @@ -2453,6 +2492,8 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) if (!quota->total_charged_sz && !quota->charged_from) { quota->charged_from = jiffies; damos_set_effective_quota(quota, c); + if (trace_damos_esz_enabled()) + damos_trace_esz(c, s, quota); } /* New charge window starts */ diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index c3e4c871b0bb2..4752d6118eb93 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -140,6 +140,7 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, * Transform it to fit in [0, DAMOS_MAX_SCORE] */ hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; + hotness = max(min(hotness, DAMOS_MAX_SCORE), 0); return hotness; } diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index a8014780edae9..f847d32a0ee33 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1767,22 +1767,19 @@ static int damon_sysfs_access_pattern_add_dirs( err = damon_sysfs_access_pattern_add_range_dir(access_pattern, &access_pattern->sz, "sz"); if (err) - goto put_sz_out; + return err; err = damon_sysfs_access_pattern_add_range_dir(access_pattern, &access_pattern->nr_accesses, "nr_accesses"); if (err) - goto put_nr_accesses_sz_out; + goto put_sz_out; err = damon_sysfs_access_pattern_add_range_dir(access_pattern, &access_pattern->age, "age"); if (err) - goto put_age_nr_accesses_sz_out; + goto put_nr_accesses_sz_out; return 0; -put_age_nr_accesses_sz_out: - kobject_put(&access_pattern->age->kobj); - access_pattern->age = NULL; put_nr_accesses_sz_out: kobject_put(&access_pattern->nr_accesses->kobj); access_pattern->nr_accesses = NULL; @@ -2286,12 +2283,12 @@ static int damon_sysfs_scheme_add_dirs(struct damon_sysfs_scheme *scheme) goto put_filters_watermarks_quotas_access_pattern_out; err = damon_sysfs_scheme_set_tried_regions(scheme); if (err) - goto put_tried_regions_out; + goto put_stats_out; return 0; -put_tried_regions_out: - kobject_put(&scheme->tried_regions->kobj); - scheme->tried_regions = NULL; +put_stats_out: + kobject_put(&scheme->stats->kobj); + scheme->stats = NULL; put_filters_watermarks_quotas_access_pattern_out: kobject_put(&scheme->ops_filters->kobj); scheme->ops_filters = NULL; diff --git a/mm/filemap.c b/mm/filemap.c index 4e636647100c1..b72f8c23c3226 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -4671,7 +4671,7 @@ static inline bool can_do_cachestat(struct file *f) { if (f->f_mode & FMODE_WRITE) return true; - if (inode_owner_or_capable(file_mnt_idmap(f), file_inode(f))) + if (file_owner_or_capable(f)) return true; return file_permission(f, MAY_WRITE) == 0; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b118bcd392cb3..3f0466729c75b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2638,6 +2638,8 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); newpmd = swp_entry_to_pmd(entry); + if (pmd_swp_uffd_wp(*pmd)) + newpmd = pmd_swp_mkuffd_wp(newpmd); } else { newpmd = *pmd; } @@ -3644,6 +3646,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order, (1L << PG_arch_3) | #endif (1L << PG_dirty) | + (1L << PG_dropbehind) | LRU_GEN_MASK | LRU_REFS_MASK)); if (handle_hwpoison && diff --git a/mm/hugetlb.c b/mm/hugetlb.c index c921287489de3..571212b80835e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2862,6 +2862,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, map_chg_state map_chg; int ret, idx; struct hugetlb_cgroup *h_cg = NULL; + struct hugetlb_cgroup *h_cg_rsvd = NULL; gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; idx = hstate_index(h); @@ -2912,7 +2913,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, */ if (map_chg) { ret = hugetlb_cgroup_charge_cgroup_rsvd( - idx, pages_per_huge_page(h), &h_cg); + idx, pages_per_huge_page(h), &h_cg_rsvd); if (ret) goto out_subpool_put; } @@ -2954,7 +2955,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, */ if (map_chg) { hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), - h_cg, folio); + h_cg_rsvd, folio); } spin_unlock_irq(&hugetlb_lock); @@ -3006,7 +3007,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, out_uncharge_cgroup_reservation: if (map_chg) hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), - h_cg); + h_cg_rsvd); out_subpool_put: /* * put page to subpool iff the quota of subpool's rsv_hpages is used diff --git a/mm/khugepaged.c b/mm/khugepaged.c index b8452dbdb043f..d6e04041f5dc1 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2094,32 +2094,43 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, goto xa_unlocked; } - if (!is_shmem) { +xa_locked: + xas_unlock_irq(&xas); +xa_unlocked: + + /* + * If collapse is successful, flush must be done now before copying. + * If collapse is unsuccessful, does flush actually need to be done? + * Do it anyway, to clear the state. + */ + try_to_unmap_flush(); + + if (result == SCAN_SUCCEED && !is_shmem && !mapping_large_folio_support(mapping)) { + /* + * invalidate_lock as shared excludes against concurrent opens + * in do_dentry_open() truncating the page cache. This is + * particularly important if there are dirty folios in transit. + */ + filemap_invalidate_lock_shared(mapping); filemap_nr_thps_inc(mapping); /* * Paired with the fence in do_dentry_open() -> get_write_access() * to ensure i_writecount is up to date and the update to nr_thps * is visible. Ensures the page cache will be truncated if the - * file is opened writable. + * file is opened writable. If collapse looks to be successful, + * flush any dirty pages out the page cache. With the nr_thps + * incremented, there won't be any new writers (nor new dirties). */ smp_mb(); - if (inode_is_open_for_write(mapping->host)) { + if (inode_is_open_for_write(mapping->host) || filemap_write_and_wait(mapping)) { result = SCAN_FAIL; filemap_nr_thps_dec(mapping); + filemap_invalidate_unlock_shared(mapping); + goto rollback; } + filemap_invalidate_unlock_shared(mapping); } -xa_locked: - xas_unlock_irq(&xas); -xa_unlocked: - - /* - * If collapse is successful, flush must be done now before copying. - * If collapse is unsuccessful, does flush actually need to be done? - * Do it anyway, to clear the state. - */ - try_to_unmap_flush(); - if (result == SCAN_SUCCEED && nr_none && !shmem_charge(mapping->host, nr_none)) result = SCAN_FAIL; diff --git a/mm/madvise.c b/mm/madvise.c index 69708e953cf56..5665785e5ac38 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -336,8 +336,7 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) * otherwise we'd be including shared non-exclusive mappings, which * opens a side channel. */ - return inode_owner_or_capable(&nop_mnt_idmap, - file_inode(vma->vm_file)) || + return file_owner_or_capable(vma->vm_file) || file_permission(vma->vm_file, MAY_WRITE) == 0; } diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 40c7915dabe05..cf4f77108c43f 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1470,7 +1470,7 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group, ret = create_memory_block_devices(cur_start, memblock_size, nid, params.altmap, group); if (ret) { - arch_remove_memory(cur_start, memblock_size, NULL); + arch_remove_memory(cur_start, memblock_size, params.altmap); kfree(params.altmap); goto out; } diff --git a/mm/mincore.c b/mm/mincore.c index 296f2e3922b5f..c8757c5085bf9 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -227,8 +227,7 @@ static inline bool can_do_mincore(struct vm_area_struct *vma) * for writing; otherwise we'd be including shared non-exclusive * mappings, which opens a side channel. */ - return inode_owner_or_capable(&nop_mnt_idmap, - file_inode(vma->vm_file)) || + return file_owner_or_capable(vma->vm_file) || file_permission(vma->vm_file, MAY_WRITE) == 0; } diff --git a/mm/mm_init.c b/mm/mm_init.c index f9f8e1af921cd..bd466a3c10c8e 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -674,6 +674,20 @@ static inline void fixup_hashdist(void) static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ +#ifdef CONFIG_ZONE_DEVICE +static __meminit void pageblock_migratetype_init_range(unsigned long pfn, + unsigned long nr_pages, int migratetype) +{ + const unsigned long end = pfn + nr_pages; + + for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) { + init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false); + if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) + cond_resched(); + } +} +#endif + /* * Initialize a reserved page unconditionally, finding its zone first. */ @@ -1011,21 +1025,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, page_folio(page)->pgmap = pgmap; page->zone_device_data = NULL; - /* - * Mark the block movable so that blocks are reserved for - * movable at startup. This will force kernel allocations - * to reserve their blocks rather than leaking throughout - * the address space during boot when many long-lived - * kernel allocations are made. - * - * Please note that MEMINIT_HOTPLUG path doesn't clear memmap - * because this is done early in section_activate() - */ - if (pageblock_aligned(pfn)) { - init_pageblock_migratetype(page, MIGRATE_MOVABLE, false); - cond_resched(); - } - /* * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released * directly to the driver page allocator which will set the page count @@ -1056,10 +1055,17 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, * of how the sparse_vmemmap internals handle compound pages in the lack * of an altmap. See vmemmap_populate_compound_pages(). */ -static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, +static inline unsigned long compound_nr_pages(unsigned long pfn, + struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - if (!vmemmap_can_optimize(altmap, pgmap)) + /* + * If DAX memory is hot-plugged into an unoccupied subsection + * of an early section, the unoptimized boot memmap is reused. + * See section_activate(). + */ + if (early_section(__pfn_to_section(pfn)) || + !vmemmap_can_optimize(altmap, pgmap)) return pgmap_vmemmap_nr(pgmap); return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); @@ -1122,13 +1128,18 @@ void __ref memmap_init_zone_device(struct zone *zone, __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) + cond_resched(); + if (pfns_per_compound == 1) continue; memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(altmap, pgmap)); + compound_nr_pages(pfn, altmap, pgmap)); } + pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE); + pr_debug("%s initialised %lu pages in %ums\n", __func__, nr_pages, jiffies_to_msecs(jiffies - start)); } diff --git a/mm/mmap.c b/mm/mmap.c index 5754d1c364624..2311ae7c2ff45 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -504,7 +504,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, break; case MAP_DROPPABLE: if (VM_DROPPABLE == VM_NONE) - return -ENOTSUPP; + return -EOPNOTSUPP; /* * A locked or stack area makes no sense to be droppable. * diff --git a/mm/numa_emulation.c b/mm/numa_emulation.c index 703c8fa050480..55f26b22bb0be 100644 --- a/mm/numa_emulation.c +++ b/mm/numa_emulation.c @@ -214,7 +214,7 @@ static u64 uniform_size(u64 max_addr, u64 base, u64 hole, int nr_nodes) * Sets up fake nodes of `size' interleaved over physical nodes ranging from * `addr' to `max_addr'. * - * Returns zero on success or negative on error. + * Returns node ID of the next node on success or negative error code. */ static int __init split_nodes_size_interleave_uniform(struct numa_meminfo *ei, struct numa_meminfo *pi, @@ -398,7 +398,7 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt) */ if (strchr(emu_cmdline, 'U')) { unsigned long n; - int nid = 0; + int nid = 0, nr_created; n = simple_strtoul(emu_cmdline, &emu_cmdline, 0); ret = -1; @@ -416,9 +416,18 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt) n, &pi.blk[0], nid); if (ret < 0) break; - if (ret < n) { + + /* + * If no memory was found for this physical node, + * skip the under-allocation check. + */ + if (ret == nid) + continue; + + nr_created = ret - nid; + if (nr_created < n) { pr_info("%s: phys: %d only got %d of %ld nodes, failing\n", - __func__, i, ret, n); + __func__, i, nr_created, n); ret = -1; break; } diff --git a/mm/shmem.c b/mm/shmem.c index 3b5dc21b323c2..7b1ea9fb598f4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1425,10 +1425,8 @@ static void shmem_evict_inode(struct inode *inode) } } - if (info->xattrs) { - simple_xattrs_free(info->xattrs, sbinfo->max_inodes ? &freed : NULL); - kfree(info->xattrs); - } + simple_xattrs_free(&sbinfo->xa_cache, &info->xattrs, sbinfo->max_inodes ? &freed : NULL); + shmem_free_inode(inode->i_sb, freed); WARN_ON(inode->i_blocks); clear_inode(inode); @@ -3086,6 +3084,7 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, inode->i_generation = get_random_u32(); info = SHMEM_I(inode); memset(info, 0, (char *)inode - (char *)info); + INIT_LIST_HEAD_RCU(&info->xattrs); spin_lock_init(&info->lock); atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; @@ -4232,11 +4231,6 @@ static int shmem_initxattrs(struct inode *inode, struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); const struct xattr *xattr; size_t ispace = 0; - size_t len; - - CLASS(simple_xattrs, xattrs)(); - if (IS_ERR(xattrs)) - return PTR_ERR(xattrs); if (sbinfo->max_inodes) { for (xattr = xattr_array; xattr->name != NULL; xattr++) { @@ -4260,19 +4254,16 @@ static int shmem_initxattrs(struct inode *inode, if (IS_ERR(new_xattr)) break; - len = strlen(xattr->name) + 1; - new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, - GFP_KERNEL_ACCOUNT); + new_xattr->name = kasprintf(GFP_KERNEL_ACCOUNT, + XATTR_SECURITY_PREFIX "%s", xattr->name); if (!new_xattr->name) break; - memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, - XATTR_SECURITY_PREFIX_LEN); - memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, - xattr->name, len); - - if (simple_xattr_add(xattrs, new_xattr)) + if (simple_xattr_add(&sbinfo->xa_cache, &info->xattrs, new_xattr)) break; + + if (sbinfo->max_inodes) + ispace -= simple_xattr_space(new_xattr->name, new_xattr->size); retain_and_null_ptr(new_xattr); } @@ -4284,8 +4275,8 @@ static int shmem_initxattrs(struct inode *inode, } return -ENOMEM; } + WARN_ON(ispace); - smp_store_release(&info->xattrs, no_free_ptr(xattrs)); return 0; } @@ -4293,15 +4284,11 @@ static int shmem_xattr_handler_get(const struct xattr_handler *handler, struct dentry *unused, struct inode *inode, const char *name, void *buffer, size_t size) { + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); struct shmem_inode_info *info = SHMEM_I(inode); - struct simple_xattrs *xattrs; - - xattrs = READ_ONCE(info->xattrs); - if (!xattrs) - return -ENODATA; name = xattr_full_name(handler, name); - return simple_xattr_get(xattrs, name, buffer, size); + return simple_xattr_get(&sbinfo->xa_cache, &info->xattrs, name, buffer, size); } static int shmem_xattr_handler_set(const struct xattr_handler *handler, @@ -4312,16 +4299,11 @@ static int shmem_xattr_handler_set(const struct xattr_handler *handler, { struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); - struct simple_xattrs *xattrs; struct simple_xattr *old_xattr; size_t ispace = 0; name = xattr_full_name(handler, name); - xattrs = simple_xattrs_lazy_alloc(&info->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); - if (value && sbinfo->max_inodes) { ispace = simple_xattr_space(name, size); raw_spin_lock(&sbinfo->stat_lock); @@ -4334,7 +4316,7 @@ static int shmem_xattr_handler_set(const struct xattr_handler *handler, return -ENOSPC; } - old_xattr = simple_xattr_set(xattrs, name, value, size, flags); + old_xattr = simple_xattr_set(&sbinfo->xa_cache, &info->xattrs, name, value, size, flags); if (!IS_ERR(old_xattr)) { ispace = 0; if (old_xattr && sbinfo->max_inodes) @@ -4382,8 +4364,7 @@ static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) { struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); - return simple_xattr_list(d_inode(dentry), READ_ONCE(info->xattrs), - buffer, size); + return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); } #endif /* CONFIG_TMPFS_XATTR */ @@ -4984,6 +4965,9 @@ static void shmem_put_super(struct super_block *sb) free_percpu(sbinfo->ino_batch); percpu_counter_destroy(&sbinfo->used_blocks); mpol_put(sbinfo->mpol); +#ifdef CONFIG_TMPFS_XATTR + simple_xattr_cache_cleanup(&sbinfo->xa_cache); +#endif kfree(sbinfo); sb->s_fs_info = NULL; } diff --git a/mm/shrinker.c b/mm/shrinker.c index 76b3f750cf655..d6c1a9b1e34a4 100644 --- a/mm/shrinker.c +++ b/mm/shrinker.c @@ -59,12 +59,14 @@ static inline int shrinker_unit_alloc(struct shrinker_info *new, return 0; } -void free_shrinker_info(struct mem_cgroup *memcg) +static void __free_shrinker_info(struct mem_cgroup *memcg) { struct mem_cgroup_per_node *pn; struct shrinker_info *info; int nid; + lockdep_assert_held(&shrinker_mutex); + for_each_node(nid) { pn = memcg->nodeinfo[nid]; info = rcu_dereference_protected(pn->shrinker_info, true); @@ -74,6 +76,13 @@ void free_shrinker_info(struct mem_cgroup *memcg) } } +void free_shrinker_info(struct mem_cgroup *memcg) +{ + mutex_lock(&shrinker_mutex); + __free_shrinker_info(memcg); + mutex_unlock(&shrinker_mutex); +} + int alloc_shrinker_info(struct mem_cgroup *memcg) { int nid, ret = 0; @@ -98,8 +107,8 @@ int alloc_shrinker_info(struct mem_cgroup *memcg) return ret; err: + __free_shrinker_info(memcg); mutex_unlock(&shrinker_mutex); - free_shrinker_info(memcg); return -ENOMEM; } diff --git a/mm/shrinker_debug.c b/mm/shrinker_debug.c index affa644373021..cafb566301326 100644 --- a/mm/shrinker_debug.c +++ b/mm/shrinker_debug.c @@ -57,8 +57,6 @@ static int shrinker_debugfs_count_show(struct seq_file *m, void *v) if (!count_per_node) return -ENOMEM; - rcu_read_lock(); - memcg_aware = shrinker->flags & SHRINKER_MEMCG_AWARE; memcg = mem_cgroup_iter(NULL, NULL, NULL); @@ -88,8 +86,6 @@ static int shrinker_debugfs_count_show(struct seq_file *m, void *v) } } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); - rcu_read_unlock(); - kfree(count_per_node); return ret; } @@ -187,10 +183,12 @@ int shrinker_debugfs_add(struct shrinker *shrinker) } shrinker->debugfs_entry = entry; - debugfs_create_file("count", 0440, entry, shrinker, - &shrinker_debugfs_count_fops); - debugfs_create_file("scan", 0220, entry, shrinker, - &shrinker_debugfs_scan_fops); + if (shrinker->count_objects) + debugfs_create_file("count", 0440, entry, shrinker, + &shrinker_debugfs_count_fops); + if (shrinker->scan_objects) + debugfs_create_file("scan", 0220, entry, shrinker, + &shrinker_debugfs_scan_fops); return 0; } diff --git a/mm/slub.c b/mm/slub.c index a2bf3756ca7d0..eed3251eb7d0c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4537,15 +4537,17 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, gfp_t init_flags = flags & gfp_allowed_mask; /* - * For kmalloc object, the allocated memory size(object_size) is likely - * larger than the requested size(orig_size). If redzone check is - * enabled for the extra space, don't zero it, as it will be redzoned - * soon. The redzone operation for this extra space could be seen as a - * replacement of current poisoning under certain debug option, and - * won't break other sanity checks. + * For kmalloc object, the allocated size (object_size) can be larger + * than the requested size (orig_size). We however need to zero the + * whole object_size to handle possible later krealloc() with + *__GFP_ZERO properly. + * + * But if we keep track of the requested size, krealloc() uses that + * information. Additionally if red zoning is enabled, the extra space + * is also red zone, so we should not overwrite it. So limit zeroing to + * orig_size if we track it. */ - if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) && - (s->flags & SLAB_KMALLOC)) + if (slub_debug_orig_size(s)) zero_size = orig_size; /* @@ -5323,6 +5325,7 @@ EXPORT_SYMBOL(__kmalloc_noprof); void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) { gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags; + size_t orig_size = size; struct kmem_cache *s; bool can_retry = true; void *ret; @@ -5371,7 +5374,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) * kfence_alloc. Hence call __slab_alloc_node() (at most twice) * and slab_post_alloc_hook() directly. */ - ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); + ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, orig_size); /* * It's possible we failed due to trylock as we preempted someone with @@ -5395,9 +5398,9 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) success: maybe_wipe_obj_freeptr(s, ret); slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, - slab_want_init_on_alloc(alloc_gfp, s), size); + slab_want_init_on_alloc(alloc_gfp, s), orig_size); - ret = kasan_kmalloc(s, ret, size, alloc_gfp); + ret = kasan_kmalloc(s, ret, orig_size, alloc_gfp); return ret; } EXPORT_SYMBOL_GPL(kmalloc_nolock_noprof); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 6eadb9d116e43..a7b11248b989c 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -656,7 +656,12 @@ static struct page * __meminit populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); + struct page *page = __populate_section_memmap(pfn, nr_pages, nid, altmap, + pgmap); + + memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)); + + return page; } static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, @@ -665,13 +670,17 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, unsigned long start = (unsigned long) pfn_to_page(pfn); unsigned long end = start + nr_pages * sizeof(struct page); + memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE))); vmemmap_free(start, end, altmap); } + static void free_map_bootmem(struct page *memmap) { unsigned long start = (unsigned long)memmap; unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); + memmap_boot_pages_add(-1L * (DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), + PAGE_SIZE))); vmemmap_free(start, end, NULL); } @@ -774,14 +783,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, * The memmap of early sections is always fully populated. See * section_activate() and pfn_valid() . */ - if (!section_is_early) { - memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE))); + if (!section_is_early) depopulate_section_memmap(pfn, nr_pages, altmap); - } else if (memmap) { - memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), - PAGE_SIZE))); + else if (memmap) free_map_bootmem(memmap); - } if (empty) ms->section_mem_map = (unsigned long)NULL; @@ -826,7 +831,6 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, section_deactivate(pfn, nr_pages, altmap); return ERR_PTR(-ENOMEM); } - memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)); return memmap; } diff --git a/mm/swap_cgroup.c b/mm/swap_cgroup.c index de779fed8c210..95c38e54dd587 100644 --- a/mm/swap_cgroup.c +++ b/mm/swap_cgroup.c @@ -124,6 +124,8 @@ unsigned short lookup_swap_cgroup_id(swp_entry_t ent) return 0; ctrl = &swap_cgroup_ctrl[swp_type(ent)]; + if (unlikely(!ctrl->map)) + return 0; return __swap_cgroup_id_lookup(ctrl->map, swp_offset(ent)); } diff --git a/mm/swapfile.c b/mm/swapfile.c index 9174f1eeffb09..74a1e324449dc 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1054,6 +1054,7 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force) swap_cluster_unlock(ci); if (to_scan <= 0) break; + cond_resched(); } } diff --git a/net/6lowpan/nhc.c b/net/6lowpan/nhc.c index 7b374595328de..a4dde85664f2f 100644 --- a/net/6lowpan/nhc.c +++ b/net/6lowpan/nhc.c @@ -117,9 +117,9 @@ int lowpan_nhc_do_uncompression(struct sk_buff *skb, return ret; } } else { - spin_unlock_bh(&lowpan_nhc_lock); netdev_warn(dev, "received nhc id for %s which is not implemented.\n", nhc->name); + spin_unlock_bh(&lowpan_nhc_lock); return -ENOTSUPP; } } else { diff --git a/net/9p/client.c b/net/9p/client.c index f0dcf252af7e1..7d1535edfce7f 100644 --- a/net/9p/client.c +++ b/net/9p/client.c @@ -600,6 +600,8 @@ p9_client_rpc(struct p9_client *c, int8_t type, const char *fmt, ...) if (err == -ERESTARTSYS && c->status == Connected && type == P9_TFLUSH) { + if (fatal_signal_pending(current)) + goto recalc_sigpending; sigpending = 1; clear_thread_flag(TIF_SIGPENDING); goto again; @@ -1092,7 +1094,8 @@ struct p9_fid *p9_client_walk(struct p9_fid *oldfid, uint16_t nwname, clunk_fid: kfree(wqids); - p9_fid_put(fid); + if (fid != oldfid) + p9_fid_put(fid); fid = NULL; error: diff --git a/net/9p/trans_rdma.c b/net/9p/trans_rdma.c index aa5bd74d333f3..b4274f10fa448 100644 --- a/net/9p/trans_rdma.c +++ b/net/9p/trans_rdma.c @@ -128,25 +128,36 @@ p9_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { struct p9_client *c = id->context; struct p9_trans_rdma *rdma = c->trans; + unsigned long flags; + switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: + spin_lock_irqsave(&rdma->req_lock, flags); BUG_ON(rdma->state != P9_RDMA_INIT); rdma->state = P9_RDMA_ADDR_RESOLVED; + spin_unlock_irqrestore(&rdma->req_lock, flags); break; case RDMA_CM_EVENT_ROUTE_RESOLVED: + spin_lock_irqsave(&rdma->req_lock, flags); BUG_ON(rdma->state != P9_RDMA_ADDR_RESOLVED); rdma->state = P9_RDMA_ROUTE_RESOLVED; + spin_unlock_irqrestore(&rdma->req_lock, flags); break; case RDMA_CM_EVENT_ESTABLISHED: + spin_lock_irqsave(&rdma->req_lock, flags); BUG_ON(rdma->state != P9_RDMA_ROUTE_RESOLVED); rdma->state = P9_RDMA_CONNECTED; + spin_unlock_irqrestore(&rdma->req_lock, flags); break; case RDMA_CM_EVENT_DISCONNECTED: - if (rdma) + if (rdma) { + spin_lock_irqsave(&rdma->req_lock, flags); rdma->state = P9_RDMA_CLOSED; + spin_unlock_irqrestore(&rdma->req_lock, flags); + } c->status = Disconnected; break; @@ -184,6 +195,7 @@ recv_done(struct ib_cq *cq, struct ib_wc *wc) struct p9_req_t *req; int err = 0; int16_t tag; + unsigned long flags; req = NULL; ib_dma_unmap_single(rdma->cm_id->device, c->busa, client->msize, @@ -220,7 +232,10 @@ recv_done(struct ib_cq *cq, struct ib_wc *wc) err_out: p9_debug(P9_DEBUG_ERROR, "req %p err %d status %d\n", req, err, wc->status); - rdma->state = P9_RDMA_FLUSHING; + spin_lock_irqsave(&rdma->req_lock, flags); + if (rdma->state < P9_RDMA_FLUSHING) + rdma->state = P9_RDMA_FLUSHING; + spin_unlock_irqrestore(&rdma->req_lock, flags); client->status = Disconnected; goto out; } diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c index 4cdab7094b273..b0d0094ec8e2c 100644 --- a/net/9p/trans_virtio.c +++ b/net/9p/trans_virtio.c @@ -532,6 +532,11 @@ p9_virtio_zc_request(struct p9_client *client, struct p9_req_t *req, p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n"); err = io_wait_event_killable(req->wq, READ_ONCE(req->status) >= REQ_STATUS_RCVD); + /* + * Make sure our req is coherent with regard to updates in other + * threads - echoes to wmb() in the callback + */ + smp_rmb(); // RERROR needs reply (== error string) in static data if (READ_ONCE(req->status) == REQ_STATUS_RCVD && unlikely(req->rc.sdata[4] == P9_RERROR)) diff --git a/net/atm/common.c b/net/atm/common.c index fe77f51f6ce18..6eb78c34c2843 100644 --- a/net/atm/common.c +++ b/net/atm/common.c @@ -720,6 +720,8 @@ static int atm_change_qos(struct atm_vcc *vcc, struct atm_qos *qos) static int check_tp(const struct atm_trafprm *tp) { /* @@@ Should be merged with adjust_tp */ + if (tp->traffic_class > ATM_ANYCLASS) + return -EINVAL; if (!tp->traffic_class || tp->traffic_class == ATM_ANYCLASS) return 0; if (tp->traffic_class != ATM_UBR && !tp->min_pcr && !tp->pcr && diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c index b8b1b997960a9..6e79f69c2fede 100644 --- a/net/batman-adv/bat_iv_ogm.c +++ b/net/batman-adv/bat_iv_ogm.c @@ -311,14 +311,23 @@ batadv_iv_ogm_aggr_packet(int buff_pos, int packet_len, const struct batadv_ogm_packet *ogm_packet) { int next_buff_pos = 0; + u16 tvlv_len; /* check if there is enough space for the header */ next_buff_pos += buff_pos + sizeof(*ogm_packet); if (next_buff_pos > packet_len) return false; + tvlv_len = ntohs(ogm_packet->tvlv_len); + + /* the fields of an aggregated OGM are accessed assuming (at least) + * 2-byte alignment, so a following OGM must start at an even offset. + */ + if (tvlv_len & 1) + return false; + /* check if there is enough space for the optional TVLV */ - next_buff_pos += ntohs(ogm_packet->tvlv_len); + next_buff_pos += tvlv_len; return next_buff_pos <= packet_len; } diff --git a/net/batman-adv/bat_v.c b/net/batman-adv/bat_v.c index de94447142642..17d2a1ccdce67 100644 --- a/net/batman-adv/bat_v.c +++ b/net/batman-adv/bat_v.c @@ -817,6 +817,7 @@ void batadv_v_hardif_init(struct batadv_hard_iface *hard_iface) hard_iface->bat_v.aggr_len = 0; skb_queue_head_init(&hard_iface->bat_v.aggr_list); + hard_iface->bat_v.aggr_list_enabled = false; INIT_DELAYED_WORK(&hard_iface->bat_v.aggr_wq, batadv_v_ogm_aggr_work); } diff --git a/net/batman-adv/bat_v_ogm.c b/net/batman-adv/bat_v_ogm.c index d66ca77b1aaa3..1f9b2d2b4831c 100644 --- a/net/batman-adv/bat_v_ogm.c +++ b/net/batman-adv/bat_v_ogm.c @@ -252,11 +252,18 @@ static void batadv_v_ogm_queue_on_if(struct batadv_priv *bat_priv, } spin_lock_bh(&hard_iface->bat_v.aggr_list.lock); + if (!hard_iface->bat_v.aggr_list_enabled) { + kfree_skb(skb); + goto unlock; + } + if (!batadv_v_ogm_queue_left(skb, hard_iface)) batadv_v_ogm_aggr_send(bat_priv, hard_iface); hard_iface->bat_v.aggr_len += batadv_v_ogm_len(skb); __skb_queue_tail(&hard_iface->bat_v.aggr_list, skb); + +unlock: spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock); } @@ -417,6 +424,10 @@ int batadv_v_ogm_iface_enable(struct batadv_hard_iface *hard_iface) { struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface); + spin_lock_bh(&hard_iface->bat_v.aggr_list.lock); + hard_iface->bat_v.aggr_list_enabled = true; + spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock); + batadv_v_ogm_start_queue_timer(hard_iface); batadv_v_ogm_start_timer(bat_priv); @@ -432,6 +443,7 @@ void batadv_v_ogm_iface_disable(struct batadv_hard_iface *hard_iface) cancel_delayed_work_sync(&hard_iface->bat_v.aggr_wq); spin_lock_bh(&hard_iface->bat_v.aggr_list.lock); + hard_iface->bat_v.aggr_list_enabled = false; batadv_v_ogm_aggr_list_free(hard_iface); spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock); } @@ -837,14 +849,23 @@ batadv_v_ogm_aggr_packet(int buff_pos, int packet_len, const struct batadv_ogm2_packet *ogm2_packet) { int next_buff_pos = 0; + u16 tvlv_len; /* check if there is enough space for the header */ next_buff_pos += buff_pos + sizeof(*ogm2_packet); if (next_buff_pos > packet_len) return false; + tvlv_len = ntohs(ogm2_packet->tvlv_len); + + /* the fields of an aggregated OGMv2 are accessed assuming (at least) + * 2-byte alignment, so a following OGMv2 must start at an even offset. + */ + if (tvlv_len & 1) + return false; + /* check if there is enough space for the optional TVLV */ - next_buff_pos += ntohs(ogm2_packet->tvlv_len); + next_buff_pos += tvlv_len; return next_buff_pos <= packet_len; } diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index ffe854018bd3a..7f643de42579d 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -512,7 +512,7 @@ batadv_bla_get_backbone_gw(struct batadv_priv *bat_priv, const u8 *orig, return NULL; entry->vid = vid; - entry->lasttime = jiffies; + WRITE_ONCE(entry->lasttime, jiffies); entry->crc = BATADV_BLA_CRC_INIT; entry->bat_priv = bat_priv; spin_lock_init(&entry->crc_lock); @@ -580,7 +580,7 @@ batadv_bla_update_own_backbone_gw(struct batadv_priv *bat_priv, if (unlikely(!backbone_gw)) return; - backbone_gw->lasttime = jiffies; + WRITE_ONCE(backbone_gw->lasttime, jiffies); batadv_backbone_gw_put(backbone_gw); } @@ -714,7 +714,7 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, ether_addr_copy(claim->addr, mac); spin_lock_init(&claim->backbone_lock); claim->vid = vid; - claim->lasttime = jiffies; + WRITE_ONCE(claim->lasttime, jiffies); kref_get(&backbone_gw->refcount); claim->backbone_gw = backbone_gw; kref_init(&claim->refcount); @@ -736,7 +736,7 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, return; } } else { - claim->lasttime = jiffies; + WRITE_ONCE(claim->lasttime, jiffies); if (claim->backbone_gw == backbone_gw) /* no need to register a new backbone */ goto claim_free_ref; @@ -769,7 +769,7 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, spin_lock_bh(&backbone_gw->crc_lock); backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); spin_unlock_bh(&backbone_gw->crc_lock); - backbone_gw->lasttime = jiffies; + WRITE_ONCE(backbone_gw->lasttime, jiffies); claim_free_ref: batadv_claim_put(claim); @@ -858,7 +858,7 @@ static bool batadv_handle_announce(struct batadv_priv *bat_priv, u8 *an_addr, return true; /* handle as ANNOUNCE frame */ - backbone_gw->lasttime = jiffies; + WRITE_ONCE(backbone_gw->lasttime, jiffies); crc = ntohs(*((__force __be16 *)(&an_addr[4]))); batadv_dbg(BATADV_DBG_BLA, bat_priv, @@ -1253,7 +1253,7 @@ static void batadv_bla_purge_backbone_gw(struct batadv_priv *bat_priv, int now) head, hash_entry) { if (now) goto purge_now; - if (!batadv_has_timed_out(backbone_gw->lasttime, + if (!batadv_has_timed_out(READ_ONCE(backbone_gw->lasttime), BATADV_BLA_BACKBONE_TIMEOUT)) continue; @@ -1334,7 +1334,7 @@ static void batadv_bla_purge_claims(struct batadv_priv *bat_priv, primary_if->net_dev->dev_addr)) goto skip; - if (!batadv_has_timed_out(claim->lasttime, + if (!batadv_has_timed_out(READ_ONCE(claim->lasttime), BATADV_BLA_CLAIM_TIMEOUT)) goto skip; @@ -1494,7 +1494,7 @@ static void batadv_bla_periodic_work(struct work_struct *work) eth_random_addr(bat_priv->bla.loopdetect_addr); bat_priv->bla.loopdetect_addr[0] = 0xba; bat_priv->bla.loopdetect_addr[1] = 0xbe; - bat_priv->bla.loopdetect_lasttime = jiffies; + WRITE_ONCE(bat_priv->bla.loopdetect_lasttime, jiffies); atomic_set(&bat_priv->bla.loopdetect_next, BATADV_BLA_LOOPDETECT_PERIODS); @@ -1515,7 +1515,7 @@ static void batadv_bla_periodic_work(struct work_struct *work) primary_if->net_dev->dev_addr)) continue; - backbone_gw->lasttime = jiffies; + WRITE_ONCE(backbone_gw->lasttime, jiffies); batadv_bla_send_announce(bat_priv, backbone_gw); if (send_loopdetect) @@ -1899,7 +1899,7 @@ batadv_bla_loopdetect_check(struct batadv_priv *bat_priv, struct sk_buff *skb, /* If the packet came too late, don't forward it on the mesh * but don't consider that as loop. It might be a coincidence. */ - if (batadv_has_timed_out(bat_priv->bla.loopdetect_lasttime, + if (batadv_has_timed_out(READ_ONCE(bat_priv->bla.loopdetect_lasttime), BATADV_BLA_LOOPDETECT_TIMEOUT)) return true; @@ -2014,7 +2014,7 @@ bool batadv_bla_rx(struct batadv_priv *bat_priv, struct sk_buff *skb, if (own_claim) { /* ... allow it in any case */ - claim->lasttime = jiffies; + WRITE_ONCE(claim->lasttime, jiffies); goto allow; } @@ -2116,7 +2116,7 @@ bool batadv_bla_tx(struct batadv_priv *bat_priv, struct sk_buff *skb, /* if yes, the client has roamed and we have * to unclaim it. */ - if (batadv_has_timed_out(claim->lasttime, 100)) { + if (batadv_has_timed_out(READ_ONCE(claim->lasttime), 100)) { /* only unclaim if the last claim entry is * older than 100 ms to make sure we really * have a roaming client here. @@ -2361,7 +2361,7 @@ batadv_bla_backbone_dump_entry(struct sk_buff *msg, u32 portid, backbone_crc = backbone_gw->crc; spin_unlock_bh(&backbone_gw->crc_lock); - msecs = jiffies_to_msecs(jiffies - backbone_gw->lasttime); + msecs = jiffies_to_msecs(jiffies - READ_ONCE(backbone_gw->lasttime)); if (is_own) if (nla_put_flag(msg, BATADV_ATTR_BLA_OWN)) { diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c index 0a8bd95e2f99e..bcfbb39889a1c 100644 --- a/net/batman-adv/distributed-arp-table.c +++ b/net/batman-adv/distributed-arp-table.c @@ -214,10 +214,13 @@ static void batadv_dat_purge(struct work_struct *work) */ static bool batadv_compare_dat(const struct hlist_node *node, const void *data2) { - const void *data1 = container_of(node, struct batadv_dat_entry, - hash_entry); + const struct batadv_dat_entry *entry1; + const struct batadv_dat_entry *entry2; - return memcmp(data1, data2, sizeof(__be32)) == 0; + entry1 = container_of(node, struct batadv_dat_entry, hash_entry); + entry2 = data2; + + return entry1->ip == entry2->ip && entry1->vid == entry2->vid; } /** @@ -344,6 +347,9 @@ batadv_dat_entry_hash_find(struct batadv_priv *bat_priv, __be32 ip, if (dat_entry->ip != ip) continue; + if (dat_entry->vid != vid) + continue; + if (!kref_get_unless_zero(&dat_entry->refcount)) continue; @@ -539,7 +545,7 @@ static bool batadv_is_orig_node_eligible(struct batadv_dat_candidate *res, * the one with the lowest address */ if (tmp_max == max && max_orig_node && - batadv_compare_eth(candidate->orig, max_orig_node->orig)) + memcmp(candidate->orig, max_orig_node->orig, ETH_ALEN) >= 0) goto out; ret = true; @@ -1059,6 +1065,9 @@ static u16 batadv_arp_get_type(struct batadv_priv *bat_priv, * @skb: the buffer containing the packet to extract the VID from * @hdr_size: the size of the batman-adv header encapsulating the packet * + * The caller must ensure that at least @hdr_size + ETH_HLEN bytes are + * accessible after skb->data. + * * Return: If the packet embedded in the skb is vlan tagged this function * returns the VID with the BATADV_VLAN_HAS_TAG flag. Otherwise BATADV_NO_FLAGS * is returned. @@ -1141,6 +1150,10 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, if (!atomic_read(&bat_priv->distributed_arp_table)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1236,6 +1249,10 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, if (!atomic_read(&bat_priv->distributed_arp_table)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1298,6 +1315,10 @@ void batadv_dat_snoop_outgoing_arp_reply(struct batadv_priv *bat_priv, if (!atomic_read(&bat_priv->distributed_arp_table)) return; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + return; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1346,6 +1367,10 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, if (!atomic_read(&bat_priv->distributed_arp_table)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1740,6 +1765,7 @@ void batadv_dat_snoop_incoming_dhcp_ack(struct batadv_priv *bat_priv, struct ethhdr *ethhdr; __be32 ip_src, yiaddr; unsigned short vid; + int hdr_size_tmp; __be16 proto; u8 *hw_src; @@ -1756,8 +1782,10 @@ void batadv_dat_snoop_incoming_dhcp_ack(struct batadv_priv *bat_priv, if (!batadv_dat_check_dhcp_ack(skb, proto, &ip_src, chaddr, &yiaddr)) return; + hdr_size_tmp = hdr_size; + vid = batadv_dat_get_vid(skb, &hdr_size_tmp); + ethhdr = (struct ethhdr *)(skb->data + hdr_size); hw_src = ethhdr->h_source; - vid = batadv_dat_get_vid(skb, &hdr_size); batadv_dat_entry_add(bat_priv, yiaddr, chaddr, vid); batadv_dat_entry_add(bat_priv, ip_src, hw_src, vid); @@ -1797,6 +1825,10 @@ bool batadv_dat_drop_broadcast_packet(struct batadv_priv *bat_priv, if (batadv_forw_packet_is_rebroadcast(forw_packet)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(forw_packet->skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(forw_packet->skb, &hdr_size); type = batadv_arp_get_type(bat_priv, forw_packet->skb, hdr_size); diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c index e9553db423491..9a8db648f957e 100644 --- a/net/batman-adv/fragmentation.c +++ b/net/batman-adv/fragmentation.c @@ -384,6 +384,8 @@ bool batadv_frag_skb_buffer(struct sk_buff **skb, * @skb: skb to forward * @recv_if: interface that the skb is received on * @orig_node_src: originator that the skb is received from + * @rx_result: set to NET_RX_SUCCESS when the fragment was forwarded and + * NET_RX_DROP when it was dropped; only valid when true is returned * * Look up the next-hop of the fragments payload and check if the merged packet * will exceed the MTU towards the next-hop. If so, the fragment is forwarded @@ -393,7 +395,8 @@ bool batadv_frag_skb_buffer(struct sk_buff **skb, */ bool batadv_frag_skb_fwd(struct sk_buff *skb, struct batadv_hard_iface *recv_if, - struct batadv_orig_node *orig_node_src) + struct batadv_orig_node *orig_node_src, + int *rx_result) { struct batadv_priv *bat_priv = netdev_priv(recv_if->mesh_iface); struct batadv_neigh_node *neigh_node = NULL; @@ -412,12 +415,29 @@ bool batadv_frag_skb_fwd(struct sk_buff *skb, */ total_size = ntohs(packet->total_size); if (total_size > neigh_node->if_incoming->net_dev->mtu) { + if (packet->ttl < 2) { + kfree_skb(skb); + *rx_result = NET_RX_DROP; + ret = true; + goto out; + } + + if (skb_cow(skb, ETH_HLEN) < 0) { + kfree_skb(skb); + *rx_result = NET_RX_DROP; + ret = true; + goto out; + } + + packet = (struct batadv_frag_packet *)skb->data; + batadv_inc_counter(bat_priv, BATADV_CNT_FRAG_FWD); batadv_add_counter(bat_priv, BATADV_CNT_FRAG_FWD_BYTES, skb->len + ETH_HLEN); packet->ttl--; batadv_send_unicast_skb(skb, neigh_node); + *rx_result = NET_RX_SUCCESS; ret = true; } @@ -496,8 +516,10 @@ int batadv_frag_send_packet(struct sk_buff *skb, mtu = min_t(unsigned int, mtu, BATADV_FRAG_MAX_FRAG_SIZE); max_fragment_size = mtu - header_size; - if (skb->len == 0 || max_fragment_size == 0) - return -EINVAL; + if (skb->len == 0 || max_fragment_size == 0) { + ret = -EINVAL; + goto free_skb; + } num_fragments = (skb->len - 1) / max_fragment_size + 1; max_fragment_size = (skb->len - 1) / num_fragments + 1; @@ -523,7 +545,7 @@ int batadv_frag_send_packet(struct sk_buff *skb, */ if (skb_has_frag_list(skb) && __skb_linearize(skb)) { ret = -ENOMEM; - goto free_skb; + goto put_primary_if; } /* Create one header to be copied to all fragments */ diff --git a/net/batman-adv/fragmentation.h b/net/batman-adv/fragmentation.h index dbf0871f87030..51e281027ab63 100644 --- a/net/batman-adv/fragmentation.h +++ b/net/batman-adv/fragmentation.h @@ -19,7 +19,8 @@ void batadv_frag_purge_orig(struct batadv_orig_node *orig, bool (*check_cb)(struct batadv_frag_table_entry *)); bool batadv_frag_skb_fwd(struct sk_buff *skb, struct batadv_hard_iface *recv_if, - struct batadv_orig_node *orig_node_src); + struct batadv_orig_node *orig_node_src, + int *rx_result); bool batadv_frag_skb_buffer(struct sk_buff **skb, struct batadv_orig_node *orig_node); int batadv_frag_send_packet(struct sk_buff *skb, diff --git a/net/batman-adv/gateway_client.c b/net/batman-adv/gateway_client.c index a9d0346e8332e..36137acb6eb22 100644 --- a/net/batman-adv/gateway_client.c +++ b/net/batman-adv/gateway_client.c @@ -683,12 +683,13 @@ bool batadv_gw_out_of_range(struct batadv_priv *bat_priv, struct batadv_gw_node *gw_node = NULL; struct batadv_gw_node *curr_gw = NULL; struct batadv_neigh_ifinfo *curr_ifinfo, *old_ifinfo; - struct ethhdr *ethhdr = (struct ethhdr *)skb->data; + struct ethhdr *ethhdr; bool out_of_range = false; u8 curr_tq_avg; unsigned short vid; vid = batadv_get_vid(skb, 0); + ethhdr = (struct ethhdr *)skb->data; if (is_multicast_ether_addr(ethhdr->h_dest)) goto out; diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c index d6732c34aeafc..c7dbd24ea1998 100644 --- a/net/batman-adv/hard-interface.c +++ b/net/batman-adv/hard-interface.c @@ -786,30 +786,6 @@ int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface, return ret; } -/** - * batadv_hardif_cnt() - get number of interfaces enslaved to mesh interface - * @mesh_iface: mesh interface to check - * - * This function is only using RCU for locking - the result can therefore be - * off when another function is modifying the list at the same time. The - * caller can use the rtnl_lock to make sure that the count is accurate. - * - * Return: number of connected/enslaved hard interfaces - */ -static size_t batadv_hardif_cnt(struct net_device *mesh_iface) -{ - struct batadv_hard_iface *hard_iface; - struct list_head *iter; - size_t count = 0; - - rcu_read_lock(); - netdev_for_each_lower_private_rcu(mesh_iface, hard_iface, iter) - count++; - rcu_read_unlock(); - - return count; -} - /** * batadv_hardif_disable_interface() - Remove hard interface from mesh interface * @hard_iface: hard interface to be removed @@ -850,8 +826,8 @@ void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface) netdev_upper_dev_unlink(hard_iface->net_dev, hard_iface->mesh_iface); batadv_hardif_recalc_extra_skbroom(hard_iface->mesh_iface); - /* nobody uses this interface anymore */ - if (batadv_hardif_cnt(hard_iface->mesh_iface) <= 1) + /* nobody uses this mesh interface anymore */ + if (list_empty(&hard_iface->mesh_iface->adj_list.lower)) batadv_gw_check_client_stop(bat_priv); hard_iface->mesh_iface = NULL; diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c index a4d33ee0fda59..b9b0bef448030 100644 --- a/net/batman-adv/main.c +++ b/net/batman-adv/main.c @@ -245,6 +245,7 @@ int batadv_mesh_init(struct net_device *mesh_iface) void batadv_mesh_free(struct net_device *mesh_iface) { struct batadv_priv *bat_priv = netdev_priv(mesh_iface); + struct batadv_meshif_vlan *vlan; atomic_set(&bat_priv->mesh_state, BATADV_MESH_DEACTIVATING); @@ -259,6 +260,13 @@ void batadv_mesh_free(struct net_device *mesh_iface) batadv_mcast_free(bat_priv); + /* destroy the "untagged" VLAN */ + vlan = batadv_meshif_vlan_get(bat_priv, BATADV_NO_FLAGS); + if (vlan) { + batadv_meshif_destroy_vlan(bat_priv, vlan); + batadv_meshif_vlan_put(vlan); + } + /* Free the TT and the originator tables only after having terminated * all the other depending components which may use these structures for * their purposes. @@ -354,7 +362,7 @@ void batadv_skb_set_priority(struct sk_buff *skb, int offset) switch (ethhdr->h_proto) { case htons(ETH_P_8021Q): - vhdr = skb_header_pointer(skb, offset + sizeof(*vhdr), + vhdr = skb_header_pointer(skb, offset, sizeof(*vhdr), &vhdr_tmp); if (!vhdr) return; @@ -566,6 +574,9 @@ void batadv_recv_handler_unregister(u8 packet_type) * @skb: the buffer containing the packet * @header_len: length of the batman header preceding the ethernet header * + * The caller must ensure that at least @header_len + ETH_HLEN bytes are + * accessible after skb->data. + * * Return: VID with the BATADV_VLAN_HAS_TAG flag when the packet embedded in the * skb is vlan tagged. Otherwise BATADV_NO_FLAGS. */ diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c index e7aa45bc6b7ad..6f4c8da4fa5d3 100644 --- a/net/batman-adv/mesh-interface.c +++ b/net/batman-adv/mesh-interface.c @@ -193,6 +193,9 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, if (atomic_read(&bat_priv->mesh_state) != BATADV_MESH_ACTIVE) goto dropped; + if (!pskb_may_pull(skb, ETH_HLEN)) + goto dropped; + /* reset control block to avoid left overs from previous users */ memset(skb->cb, 0, sizeof(struct batadv_skb_cb)); @@ -432,6 +435,7 @@ void batadv_interface_rx(struct net_device *mesh_iface, if (!pskb_may_pull(skb, VLAN_ETH_HLEN)) goto dropped; + ethhdr = eth_hdr(skb); vhdr = skb_vlan_eth_hdr(skb); /* drop batman-in-batman packets to prevent loops */ @@ -589,8 +593,8 @@ int batadv_meshif_create_vlan(struct batadv_priv *bat_priv, unsigned short vid) * @bat_priv: the bat priv with all the mesh interface information * @vlan: the object to remove */ -static void batadv_meshif_destroy_vlan(struct batadv_priv *bat_priv, - struct batadv_meshif_vlan *vlan) +void batadv_meshif_destroy_vlan(struct batadv_priv *bat_priv, + struct batadv_meshif_vlan *vlan) { /* explicitly remove the associated TT local entry because it is marked * with the NOPURGE flag @@ -1085,22 +1089,13 @@ static int batadv_meshif_newlink(struct net_device *dev, static void batadv_meshif_destroy_netlink(struct net_device *mesh_iface, struct list_head *head) { - struct batadv_priv *bat_priv = netdev_priv(mesh_iface); struct batadv_hard_iface *hard_iface; - struct batadv_meshif_vlan *vlan; while (!list_empty(&mesh_iface->adj_list.lower)) { hard_iface = netdev_adjacent_get_private(mesh_iface->adj_list.lower.next); batadv_hardif_disable_interface(hard_iface); } - /* destroy the "untagged" VLAN */ - vlan = batadv_meshif_vlan_get(bat_priv, BATADV_NO_FLAGS); - if (vlan) { - batadv_meshif_destroy_vlan(bat_priv, vlan); - batadv_meshif_vlan_put(vlan); - } - unregister_netdevice_queue(mesh_iface, head); } diff --git a/net/batman-adv/mesh-interface.h b/net/batman-adv/mesh-interface.h index 53756c5a45e04..5e1e83e04ffbc 100644 --- a/net/batman-adv/mesh-interface.h +++ b/net/batman-adv/mesh-interface.h @@ -21,6 +21,8 @@ void batadv_interface_rx(struct net_device *mesh_iface, bool batadv_meshif_is_valid(const struct net_device *net_dev); extern struct rtnl_link_ops batadv_link_ops; int batadv_meshif_create_vlan(struct batadv_priv *bat_priv, unsigned short vid); +void batadv_meshif_destroy_vlan(struct batadv_priv *bat_priv, + struct batadv_meshif_vlan *vlan); void batadv_meshif_vlan_release(struct kref *ref); struct batadv_meshif_vlan *batadv_meshif_vlan_get(struct batadv_priv *bat_priv, unsigned short vid); diff --git a/net/batman-adv/multicast_forw.c b/net/batman-adv/multicast_forw.c index b8668a80b94a1..1404a3b7adfb1 100644 --- a/net/batman-adv/multicast_forw.c +++ b/net/batman-adv/multicast_forw.c @@ -927,11 +927,11 @@ static int batadv_mcast_forw_packet(struct batadv_priv *bat_priv, { struct batadv_tvlv_mcast_tracker *mcast_tracker; struct batadv_neigh_node *neigh_node; - unsigned long offset, num_dests_off; struct sk_buff *nexthop_skb; unsigned char *skb_net_hdr; bool local_recv = false; unsigned int tvlv_len; + unsigned long offset; bool xmitted = false; u8 *dest, *next_dest; u16 num_dests; @@ -940,9 +940,8 @@ static int batadv_mcast_forw_packet(struct batadv_priv *bat_priv, /* (at least) TVLV part needs to be linearized */ SKB_LINEAR_ASSERT(skb); - /* check if num_dests is within skb length */ - num_dests_off = offsetof(struct batadv_tvlv_mcast_tracker, num_dests); - if (num_dests_off > skb_network_header_len(skb)) + /* check if batadv_tvlv_mcast_tracker header is within skb length */ + if (sizeof(*mcast_tracker) > skb_network_header_len(skb)) return -EINVAL; skb_net_hdr = skb_network_header(skb); diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c index 78c651f634cd3..1d144d8cc0928 100644 --- a/net/batman-adv/netlink.c +++ b/net/batman-adv/netlink.c @@ -917,9 +917,15 @@ static int batadv_netlink_set_hardif(struct sk_buff *skb, #ifdef CONFIG_BATMAN_ADV_BATMAN_V if (info->attrs[BATADV_ATTR_ELP_INTERVAL]) { + u32 elp_interval; + attr = info->attrs[BATADV_ATTR_ELP_INTERVAL]; + elp_interval = nla_get_u32(attr); + + elp_interval = min_t(u32, elp_interval, INT_MAX); + elp_interval = max_t(u32, elp_interval, BATADV_JITTER); - atomic_set(&hard_iface->bat_v.elp_interval, nla_get_u32(attr)); + atomic_set(&hard_iface->bat_v.elp_interval, elp_interval); } if (info->attrs[BATADV_ATTR_THROUGHPUT_OVERRIDE]) { diff --git a/net/batman-adv/routing.c b/net/batman-adv/routing.c index 12c16f81cc51d..9de5bc552bfdc 100644 --- a/net/batman-adv/routing.c +++ b/net/batman-adv/routing.c @@ -8,6 +8,7 @@ #include "main.h" #include +#include #include #include #include @@ -204,6 +205,59 @@ bool batadv_check_management_packet(struct sk_buff *skb, return true; } +/** + * batadv_skb_decrement_ttl() - decrement ttl in a batman-adv header, csum-safe + * @skb: the received packet with @skb->data pointing to the batman-adv header + * + * Supports the following packet types, all of which carry the TTL at offset 2: + * + * - batadv_ogm_packet + * - batadv_ogm2_packet + * - batadv_icmp_header + * - batadv_icmp_packet + * - batadv_icmp_tp_packet + * - batadv_icmp_packet_rr + * - batadv_unicast_packet + * - batadv_frag_packet + * - batadv_bcast_packet + * - batadv_mcast_packet + * - batadv_coded_packet + * - batadv_unicast_tvlv_packet + * + * Return: true if the packet may be forwarded (ttl decremented), + * false if it must be dropped (ttl would expire) + */ +static bool batadv_skb_decrement_ttl(struct sk_buff *skb) +{ + static const size_t ttl_offset = 2; + u8 *ttl_pos; + + BUILD_BUG_ON(offsetof(struct batadv_ogm_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_ogm2_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_header, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_tp_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_packet_rr, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_unicast_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_frag_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_bcast_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_mcast_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_coded_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_unicast_tvlv_packet, ttl) != ttl_offset); + + ttl_pos = skb->data + ttl_offset; + + /* would expire on this hop -> drop, leave header + csum untouched */ + if (*ttl_pos < 2) + return false; + + skb_postpull_rcsum(skb, ttl_pos, 1); + (*ttl_pos)--; + skb_postpush_rcsum(skb, ttl_pos, 1); + + return true; +} + /** * batadv_recv_my_icmp_packet() - receive an icmp packet locally * @bat_priv: the bat priv with all the mesh interface information @@ -801,8 +855,8 @@ static bool batadv_check_unicast_ttvn(struct batadv_priv *bat_priv, if (skb_cow(skb, sizeof(*unicast_packet)) < 0) return false; - unicast_packet = (struct batadv_unicast_packet *)skb->data; vid = batadv_get_vid(skb, hdr_len); + unicast_packet = (struct batadv_unicast_packet *)skb->data; ethhdr = (struct ethhdr *)(skb->data + hdr_len); /* do not reroute multicast frames in a unicast header */ @@ -975,6 +1029,7 @@ int batadv_recv_unicast_packet(struct sk_buff *skb, hdr_size); batadv_orig_node_put(orig_node_gw); if (is_gw) { + orig_addr_gw = eth_hdr(skb)->h_source; batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): Dropped unicast pkt received from another backbone gw %pM.\n", __func__, orig_addr_gw); @@ -1114,10 +1169,9 @@ int batadv_recv_frag_packet(struct sk_buff *skb, /* Route the fragment if it is not for us and too big to be merged. */ if (!batadv_is_my_mac(bat_priv, frag_packet->dest) && - batadv_frag_skb_fwd(skb, recv_if, orig_node_src)) { + batadv_frag_skb_fwd(skb, recv_if, orig_node_src, &ret)) { /* skb was consumed */ skb = NULL; - ret = NET_RX_SUCCESS; goto put_orig_node; } @@ -1191,7 +1245,13 @@ int batadv_recv_bcast_packet(struct sk_buff *skb, if (batadv_is_my_mac(bat_priv, bcast_packet->orig)) goto free_skb; - if (bcast_packet->ttl-- < 2) + /* create a copy of the skb, if needed, to modify it. */ + if (skb_cow(skb, ETH_HLEN) < 0) + goto free_skb; + + bcast_packet = (struct batadv_bcast_packet *)skb->data; + + if (!batadv_skb_decrement_ttl(skb)) goto free_skb; orig_node = batadv_orig_hash_find(bat_priv, bcast_packet->orig); @@ -1298,7 +1358,7 @@ int batadv_recv_mcast_packet(struct sk_buff *skb, goto free_skb; mcast_packet = (struct batadv_mcast_packet *)skb->data; - if (mcast_packet->ttl-- < 2) + if (!batadv_skb_decrement_ttl(skb)) goto free_skb; tvlv_buff = (unsigned char *)(skb->data + hdr_size); @@ -1307,6 +1367,12 @@ int batadv_recv_mcast_packet(struct sk_buff *skb, if (tvlv_buff_len > skb->len - hdr_size) goto free_skb; + /* the fields of an multicast payload are accessed assuming (at least) + * 2-byte alignment, so a following packet must start at an even offset. + */ + if (tvlv_buff_len & 1) + goto free_skb; + ret = batadv_tvlv_containers_process(bat_priv, BATADV_MCAST, NULL, skb, tvlv_buff, tvlv_buff_len); if (ret >= 0) { diff --git a/net/batman-adv/tp_meter.c b/net/batman-adv/tp_meter.c index 0fc4ca78e84eb..dbaae33db0f1f 100644 --- a/net/batman-adv/tp_meter.c +++ b/net/batman-adv/tp_meter.c @@ -87,6 +87,11 @@ #define BATADV_TP_PLEN (BATADV_TP_PACKET_LEN - ETH_HLEN - \ sizeof(struct batadv_unicast_packet)) +/** + * BATADV_TP_MAX_UNACKED - maximum number of packets a receiver didn't yet ack + */ +#define BATADV_TP_MAX_UNACKED 100 + static u8 batadv_tp_prerandom[4096] __read_mostly; /** @@ -154,9 +159,12 @@ static void batadv_tp_update_cwnd(struct batadv_tp_vars *tp_vars, u32 mss) return; } + /* prevent overflow in (mss * mss) << 3 */ + mss = min_t(u32, mss, (1U << 14) - 1); + /* increment CWND at least of 1 (section 3.1 of RFC5681) */ tp_vars->dec_cwnd += max_t(u32, 1U << 3, - ((mss * mss) << 6) / (tp_vars->cwnd << 3)); + ((mss * mss) << 3) / tp_vars->cwnd); if (tp_vars->dec_cwnd < (mss << 3)) { spin_unlock_bh(&tp_vars->cwnd_lock); return; @@ -730,7 +738,7 @@ static void batadv_tp_recv_ack(struct batadv_priv *bat_priv, if (atomic_read(&tp_vars->dup_acks) != 3) goto out; - if (recv_ack >= tp_vars->recover) + if (!batadv_seq_before(tp_vars->recover, recv_ack)) goto out; /* if this is the third duplicate ACK do Fast Retransmit */ @@ -817,10 +825,15 @@ static void batadv_tp_recv_ack(struct batadv_priv *bat_priv, static bool batadv_tp_avail(struct batadv_tp_vars *tp_vars, size_t payload_len) { + u32 last_sent = READ_ONCE(tp_vars->last_sent); u32 win_left, win_limit; win_limit = atomic_read(&tp_vars->last_acked) + tp_vars->cwnd; - win_left = win_limit - tp_vars->last_sent; + + if (batadv_seq_before(last_sent, win_limit)) + win_left = win_limit - last_sent; + else + win_left = 0; return win_left >= payload_len; } @@ -1045,6 +1058,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, tp_vars->icmp_uid = icmp_uid; tp_vars->last_sent = BATADV_TP_FIRST_SEQ; + atomic_set(&tp_vars->dup_acks, 0); atomic_set(&tp_vars->last_acked, BATADV_TP_FIRST_SEQ); tp_vars->fast_recovery = false; tp_vars->recover = BATADV_TP_FIRST_SEQ; @@ -1054,6 +1068,8 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, * mesh_interface, hence its MTU */ tp_vars->cwnd = BATADV_TP_PLEN * 3; + tp_vars->dec_cwnd = 0; + /* at the beginning initialise the SS threshold to the biggest possible * window size, hence the AWND size */ @@ -1085,21 +1101,21 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, tp_vars->prerandom_offset = 0; spin_lock_init(&tp_vars->prerandom_lock); - kref_get(&tp_vars->refcount); - hlist_add_head_rcu(&tp_vars->list, &bat_priv->tp_list); - spin_unlock_bh(&bat_priv->tp_list_lock); - tp_vars->test_length = test_length; if (!tp_vars->test_length) tp_vars->test_length = BATADV_TP_DEF_TEST_LENGTH; + /* init work item for finished tp tests */ + INIT_DELAYED_WORK(&tp_vars->finish_work, batadv_tp_sender_finish); + + kref_get(&tp_vars->refcount); + hlist_add_head_rcu(&tp_vars->list, &bat_priv->tp_list); + spin_unlock_bh(&bat_priv->tp_list_lock); + batadv_dbg(BATADV_DBG_TP_METER, bat_priv, "Meter: starting throughput meter towards %pM (length=%ums)\n", dst, test_length); - /* init work item for finished tp tests */ - INIT_DELAYED_WORK(&tp_vars->finish_work, batadv_tp_sender_finish); - /* start tp kthread. This way the write() call issued from userspace can * happily return and avoid to block */ @@ -1167,7 +1183,7 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t) bat_priv = tp_vars->bat_priv; /* if there is recent activity rearm the timer */ - if (!batadv_has_timed_out(tp_vars->last_recv_time, + if (!batadv_has_timed_out(READ_ONCE(tp_vars->last_recv_time), BATADV_TP_RECV_TIMEOUT)) { /* reset the receiver shutdown timer */ batadv_tp_reset_receiver_timer(tp_vars); @@ -1184,6 +1200,7 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t) list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) { list_del(&un->list); kfree(un); + tp_vars->unacked_count--; } spin_unlock_bh(&tp_vars->unacked_lock); @@ -1267,7 +1284,8 @@ static int batadv_tp_send_ack(struct batadv_priv *bat_priv, const u8 *dst, /** * batadv_tp_handle_out_of_order() - store an out of order packet * @tp_vars: the private data of the current TP meter session - * @skb: the buffer containing the received packet + * @seqno: sequence number of new received packet + * @payload_len: length of the received packet * * Store the out of order packet in the unacked list for late processing. This * packets are kept in this list so that they can be ACKed at once as soon as @@ -1276,28 +1294,24 @@ static int batadv_tp_send_ack(struct batadv_priv *bat_priv, const u8 *dst, * Return: true if the packed has been successfully processed, false otherwise */ static bool batadv_tp_handle_out_of_order(struct batadv_tp_vars *tp_vars, - const struct sk_buff *skb) + u32 seqno, u32 payload_len) + __must_hold(&tp_vars->unacked_lock) { - const struct batadv_icmp_tp_packet *icmp; struct batadv_tp_unacked *un, *new; - u32 payload_len; bool added = false; new = kmalloc_obj(*new, GFP_ATOMIC); if (unlikely(!new)) return false; - icmp = (struct batadv_icmp_tp_packet *)skb->data; - - new->seqno = ntohl(icmp->seqno); - payload_len = skb->len - sizeof(struct batadv_unicast_packet); + new->seqno = seqno; new->len = payload_len; - spin_lock_bh(&tp_vars->unacked_lock); /* if the list is empty immediately attach this new object */ if (list_empty(&tp_vars->unacked_list)) { list_add(&new->list, &tp_vars->unacked_list); - goto out; + tp_vars->unacked_count++; + return true; } /* otherwise loop over the list and either drop the packet because this @@ -1325,17 +1339,26 @@ static bool batadv_tp_handle_out_of_order(struct batadv_tp_vars *tp_vars, * one is attached _after_ it. In this way the list is kept in * ascending order */ - list_add_tail(&new->list, &un->list); + list_add(&new->list, &un->list); added = true; + tp_vars->unacked_count++; break; } /* received packet with smallest seqno out of order; add it to front */ - if (!added) + if (!added) { list_add(&new->list, &tp_vars->unacked_list); + tp_vars->unacked_count++; + } -out: - spin_unlock_bh(&tp_vars->unacked_lock); + /* remove the last (biggest) unacked seqno when list is too large */ + if (tp_vars->unacked_count > BATADV_TP_MAX_UNACKED) { + un = list_last_entry(&tp_vars->unacked_list, + struct batadv_tp_unacked, list); + list_del(&un->list); + kfree(un); + tp_vars->unacked_count--; + } return true; } @@ -1346,6 +1369,7 @@ static bool batadv_tp_handle_out_of_order(struct batadv_tp_vars *tp_vars, * @tp_vars: the private data of the current TP meter session */ static void batadv_tp_ack_unordered(struct batadv_tp_vars *tp_vars) + __must_hold(&tp_vars->unacked_lock) { struct batadv_tp_unacked *un, *safe; u32 to_ack; @@ -1353,7 +1377,6 @@ static void batadv_tp_ack_unordered(struct batadv_tp_vars *tp_vars) /* go through the unacked packet list and possibly ACK them as * well */ - spin_lock_bh(&tp_vars->unacked_lock); list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) { /* the list is ordered, therefore it is possible to stop as soon * there is a gap between the last acked seqno and the seqno of @@ -1369,8 +1392,8 @@ static void batadv_tp_ack_unordered(struct batadv_tp_vars *tp_vars) list_del(&un->list); kfree(un); + tp_vars->unacked_count--; } - spin_unlock_bh(&tp_vars->unacked_lock); } /** @@ -1392,8 +1415,10 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv, tp_vars = batadv_tp_list_find_session(bat_priv, icmp->orig, icmp->session, BATADV_TP_RECEIVER); - if (tp_vars) + if (tp_vars) { + WRITE_ONCE(tp_vars->last_recv_time, jiffies); goto out_unlock; + } if (!atomic_add_unless(&bat_priv->tp_num, 1, BATADV_TP_MAX_NUM)) { batadv_dbg(BATADV_DBG_TP_METER, bat_priv, @@ -1417,12 +1442,15 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv, spin_lock_init(&tp_vars->unacked_lock); INIT_LIST_HEAD(&tp_vars->unacked_list); + tp_vars->unacked_count = 0; kref_get(&tp_vars->refcount); - hlist_add_head_rcu(&tp_vars->list, &bat_priv->tp_list); + timer_setup(&tp_vars->timer, batadv_tp_receiver_shutdown, 0); + + WRITE_ONCE(tp_vars->last_recv_time, jiffies); kref_get(&tp_vars->refcount); - timer_setup(&tp_vars->timer, batadv_tp_receiver_shutdown, 0); + hlist_add_head_rcu(&tp_vars->list, &bat_priv->tp_list); batadv_tp_reset_receiver_timer(tp_vars); @@ -1444,7 +1472,8 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv, { const struct batadv_icmp_tp_packet *icmp; struct batadv_tp_vars *tp_vars; - size_t packet_size; + u32 payload_len; + u32 to_ack; u32 seqno; icmp = (struct batadv_icmp_tp_packet *)skb->data; @@ -1469,41 +1498,49 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv, icmp->orig); goto out; } + + WRITE_ONCE(tp_vars->last_recv_time, jiffies); } - tp_vars->last_recv_time = jiffies; + spin_lock_bh(&tp_vars->unacked_lock); /* if the packet is a duplicate, it may be the case that an ACK has been * lost. Resend the ACK */ - if (batadv_seq_before(seqno, tp_vars->last_recv)) + payload_len = skb->len - sizeof(struct batadv_unicast_packet); + to_ack = seqno + payload_len; + if (batadv_seq_before(to_ack, tp_vars->last_recv)) goto send_ack; /* if the packet is out of order enqueue it */ - if (ntohl(icmp->seqno) != tp_vars->last_recv) { + if (batadv_seq_before(tp_vars->last_recv, seqno)) { /* exit immediately (and do not send any ACK) if the packet has * not been enqueued correctly */ - if (!batadv_tp_handle_out_of_order(tp_vars, skb)) + if (!batadv_tp_handle_out_of_order(tp_vars, seqno, payload_len)) { + spin_unlock_bh(&tp_vars->unacked_lock); goto out; + } /* send a duplicate ACK */ goto send_ack; } /* if everything was fine count the ACKed bytes */ - packet_size = skb->len - sizeof(struct batadv_unicast_packet); - tp_vars->last_recv += packet_size; + tp_vars->last_recv = to_ack; /* check if this ordered message filled a gap.... */ batadv_tp_ack_unordered(tp_vars); send_ack: + to_ack = tp_vars->last_recv; + spin_unlock_bh(&tp_vars->unacked_lock); + /* send the ACK. If the received packet was out of order, the ACK that * is going to be sent is a duplicate (the sender will count them and * possibly enter Fast Retransmit as soon as it has reached 3) */ - batadv_tp_send_ack(bat_priv, icmp->orig, tp_vars->last_recv, + batadv_tp_send_ack(bat_priv, icmp->orig, to_ack, icmp->timestamp, icmp->session, icmp->uid); out: batadv_tp_vars_put(tp_vars); diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c index 9f6e67771ffa8..877475c688252 100644 --- a/net/batman-adv/translation-table.c +++ b/net/batman-adv/translation-table.c @@ -446,6 +446,9 @@ static void batadv_tt_local_event(struct batadv_priv *bat_priv, if (!batadv_compare_eth(entry->change.addr, common->addr)) continue; + if (entry->change.vid != tt_change_node->change.vid) + continue; + del_op_entry = entry->change.flags & BATADV_TT_CLIENT_DEL; if (del_op_requested != del_op_entry) { /* DEL+ADD in the same orig interval have no effect and @@ -2960,7 +2963,7 @@ static bool batadv_send_tt_request(struct batadv_priv *bat_priv, out: batadv_hardif_put(primary_if); - if (ret && tt_req_node) { + if (!ret && tt_req_node) { spin_lock_bh(&bat_priv->tt.req_list_lock); if (!hlist_unhashed(&tt_req_node->list)) { hlist_del_init(&tt_req_node->list); @@ -3439,6 +3442,7 @@ static void batadv_tt_roam_purge(struct batadv_priv *bat_priv) * batadv_tt_check_roam_count() - check if a client has roamed too frequently * @bat_priv: the bat priv with all the mesh interface information * @client: mac address of the roaming client + * @vid: VLAN identifier * * This function checks whether the client already reached the * maximum number of possible roaming phases. In this case the ROAMING_ADV @@ -3446,7 +3450,7 @@ static void batadv_tt_roam_purge(struct batadv_priv *bat_priv) * * Return: true if the ROAMING_ADV can be sent, false otherwise */ -static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client) +static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client, u16 vid) { struct batadv_tt_roam_node *tt_roam_node; bool ret = false; @@ -3459,6 +3463,9 @@ static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client) if (!batadv_compare_eth(tt_roam_node->addr, client)) continue; + if (tt_roam_node->vid != vid) + continue; + if (batadv_has_timed_out(tt_roam_node->first_time, BATADV_ROAMING_MAX_TIME)) continue; @@ -3480,6 +3487,7 @@ static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client) atomic_set(&tt_roam_node->counter, BATADV_ROAMING_MAX_COUNT - 1); ether_addr_copy(tt_roam_node->addr, client); + tt_roam_node->vid = vid; list_add(&tt_roam_node->list, &bat_priv->tt.roam_list); ret = true; @@ -3516,7 +3524,7 @@ static void batadv_send_roam_adv(struct batadv_priv *bat_priv, u8 *client, /* before going on we have to check whether the client has * already roamed to us too many times */ - if (!batadv_tt_check_roam_count(bat_priv, client)) + if (!batadv_tt_check_roam_count(bat_priv, client, vid)) goto out; batadv_dbg(BATADV_DBG_TT, bat_priv, @@ -4017,7 +4025,8 @@ static int batadv_tt_tvlv_unicast_handler_v1(struct batadv_priv *bat_priv, u16 tvlv_value_len) { struct batadv_tvlv_tt_data *tt_data; - u16 tt_vlan_len, tt_num_entries; + u16 tt_num_entries; + size_t tt_vlan_len; char tt_flag; bool ret; diff --git a/net/batman-adv/tvlv.c b/net/batman-adv/tvlv.c index cc6ac580c6208..afb61a46d7a67 100644 --- a/net/batman-adv/tvlv.c +++ b/net/batman-adv/tvlv.c @@ -398,7 +398,6 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv, tvlv_handler->ogm_handler(bat_priv, orig_node, BATADV_NO_FLAGS, tvlv_value, tvlv_value_len); - tvlv_handler->flags |= BATADV_TVLV_HANDLER_OGM_CALLED; break; case BATADV_UNICAST_TVLV: if (!skb) @@ -430,6 +429,48 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv, return NET_RX_SUCCESS; } +/** + * batadv_tvlv_containers_contain() - check if a tvlv buffer holds a container + * @tvlv_value: tvlv content + * @tvlv_value_len: tvlv content length + * @type: tvlv container type to look for + * @version: tvlv container version to look for + * + * Return: true if a container of the given type and version is present in the + * tvlv buffer, false otherwise. + */ +static bool batadv_tvlv_containers_contain(void *tvlv_value, + u16 tvlv_value_len, u8 type, + u8 version) +{ + struct batadv_tvlv_hdr *tvlv_hdr; + u16 tvlv_value_cont_len; + + while (tvlv_value_len >= sizeof(*tvlv_hdr)) { + tvlv_hdr = tvlv_value; + tvlv_value_cont_len = ntohs(tvlv_hdr->len); + tvlv_value = tvlv_hdr + 1; + tvlv_value_len -= sizeof(*tvlv_hdr); + + if (tvlv_value_cont_len > tvlv_value_len) + break; + + /* the next tvlv header is accessed assuming (at least) 2-byte + * alignment, so it must start at an even offset. + */ + if (tvlv_value_cont_len & 1) + break; + + if (tvlv_hdr->type == type && tvlv_hdr->version == version) + return true; + + tvlv_value = (u8 *)tvlv_value + tvlv_value_cont_len; + tvlv_value_len -= tvlv_value_cont_len; + } + + return false; +} + /** * batadv_tvlv_containers_process() - parse the given tvlv buffer to call the * appropriate handlers @@ -449,7 +490,9 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv, struct sk_buff *skb, void *tvlv_value, u16 tvlv_value_len) { + u16 tvlv_value_start_len = tvlv_value_len; struct batadv_tvlv_handler *tvlv_handler; + void *tvlv_value_start = tvlv_value; struct batadv_tvlv_hdr *tvlv_hdr; u16 tvlv_value_cont_len; u8 cifnotfound = BATADV_TVLV_HANDLER_OGM_CIFNOTFND; @@ -464,6 +507,12 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv, if (tvlv_value_cont_len > tvlv_value_len) break; + /* the next tvlv header is accessed assuming (at least) 2-byte + * alignment, so it must start at an even offset. + */ + if (tvlv_value_cont_len & 1) + break; + tvlv_handler = batadv_tvlv_handler_get(bat_priv, tvlv_hdr->type, tvlv_hdr->version); @@ -487,12 +536,20 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv, if (!tvlv_handler->ogm_handler) continue; - if ((tvlv_handler->flags & BATADV_TVLV_HANDLER_OGM_CIFNOTFND) && - !(tvlv_handler->flags & BATADV_TVLV_HANDLER_OGM_CALLED)) - tvlv_handler->ogm_handler(bat_priv, orig_node, - cifnotfound, NULL, 0); + if (!(tvlv_handler->flags & BATADV_TVLV_HANDLER_OGM_CIFNOTFND)) + continue; + + /* if the corresponding container was present then the handler + * was already called from the loop above + */ + if (batadv_tvlv_containers_contain(tvlv_value_start, + tvlv_value_start_len, + tvlv_handler->type, + tvlv_handler->version)) + continue; - tvlv_handler->flags &= ~BATADV_TVLV_HANDLER_OGM_CALLED; + tvlv_handler->ogm_handler(bat_priv, orig_node, + cifnotfound, NULL, 0); } rcu_read_unlock(); diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index a01ee46d97f34..30c870cacf437 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -130,6 +130,12 @@ struct batadv_hard_iface_bat_v { /** @aggr_list: queue for to be aggregated OGM packets */ struct sk_buff_head aggr_list; + /** + * @aggr_list_enabled: aggr_list is active and new skbs can be + * enqueued. Protected by aggr_list.lock after initialization + */ + bool aggr_list_enabled:1; + /** @aggr_len: size of the OGM aggregate (excluding ethernet header) */ unsigned int aggr_len; @@ -1417,9 +1423,12 @@ struct batadv_tp_vars { /** @unacked_list: list of unacked packets (meta-info only) */ struct list_head unacked_list; - /** @unacked_lock: protect unacked_list */ + /** @unacked_lock: protect unacked_list + &batadv_tp_receiver.last_recv */ spinlock_t unacked_lock; + /** @unacked_count: number of unacked entries */ + size_t unacked_count; + /** @last_recv_time: time (jiffies) a msg was received */ unsigned long last_recv_time; @@ -1903,6 +1912,9 @@ struct batadv_tt_roam_node { /** @addr: mac address of the client in the roaming phase */ u8 addr[ETH_ALEN]; + /** @vid: VLAN identifier */ + u16 vid; + /** * @counter: number of allowed roaming events per client within a single * OGM interval (changes are committed with each OGM) @@ -2233,13 +2245,6 @@ enum batadv_tvlv_handler_flags { * will call this handler even if its type was not found (with no data) */ BATADV_TVLV_HANDLER_OGM_CIFNOTFND = BIT(1), - - /** - * @BATADV_TVLV_HANDLER_OGM_CALLED: interval tvlv handling flag - the - * API marks a handler as being called, so it won't be called if the - * BATADV_TVLV_HANDLER_OGM_CIFNOTFND flag was set - */ - BATADV_TVLV_HANDLER_OGM_CALLED = BIT(2), }; #endif /* _NET_BATMAN_ADV_TYPES_H_ */ diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index 960a19b3e26da..d504a363a30f3 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -632,7 +632,7 @@ static struct l2cap_chan *chan_create(void) if (!chan) return NULL; - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); chan->chan_type = L2CAP_CHAN_CONN_ORIENTED; chan->mode = L2CAP_MODE_LE_FLOWCTL; @@ -745,19 +745,24 @@ static inline void chan_ready_cb(struct l2cap_chan *chan) ifup(dev->netdev); } -static inline struct l2cap_chan *chan_new_conn_cb(struct l2cap_chan *pchan) +static void unregister_dev(struct lowpan_btle_dev *dev) { - struct l2cap_chan *chan; - - chan = chan_create(); - if (!chan) - return NULL; - - chan->ops = pchan->ops; - - BT_DBG("chan %p pchan %p", chan, pchan); - - return chan; + struct hci_dev *hdev = READ_ONCE(dev->hdev); + + /* If netdev holds last reference to hci_dev (its parent device), this + * leads to theoretical cyclic locking on lowpan_unregister_netdev: + * + * rtnl_lock -> put_device(parent) -> hci_release_dev -> + * destroy_workqueue -> hci_rx_work -> l2cap_recv_acldata -> + * chan_ready_cb -> ifup -> rtnl_lock + * + * However, hci_rx_work is disabled in hci_unregister_dev, so this + * should not occur. Make lockdep happy by postponing hdev release after + * netdev put. + */ + hci_dev_hold(hdev); + lowpan_unregister_netdev(dev->netdev); + hci_dev_put(hdev); } static void delete_netdev(struct work_struct *work) @@ -766,7 +771,7 @@ static void delete_netdev(struct work_struct *work) struct lowpan_btle_dev, delete_netdev); - lowpan_unregister_netdev(entry->netdev); + unregister_dev(entry); /* The entry pointer is deleted by the netdev destructor. */ } @@ -777,20 +782,10 @@ static void chan_close_cb(struct l2cap_chan *chan) struct lowpan_btle_dev *dev = NULL; struct lowpan_peer *peer; int err = -ENOENT; - bool last = false, remove = true; + bool last = false; BT_DBG("chan %p conn %p", chan, chan->conn); - if (chan->conn && chan->conn->hcon) { - if (!is_bt_6lowpan(chan->conn->hcon)) - return; - - /* If conn is set, then the netdev is also there and we should - * not remove it. - */ - remove = false; - } - spin_lock(&devices_lock); list_for_each_entry_rcu(entry, &bt_6lowpan_devices, list) { @@ -817,10 +812,8 @@ static void chan_close_cb(struct l2cap_chan *chan) ifdown(dev->netdev); - if (remove) { - INIT_WORK(&entry->delete_netdev, delete_netdev); - schedule_work(&entry->delete_netdev); - } + INIT_WORK(&entry->delete_netdev, delete_netdev); + schedule_work(&entry->delete_netdev); } else { spin_unlock(&devices_lock); } @@ -881,7 +874,6 @@ static long chan_get_sndtimeo_cb(struct l2cap_chan *chan) static const struct l2cap_ops bt_6lowpan_chan_ops = { .name = "L2CAP 6LoWPAN channel", - .new_connection = chan_new_conn_cb, .recv = chan_recv_cb, .close = chan_close_cb, .state_change = chan_state_change_cb, @@ -1009,16 +1001,19 @@ static int get_l2cap_conn(char *buf, bdaddr_t *addr, u8 *addr_type, hci_dev_lock(hdev); hcon = hci_conn_hash_lookup_le(hdev, addr, le_addr_type); - hci_dev_unlock(hdev); - hci_dev_put(hdev); - - if (!hcon) + if (!hcon) { + hci_dev_unlock(hdev); + hci_dev_put(hdev); return -ENOENT; + } - *conn = (struct l2cap_conn *)hcon->l2cap_data; + *conn = l2cap_conn_hold_unless_zero(hcon->l2cap_data); BT_DBG("conn %p dst %pMR type %u", *conn, &hcon->dst, hcon->dst_type); + hci_dev_unlock(hdev); + hci_dev_put(hdev); + return 0; } @@ -1073,23 +1068,15 @@ static void disconnect_all_peers(void) } while (nchans); } -struct set_enable { - struct work_struct work; - bool flag; -}; - -static void do_enable_set(struct work_struct *work) +static void do_enable_set(bool flag) { - struct set_enable *set_enable = container_of(work, - struct set_enable, work); - - if (!set_enable->flag || enable_6lowpan != set_enable->flag) + if (!flag || enable_6lowpan != flag) /* Disconnect existing connections if 6lowpan is * disabled */ disconnect_all_peers(); - enable_6lowpan = set_enable->flag; + enable_6lowpan = flag; mutex_lock(&set_lock); if (listen_chan) { @@ -1101,22 +1088,11 @@ static void do_enable_set(struct work_struct *work) listen_chan = bt_6lowpan_listen(); mutex_unlock(&set_lock); - - kfree(set_enable); } static int lowpan_enable_set(void *data, u64 val) { - struct set_enable *set_enable; - - set_enable = kzalloc_obj(*set_enable); - if (!set_enable) - return -ENOMEM; - - set_enable->flag = !!val; - INIT_WORK(&set_enable->work, do_enable_set); - - schedule_work(&set_enable->work); + do_enable_set(!!val); return 0; } @@ -1165,18 +1141,22 @@ static ssize_t lowpan_control_write(struct file *fp, if (conn) { struct lowpan_peer *peer; - if (!is_bt_6lowpan(conn->hcon)) + if (!is_bt_6lowpan(conn->hcon)) { + l2cap_conn_put(conn); return -EINVAL; + } peer = lookup_peer(conn); if (peer) { BT_DBG("6LoWPAN connection already exists"); + l2cap_conn_put(conn); return -EALREADY; } BT_DBG("conn %p dst %pMR type %d user %u", conn, &conn->hcon->dst, conn->hcon->dst_type, addr_type); + l2cap_conn_put(conn); } ret = bt_6lowpan_connect(&addr, addr_type); @@ -1192,6 +1172,8 @@ static ssize_t lowpan_control_write(struct file *fp, return ret; ret = bt_6lowpan_disconnect(conn, addr_type); + if (conn) + l2cap_conn_put(conn); if (ret < 0) return ret; @@ -1252,6 +1234,7 @@ static void disconnect_devices(void) break; new_dev->netdev = entry->netdev; + new_dev->hdev = entry->hdev; INIT_LIST_HEAD(&new_dev->list); list_add_rcu(&new_dev->list, &devices); @@ -1263,7 +1246,7 @@ static void disconnect_devices(void) ifdown(entry->netdev); BT_DBG("Unregistering netdev %s %p", entry->netdev->name, entry->netdev); - lowpan_unregister_netdev(entry->netdev); + unregister_dev(entry); kfree(entry); } } diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index 1a6aa3f8d4d61..ee8d92ea1e5ac 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -308,7 +308,7 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) restart: for (sk = bt_accept_get(parent, NULL); sk; sk = next) { - /* Prevent early freeing of sk due to unlink and sock_kill */ + /* The reference from bt_accept_get() keeps sk alive. */ lock_sock(sk); /* Check sk has not already been unlinked via @@ -324,13 +324,11 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) next = bt_accept_get(parent, sk); - /* sk is safely in the parent list so reduce reference count */ - sock_put(sk); - /* FIXME: Is this check still needed */ if (sk->sk_state == BT_CLOSED) { bt_accept_unlink(sk); release_sock(sk); + sock_put(sk); continue; } @@ -340,16 +338,6 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) if (newsock) sock_graft(sk, newsock); - /* Hand the caller a reference taken while sk is - * still locked. bt_accept_unlink() just dropped - * the accept-queue reference; without this hold a - * concurrent teardown (e.g. l2cap_conn_del() -> - * l2cap_sock_kill()) could free sk between - * release_sock() and the caller using it. Every - * caller drops this with sock_put() when done. - */ - sock_hold(sk); - release_sock(sk); if (next) sock_put(next); @@ -357,6 +345,7 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) } release_sock(sk); + sock_put(sk); } return NULL; diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c index 5c5f53ff30e8e..ed3ca38d8a6e1 100644 --- a/net/bluetooth/bnep/core.c +++ b/net/bluetooth/bnep/core.c @@ -562,14 +562,18 @@ static int bnep_session(void *arg) return 0; } -static struct device *bnep_get_device(struct bnep_session *session) +static struct l2cap_conn *bnep_get_conn(struct bnep_session *session) { - struct l2cap_conn *conn = l2cap_pi(session->sock->sk)->chan->conn; + struct l2cap_chan *chan = l2cap_pi(session->sock->sk)->chan; + struct l2cap_conn *conn; - if (!conn || !conn->hcon) - return NULL; + l2cap_chan_lock(chan); + conn = chan->conn; + if (conn) + l2cap_conn_get(conn); + l2cap_chan_unlock(chan); - return &conn->hcon->dev; + return conn; } static const struct device_type bnep_type = { @@ -581,6 +585,7 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) u32 valid_flags = BIT(BNEP_SETUP_RESPONSE); struct net_device *dev; struct bnep_session *s, *ss; + struct l2cap_conn *conn = NULL; u8 dst[ETH_ALEN], src[ETH_ALEN]; int err; @@ -640,10 +645,18 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) bnep_set_default_proto_filter(s); #endif - SET_NETDEV_DEV(dev, bnep_get_device(s)); + conn = bnep_get_conn(s); + if (!conn) { + err = -ENOTCONN; + goto failed; + } + + SET_NETDEV_DEV(dev, &conn->hcon->dev); SET_NETDEV_DEVTYPE(dev, &bnep_type); err = register_netdev(dev); + l2cap_conn_put(conn); + conn = NULL; if (err) goto failed; @@ -665,6 +678,8 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) return 0; failed: + if (conn) + l2cap_conn_put(conn); up_write(&bnep_session_sem); free_netdev(dev); return err; diff --git a/net/bluetooth/eir.c b/net/bluetooth/eir.c index 3f72111ba651f..1de5f9df6eec0 100644 --- a/net/bluetooth/eir.c +++ b/net/bluetooth/eir.c @@ -283,10 +283,12 @@ u8 eir_create_adv_data(struct hci_dev *hdev, u8 instance, u8 *ptr, u8 size) if (!flags) flags |= mgmt_get_adv_discov_flags(hdev); - /* If flags would still be empty, then there is no need to - * include the "Flags" AD field". + /* Only add the "Flags" if it fits together with the instance + * advertising data; drop it rather than overflow the buffer. */ - if (flags && (ad_len + eir_precalc_len(1) <= size)) { + if (flags && + (ad_len + eir_precalc_len(1) + + (adv ? adv->adv_data_len : 0) <= size)) { ptr[0] = 0x02; ptr[1] = EIR_FLAGS; ptr[2] = flags; diff --git a/net/bluetooth/hci_codec.c b/net/bluetooth/hci_codec.c index 3cc135bb1d30c..5bc5003c387c6 100644 --- a/net/bluetooth/hci_codec.c +++ b/net/bluetooth/hci_codec.c @@ -100,7 +100,7 @@ static void hci_read_codec_capabilities(struct hci_dev *hdev, __u8 transport, caps = (void *)skb->data; if (skb->len < sizeof(*caps)) goto error; - if (skb->len < caps->len) + if (skb->len < sizeof(caps->len) + caps->len) goto error; len += sizeof(caps->len) + caps->len; skb_pull(skb, sizeof(caps->len) + caps->len); diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 54eabaa46960b..eba4a548bef52 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -3181,26 +3181,11 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) conn->abort_reason = reason; - /* If the connection is pending check the command opcode since that - * might be blocking on hci_cmd_sync_work while waiting its respective - * event so we need to hci_cmd_sync_cancel to cancel it. - * - * hci_connect_le serializes the connection attempts so only one - * connection can be in BT_CONNECT at time. + /* Cancel the connect attempt. A return of 0 means the create command + * was still queued and got dequeued, so there is nothing to disconnect. */ - if (conn->state == BT_CONNECT && READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { - switch (hci_skb_event(hdev->sent_cmd)) { - case HCI_EV_CONN_COMPLETE: - case HCI_EV_LE_CONN_COMPLETE: - case HCI_EV_LE_ENHANCED_CONN_COMPLETE: - case HCI_EVT_LE_CIS_ESTABLISHED: - hci_cmd_sync_cancel(hdev, ECANCELED); - break; - } - /* Cancel connect attempt if still queued/pending */ - } else if (!hci_cancel_connect_sync(hdev, conn)) { + if (!hci_cancel_connect_sync(hdev, conn)) return 0; - } /* Run immediately if on cmd_sync_work since this may be called * as a result to MGMT_OP_DISCONNECT/MGMT_OP_UNPAIR which does diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 28d7929dc5937..1cbc666527c57 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -2671,6 +2671,8 @@ void hci_unregister_dev(struct hci_dev *hdev) disable_work_sync(&hdev->tx_work); disable_work_sync(&hdev->power_on); disable_work_sync(&hdev->error_reset); + disable_delayed_work_sync(&hdev->cmd_timer); + disable_delayed_work_sync(&hdev->ncmd_timer); hci_cmd_sync_clear(hdev); diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index eea2f810aafab..1cd5f97daafe3 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -1769,6 +1769,13 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable) hci_dev_clear_flag(hdev, HCI_LE_SCAN); + if (hdev->discovery.type == DISCOV_TYPE_INTERLEAVED && + hci_test_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY) && + !test_bit(HCI_INQUIRY, &hdev->flags) && + hdev->discovery.state == DISCOVERY_FINDING) { + hci_discovery_set_state(hdev, DISCOVERY_STOPPED); + } + /* The HCI_LE_SCAN_INTERRUPTED flag indicates that we * interrupted scanning due to a connect request. Mark * therefore discovery as stopped. diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index df23245d6ccda..a3df69bdec1e0 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -6611,6 +6611,11 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) bt_dev_dbg(hdev, "conn %p", conn); + /* Hold a reference so conn stays valid for the HCI_CONN_CREATE + * clear_bit() at done. + */ + hci_conn_get(conn); + clear_bit(HCI_CONN_SCANNING, &conn->flags); conn->state = BT_CONNECT; @@ -6623,6 +6628,7 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) hdev->le_scan_type == LE_SCAN_ACTIVE && !hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES)) { hci_conn_del(conn); + hci_conn_put(conn); return -EBUSY; } @@ -6668,6 +6674,12 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) &own_addr_type); if (err) goto done; + + /* Mark create connection in flight so hci_cancel_connect_sync() can + * cancel it while blocking on the connection complete event. + */ + set_bit(HCI_CONN_CREATE, &conn->flags); + /* Send command LE Extended Create Connection if supported */ if (use_ext_conn(hdev)) { err = hci_le_ext_create_conn_sync(hdev, conn, own_addr_type); @@ -6703,11 +6715,14 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) conn->conn_timeout, NULL); done: + clear_bit(HCI_CONN_CREATE, &conn->flags); + if (err == -ETIMEDOUT) hci_le_connect_cancel_sync(hdev, conn, 0x00); /* Re-enable advertising after the connection attempt is finished. */ hci_resume_advertising_sync(hdev); + hci_conn_put(conn); return err; } @@ -6982,10 +6997,25 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) else cp.role_switch = 0x00; - return __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN, - sizeof(cp), &cp, - HCI_EV_CONN_COMPLETE, - conn->conn_timeout, NULL); + /* Hold a reference so conn stays valid for the HCI_CONN_CREATE + * clear_bit() below. + */ + hci_conn_get(conn); + + /* Mark create connection in flight so hci_cancel_connect_sync() can + * cancel it while blocking on the connection complete event. + */ + set_bit(HCI_CONN_CREATE, &conn->flags); + + err = __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN, + sizeof(cp), &cp, + HCI_EV_CONN_COMPLETE, + conn->conn_timeout, NULL); + + clear_bit(HCI_CONN_CREATE, &conn->flags); + hci_conn_put(conn); + + return err; } int hci_connect_acl_sync(struct hci_dev *hdev, struct hci_conn *conn) @@ -7037,22 +7067,97 @@ int hci_connect_le_sync(struct hci_dev *hdev, struct hci_conn *conn) return (err == -EEXIST) ? 0 : err; } -int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn) +static int hci_acl_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) { - if (conn->state != BT_OPEN) - return -EINVAL; + struct hci_cmd_sync_work_entry *entry; + int err = -EBUSY; + + /* cmd_sync_work_lock makes the HCI_CONN_CREATE test and the cancel + * atomic against the worker, which takes this lock to dequeue every + * entry: while it is held no other command can become pending, so + * hci_cmd_sync_cancel() cannot cancel an unrelated command. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + + /* In flight: this connection owns the pending request, cancel it. */ + if (test_bit(HCI_CONN_CREATE, &conn->flags)) { + hci_cmd_sync_cancel(hdev, ECANCELED); + goto unlock; + } + + /* Still queued: a successful dequeue means it never started, so there + * is nothing to disconnect. + */ + entry = _hci_cmd_sync_lookup_entry(hdev, hci_acl_create_conn_sync, conn, + NULL); + if (entry) { + _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); + err = 0; + } + +unlock: + mutex_unlock(&hdev->cmd_sync_work_lock); + return err; +} + +static int hci_le_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) +{ + struct hci_cmd_sync_work_entry *entry; + int err = -EBUSY; + + /* cmd_sync_work_lock keeps the HCI_CONN_CREATE test and the cancel + * atomic against the cmd_sync worker. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + if (test_bit(HCI_CONN_CREATE, &conn->flags)) { + hci_cmd_sync_cancel(hdev, ECANCELED); + goto unlock; + } + + entry = _hci_cmd_sync_lookup_entry(hdev, hci_le_create_conn_sync, conn, + create_le_conn_complete); + if (entry) { + _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); + err = 0; + } + +unlock: + mutex_unlock(&hdev->cmd_sync_work_lock); + return err; +} + +static int hci_cis_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) +{ + /* LE Create CIS is shared by the whole CIG and cannot be dequeued + * per-connection, so only an in-flight command can be cancelled. + * cmd_sync_work_lock keeps the test and the cancel atomic against the + * cmd_sync worker. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + + if (test_bit(HCI_CONN_CREATE_CIS, &conn->flags)) + hci_cmd_sync_cancel(hdev, ECANCELED); + + mutex_unlock(&hdev->cmd_sync_work_lock); + return -EBUSY; +} + +int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn) +{ switch (conn->type) { case ACL_LINK: - return !hci_cmd_sync_dequeue_once(hdev, - hci_acl_create_conn_sync, - conn, NULL); + return hci_acl_cancel_create_conn_sync(hdev, conn); case LE_LINK: - return !hci_cmd_sync_dequeue_once(hdev, hci_le_create_conn_sync, - conn, create_le_conn_complete); + return hci_le_cancel_create_conn_sync(hdev, conn); + case CIS_LINK: + return hci_cis_cancel_create_conn_sync(hdev, conn); + default: + return -ENOENT; } - - return -ENOENT; } int hci_le_conn_update_sync(struct hci_dev *hdev, struct hci_conn *conn, diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index 3abd8111dda83..1d5ce87e6496e 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -1589,6 +1589,7 @@ static void iso_conn_big_sync(struct sock *sk) { int err; struct hci_dev *hdev; + struct iso_conn *conn; bdaddr_t src, dst; u8 src_type; @@ -1611,8 +1612,17 @@ static void iso_conn_big_sync(struct sock *sk) hci_dev_lock(hdev); lock_sock(sk); + /* The socket lock was dropped for hci_get_route(), so the connection + * may have been torn down meanwhile: iso_chan_del() clears conn and + * the broadcast teardown path can clear conn->hcon on its own. Check + * both before dereferencing conn->hcon. + */ + conn = iso_pi(sk)->conn; + if (!conn || !conn->hcon) + goto unlock; + if (!test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags)) { - err = hci_conn_big_create_sync(hdev, iso_pi(sk)->conn->hcon, + err = hci_conn_big_create_sync(hdev, conn->hcon, &iso_pi(sk)->qos, iso_pi(sk)->sync_handle, iso_pi(sk)->bc_num_bis, @@ -1621,6 +1631,7 @@ static void iso_conn_big_sync(struct sock *sk) bt_dev_err(hdev, "hci_big_create_sync: %d", err); } +unlock: release_sock(sk); hci_dev_unlock(hdev); hci_dev_put(hdev); @@ -2528,7 +2539,7 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) switch (pb) { case ISO_START: case ISO_SINGLE: - if (conn->rx_len) { + if (conn->rx_skb || conn->rx_len) { BT_ERR("Unexpected start frame (len %d)", skb->len); kfree_skb(conn->rx_skb); conn->rx_skb = NULL; @@ -2609,12 +2620,14 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) break; case ISO_CONT: - BT_DBG("Cont: frag len %d (expecting %d)", skb->len, + case ISO_END: + BT_DBG("%s: frag len %d (expecting %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len, conn->rx_len); - if (!conn->rx_len) { - BT_ERR("Unexpected continuation frame (len %d)", - skb->len); + if (!conn->rx_skb) { + BT_ERR("Unexpected ISO %s frame (len %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len); goto drop; } @@ -2630,17 +2643,9 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), skb->len); conn->rx_len -= skb->len; - break; - - case ISO_END: - if (!conn->rx_len) { - BT_ERR("Unexpected end frame (len %d)", skb->len); - goto drop; - } - skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), - skb->len); - conn->rx_len -= skb->len; + if (pb == ISO_CONT) + break; if (!conn->rx_len) { struct sk_buff *rx_skb = conn->rx_skb; @@ -2651,6 +2656,13 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) */ conn->rx_skb = NULL; iso_recv_frame(conn, rx_skb); + } else { + BT_ERR("ISO fragment incomplete (len %d, expected %d)", + skb->len, conn->rx_len); + kfree_skb(conn->rx_skb); + conn->rx_skb = NULL; + conn->rx_len = 0; + goto drop; } break; } diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index c4ccfbda9d789..189085000c73b 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -411,7 +411,7 @@ static void l2cap_chan_timeout(struct work_struct *work) BT_DBG("chan %p state %s", chan, state_to_string(chan->state)); - if (!conn) { + if (test_bit(FLAG_DEL, &chan->flags)) { l2cap_chan_put(chan); return; } @@ -422,6 +422,9 @@ static void l2cap_chan_timeout(struct work_struct *work) */ l2cap_chan_lock(chan); + if (test_bit(FLAG_DEL, &chan->flags)) + goto unlock; + if (chan->state == BT_CONNECTED || chan->state == BT_CONFIG) reason = ECONNREFUSED; else if (chan->state == BT_CONNECT && @@ -434,10 +437,10 @@ static void l2cap_chan_timeout(struct work_struct *work) chan->ops->close(chan); +unlock: l2cap_chan_unlock(chan); - l2cap_chan_put(chan); - mutex_unlock(&conn->lock); + l2cap_chan_put(chan); } struct l2cap_chan *l2cap_chan_create(void) @@ -490,6 +493,9 @@ static void l2cap_chan_destroy(struct kref *kref) list_del(&chan->global_l); write_unlock(&chan_list_lock); + if (chan->conn) + l2cap_conn_put(chan->conn); + kfree(chan); } @@ -519,7 +525,10 @@ void l2cap_chan_put(struct l2cap_chan *c) } EXPORT_SYMBOL_GPL(l2cap_chan_put); -void l2cap_chan_set_defaults(struct l2cap_chan *chan) +/* Initialise @chan with default values, inheriting from the parent channel + * @pchan when it is given. + */ +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan) { chan->fcs = L2CAP_FCS_CRC16; chan->max_tx = L2CAP_DEFAULT_MAX_TX; @@ -533,6 +542,31 @@ void l2cap_chan_set_defaults(struct l2cap_chan *chan) chan->retrans_timeout = L2CAP_DEFAULT_RETRANS_TO; chan->monitor_timeout = L2CAP_DEFAULT_MONITOR_TO; + if (pchan) { + BT_DBG("chan %p pchan %p", chan, pchan); + + chan->chan_type = pchan->chan_type; + chan->imtu = pchan->imtu; + chan->omtu = pchan->omtu; + chan->mode = pchan->mode; + chan->fcs = pchan->fcs; + chan->max_tx = pchan->max_tx; + chan->tx_win = pchan->tx_win; + chan->tx_win_max = pchan->tx_win_max; + chan->sec_level = pchan->sec_level; + chan->conf_state = pchan->conf_state; + chan->flags = pchan->flags; + chan->tx_credits = pchan->tx_credits; + chan->rx_credits = pchan->rx_credits; + + if (chan->chan_type == L2CAP_CHAN_FIXED) { + chan->scid = pchan->scid; + chan->dcid = pchan->scid; + } + + return; + } + chan->conf_state = 0; set_bit(CONF_NOT_COMPLETE, &chan->conf_state); @@ -593,7 +627,7 @@ void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan) conn->disc_reason = HCI_ERROR_REMOTE_USER_TERM; - chan->conn = conn; + chan->conn = l2cap_conn_get(conn); switch (chan->chan_type) { case L2CAP_CHAN_CONN_ORIENTED: @@ -648,30 +682,26 @@ void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan) void l2cap_chan_del(struct l2cap_chan *chan, int err) { - struct l2cap_conn *conn = chan->conn; - __clear_chan_timer(chan); - BT_DBG("chan %p, conn %p, err %d, state %s", chan, conn, err, + BT_DBG("chan %p, err %d, state %s", chan, err, state_to_string(chan->state)); chan->ops->teardown(chan, err); - if (conn) { + if (!test_and_set_bit(FLAG_DEL, &chan->flags)) { /* Delete from channel list */ list_del(&chan->list); l2cap_chan_put(chan); - chan->conn = NULL; - /* Reference was only held for non-fixed channels or * fixed channels that explicitly requested it using the * FLAG_HOLD_HCI_CONN flag. */ if (chan->chan_type != L2CAP_CHAN_FIXED || test_bit(FLAG_HOLD_HCI_CONN, &chan->flags)) - hci_conn_drop(conn->hcon); + hci_conn_drop(chan->conn->hcon); } if (test_bit(CONF_NOT_COMPLETE, &chan->conf_state)) @@ -1776,19 +1806,13 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) disable_delayed_work_sync(&conn->info_timer); disable_delayed_work_sync(&conn->id_addr_timer); + cancel_work_sync(&conn->pending_rx_work); + mutex_lock(&conn->lock); kfree_skb(conn->rx_skb); skb_queue_purge(&conn->pending_rx); - - /* We can not call flush_work(&conn->pending_rx_work) here since we - * might block if we are running on a worker from the same workqueue - * pending_rx_work is waiting on. - */ - if (work_pending(&conn->pending_rx_work)) - cancel_work_sync(&conn->pending_rx_work); - ida_destroy(&conn->tx_ida); l2cap_unregister_all_users(conn); @@ -1903,7 +1927,7 @@ static void l2cap_monitor_timeout(struct work_struct *work) l2cap_chan_lock(chan); - if (!chan->conn) { + if (test_bit(FLAG_DEL, &chan->flags)) { l2cap_chan_unlock(chan); l2cap_chan_put(chan); return; @@ -1924,7 +1948,7 @@ static void l2cap_retrans_timeout(struct work_struct *work) l2cap_chan_lock(chan); - if (!chan->conn) { + if (test_bit(FLAG_DEL, &chan->flags)) { l2cap_chan_unlock(chan); l2cap_chan_put(chan); return; @@ -2565,7 +2589,7 @@ int l2cap_chan_send(struct l2cap_chan *chan, struct msghdr *msg, size_t len, int err; struct sk_buff_head seg_queue; - if (!chan->conn) + if (test_bit(FLAG_DEL, &chan->flags)) return -ENOTCONN; /* Connectionless channel */ @@ -3052,13 +3076,24 @@ static struct sk_buff *l2cap_build_cmd(struct l2cap_conn *conn, u8 code, return NULL; } -static inline int l2cap_get_conf_opt(void **ptr, int *type, int *olen, - unsigned long *val) +static inline int l2cap_get_conf_opt(void **ptr, void *end, int *type, + int *olen, unsigned long *val) { struct l2cap_conf_opt *opt = *ptr; int len; + /* opt->len is attacker-controlled. Validate that the full option + * (header + value) actually fits in the buffer before touching + * opt->val, otherwise the switch below reads past the end of the + * caller's buffer. + */ + if (end - *ptr < L2CAP_CONF_OPT_SIZE) + return -EINVAL; + len = L2CAP_CONF_OPT_SIZE + opt->len; + if (end - *ptr < len) + return -EINVAL; + *ptr += len; *type = opt->type; @@ -3160,12 +3195,16 @@ static void l2cap_ack_timeout(struct work_struct *work) l2cap_chan_lock(chan); + if (test_bit(FLAG_DEL, &chan->flags)) + goto unlock; + frames_to_ack = __seq_offset(chan, chan->buffer_seq, chan->last_acked_seq); if (frames_to_ack) l2cap_send_rr_or_rnr(chan, 0); +unlock: l2cap_chan_unlock(chan); l2cap_chan_put(chan); } @@ -3426,6 +3465,7 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data void *ptr = rsp->data; void *endptr = data + data_size; void *req = chan->conf_req; + void *req_end = req + chan->conf_len; int len = chan->conf_len; int type, hint, olen; unsigned long val; @@ -3439,9 +3479,11 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data BT_DBG("chan %p", chan); while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&req, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&req, req_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; hint = type & L2CAP_CONF_HINT; type &= L2CAP_CONF_MASK; @@ -3669,6 +3711,7 @@ static int l2cap_parse_conf_rsp(struct l2cap_chan *chan, void *rsp, int len, struct l2cap_conf_req *req = data; void *ptr = req->data; void *endptr = data + size; + void *rsp_end = rsp + len; int type, olen; unsigned long val; struct l2cap_conf_rfc rfc = { .mode = L2CAP_MODE_BASIC }; @@ -3677,9 +3720,11 @@ static int l2cap_parse_conf_rsp(struct l2cap_chan *chan, void *rsp, int len, BT_DBG("chan %p, rsp %p, len %d, req %p", chan, rsp, len, data); while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&rsp, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&rsp, rsp_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; switch (type) { case L2CAP_CONF_MTU: @@ -3930,6 +3975,7 @@ static void l2cap_conf_rfc_get(struct l2cap_chan *chan, void *rsp, int len) { int type, olen; unsigned long val; + void *rsp_end = rsp + len; /* Use sane default values in case a misbehaving remote device * did not send an RFC or extended window size option. */ @@ -3948,9 +3994,11 @@ static void l2cap_conf_rfc_get(struct l2cap_chan *chan, void *rsp, int len) return; while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&rsp, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&rsp, rsp_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; switch (type) { case L2CAP_CONF_RFC: @@ -4007,6 +4055,38 @@ static inline int l2cap_command_rej(struct l2cap_conn *conn, return 0; } +/* Allocate and initialise a channel for an incoming connection. + * + * The channel inherits its configuration from @pchan and is linked into @conn + * before ->new_connection() runs, so the conn list reference keeps it alive if + * the callback exposes it (e.g. via the socket accept queue) before this + * returns. The l2cap_chan_create() reference is taken over by the subsystem on + * success and dropped here on failure. + */ +static struct l2cap_chan *l2cap_new_connection(struct l2cap_conn *conn, + struct l2cap_chan *pchan) +{ + struct l2cap_chan *chan; + + chan = l2cap_chan_create(); + if (!chan) + return NULL; + + l2cap_chan_set_defaults(chan, pchan); + chan->ops = pchan->ops; + + __l2cap_chan_add(conn, chan); + + if (pchan->ops->new_connection && + pchan->ops->new_connection(pchan, chan) < 0) { + l2cap_chan_del(chan, 0); + l2cap_chan_put(chan); + return NULL; + } + + return chan; +} + static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, u8 *data, u8 rsp_code) { @@ -4053,7 +4133,7 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, goto response; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) goto response; @@ -4071,8 +4151,6 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, chan->psm = psm; chan->dcid = scid; - __l2cap_chan_add(conn, chan); - dcid = chan->scid; __set_chan_timer(chan, chan->ops->get_sndtimeo(chan)); @@ -4804,6 +4882,7 @@ static void l2cap_put_ident(struct l2cap_conn *conn, u8 code, u8 id) case L2CAP_ECHO_RSP: case L2CAP_INFO_RSP: case L2CAP_CONN_PARAM_UPDATE_RSP: + case L2CAP_LE_CONN_RSP: case L2CAP_ECRED_CONN_RSP: case L2CAP_ECRED_RECONF_RSP: /* First do a lookup since the remote may send bogus ids that @@ -4955,7 +5034,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; goto response_unlock; @@ -4970,8 +5049,6 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_le_flowctl_init(chan, __le16_to_cpu(req->credits)); dcid = chan->scid; @@ -5179,7 +5256,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, continue; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; continue; @@ -5194,8 +5271,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_ecred_init(chan, __le16_to_cpu(req->credits)); /* Init response */ @@ -6701,6 +6776,7 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) struct l2cap_conn *conn = chan->conn; struct l2cap_le_credits pkt; u16 return_credits = l2cap_le_rx_credits(chan); + int ident; if (chan->mode != L2CAP_MODE_LE_FLOWCTL && chan->mode != L2CAP_MODE_EXT_FLOWCTL) @@ -6718,9 +6794,18 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) pkt.cid = cpu_to_le16(chan->scid); pkt.credits = cpu_to_le16(return_credits); - chan->ident = l2cap_get_ident(conn); + ident = l2cap_get_ident(conn); - l2cap_send_cmd(conn, chan->ident, L2CAP_LE_CREDITS, sizeof(pkt), &pkt); + l2cap_send_cmd(conn, ident, L2CAP_LE_CREDITS, sizeof(pkt), &pkt); + + /* L2CAP_LE_CREDITS has no response so the ident is never released by + * l2cap_put_ident() - release it right away, otherwise the tx_ida + * range is exhausted after 254 packets and from then on credits are + * sent with the invalid ident 0, which some remote stacks ignore, + * stalling the channel. + */ + if (ident > 0) + ida_free(&conn->tx_ida, ident); } void l2cap_chan_rx_avail(struct l2cap_chan *chan, ssize_t rx_avail) @@ -7470,14 +7555,12 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status) goto next; l2cap_chan_lock(pchan); - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (chan) { bacpy(&chan->src, &hcon->src); bacpy(&chan->dst, &hcon->dst); chan->src_type = bdaddr_src_type(hcon); chan->dst_type = dst_type; - - __l2cap_chan_add(conn, chan); } l2cap_chan_unlock(pchan); @@ -7694,6 +7777,7 @@ struct l2cap_conn *l2cap_conn_hold_unless_zero(struct l2cap_conn *c) return c; } +EXPORT_SYMBOL(l2cap_conn_hold_unless_zero); int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index c138aa4ae2669..d8cbe27844377 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -45,7 +45,8 @@ static struct bt_sock_list l2cap_sk_list = { static const struct proto_ops l2cap_sock_ops; static void l2cap_sock_init(struct sock *sk, struct sock *parent); static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern); + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan); static void l2cap_sock_cleanup_listen(struct sock *parent); bool l2cap_is_socket(struct socket *sock) @@ -1274,6 +1275,23 @@ static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg, return err; } +/* Release the sock's ref on chan and clear the pointer so that the ref is + * dropped exactly once even if both l2cap_sock_kill() and + * l2cap_sock_destruct() run. Setting chan->data to NULL first stops any other + * task from dereferencing the now-dead sock pointer. + */ +static void l2cap_sock_put_chan(struct sock *sk) +{ + struct l2cap_chan *chan = l2cap_pi(sk)->chan; + + if (!chan) + return; + + chan->data = NULL; + l2cap_pi(sk)->chan = NULL; + l2cap_chan_put(chan); +} + /* Kill socket (only if zapped and orphan) * Must be called on unlocked socket, with l2cap channel lock. */ @@ -1284,13 +1302,9 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); - /* Sock is dead, so set chan data to NULL, avoid other task use invalid - * sock pointer. - */ - l2cap_pi(sk)->chan->data = NULL; - /* Kill poor orphan */ + l2cap_sock_put_chan(sk); - l2cap_chan_put(l2cap_pi(sk)->chan); + /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); sock_put(sk); } @@ -1482,8 +1496,8 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) /* Close not yet accepted channels. * - * bt_accept_dequeue() now returns sk with an extra reference held - * (taken while sk was still locked) so a concurrent l2cap_conn_del() + * bt_accept_dequeue() returns sk with its temporary queue-walk + * reference held, so a concurrent l2cap_conn_del() * -> l2cap_sock_kill() cannot free sk under us. * * cleanup_listen() runs under the parent sk lock, so unlike @@ -1533,12 +1547,13 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) } } -static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) +static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { struct sock *sk, *parent = chan->data; if (!parent) - return NULL; + return -EINVAL; lock_sock(parent); @@ -1546,25 +1561,28 @@ static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); release_sock(parent); - return NULL; + return -ENOBUFS; } sk = l2cap_sock_alloc(sock_net(parent), NULL, BTPROTO_L2CAP, - GFP_ATOMIC, 0); + GFP_ATOMIC, 0, new_chan); if (!sk) { release_sock(parent); - return NULL; - } + return -ENOMEM; + } bt_sock_reclassify_lock(sk, BTPROTO_L2CAP); l2cap_sock_init(sk, parent); + /* The conn list reference taken by l2cap_new_connection() keeps new_chan + * alive once release_sock() lets another task free this socket. + */ bt_accept_enqueue(parent, sk, false); release_sock(parent); - return l2cap_pi(sk)->chan; + return 0; } static int l2cap_sock_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb) @@ -1861,10 +1879,7 @@ static void l2cap_sock_destruct(struct sock *sk) BT_DBG("sk %p", sk); - if (l2cap_pi(sk)->chan) { - l2cap_pi(sk)->chan->data = NULL; - l2cap_chan_put(l2cap_pi(sk)->chan); - } + l2cap_sock_put_chan(sk); list_for_each_entry_safe(rx_busy, next, &l2cap_pi(sk)->rx_busy, list) { kfree_skb(rx_busy->skb); @@ -1897,30 +1912,12 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) BT_DBG("sk %p", sk); if (parent) { - struct l2cap_chan *pchan = l2cap_pi(parent)->chan; - sk->sk_type = parent->sk_type; bt_sk(sk)->flags = bt_sk(parent)->flags; - chan->chan_type = pchan->chan_type; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->conf_state = pchan->conf_state; - chan->mode = pchan->mode; - chan->fcs = pchan->fcs; - chan->max_tx = pchan->max_tx; - chan->tx_win = pchan->tx_win; - chan->tx_win_max = pchan->tx_win_max; - chan->sec_level = pchan->sec_level; - chan->flags = pchan->flags; - chan->tx_credits = pchan->tx_credits; - chan->rx_credits = pchan->rx_credits; - - if (chan->chan_type == L2CAP_CHAN_FIXED) { - chan->scid = pchan->scid; - chan->dcid = pchan->scid; - } - + /* Channel configuration is inherited from the parent by + * l2cap_new_connection(). + */ security_sk_clone(parent, sk); } else { switch (sk->sk_type) { @@ -1946,7 +1943,7 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) chan->mode = L2CAP_MODE_BASIC; } - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); } /* Default config options */ @@ -1965,10 +1962,10 @@ static struct proto l2cap_proto = { }; static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern) + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan) { struct sock *sk; - struct l2cap_chan *chan; sk = bt_sock_alloc(net, sock, &l2cap_proto, proto, prio, kern); if (!sk) @@ -1979,16 +1976,7 @@ static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, INIT_LIST_HEAD(&l2cap_pi(sk)->rx_busy); - chan = l2cap_chan_create(); - if (!chan) { - sk_free(sk); - if (sock) - sock->sk = NULL; - return NULL; - } - - l2cap_chan_hold(chan); - + /* The sock takes ownership of the caller's reference on chan. */ l2cap_pi(sk)->chan = chan; return sk; @@ -1998,6 +1986,7 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, int kern) { struct sock *sk; + struct l2cap_chan *chan; BT_DBG("sock %p", sock); @@ -2012,10 +2001,16 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, sock->ops = &l2cap_sock_ops; - sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern); - if (!sk) + chan = l2cap_chan_create(); + if (!chan) return -ENOMEM; + sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern, chan); + if (!sk) { + l2cap_chan_put(chan); + return -ENOMEM; + } + l2cap_sock_init(sk, NULL); bt_sock_link(&l2cap_sk_list, sk); return 0; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index f4aa814a03975..e8544194a5242 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -5378,6 +5378,8 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, if (monitor->state == ADV_MONITOR_STATE_NOT_REGISTERED) monitor->state = ADV_MONITOR_STATE_REGISTERED; hci_update_passive_scan(hdev); + } else { + hci_free_adv_monitor(hdev, monitor); } mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, @@ -7661,6 +7663,8 @@ static void add_device_complete(struct hci_dev *hdev, void *data, int err) if (!err) { struct hci_conn_params *params; + hci_dev_lock(hdev); + params = hci_conn_params_lookup(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); @@ -7669,6 +7673,7 @@ static void add_device_complete(struct hci_dev *hdev, void *data, int err) device_flags_changed(NULL, hdev, &cp->addr.bdaddr, cp->addr.type, hdev->conn_flags, params ? params->flags : 0); + hci_dev_unlock(hdev); } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_ADD_DEVICE, diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index 2f008167cbaa2..d7badce8746ca 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -291,7 +291,7 @@ static int msft_le_monitor_advertisement_cb(struct hci_dev *hdev, u16 opcode, monitor->state = ADV_MONITOR_STATE_OFFLOADED; unlock: - if (status) + if (status && msft->resuming) hci_free_adv_monitor(hdev, monitor); hci_dev_unlock(hdev); diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 140869e5b2df2..b2c51f3ebc404 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -572,10 +572,23 @@ static void __sco_sock_close(struct sock *sk) /* Must be called on unlocked socket. */ static void sco_sock_close(struct sock *sk) { + struct sco_conn *conn; + + lock_sock(sk); + conn = sco_pi(sk)->conn; + if (conn) + sco_conn_hold(conn); + release_sock(sk); + + if (conn) + disable_delayed_work_sync(&conn->timeout_work); + lock_sock(sk); - sco_sock_clear_timer(sk); __sco_sock_close(sk); release_sock(sk); + + if (conn) + sco_conn_put(conn); } static void sco_sock_init(struct sock *sk, struct sock *parent) diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 98f1da4f5f559..4787f8cd29e41 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -3234,34 +3234,19 @@ static const struct l2cap_ops smp_chan_ops = { .get_sndtimeo = l2cap_chan_no_get_sndtimeo, }; -static inline struct l2cap_chan *smp_new_conn_cb(struct l2cap_chan *pchan) +static inline int smp_new_conn_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - struct l2cap_chan *chan; - - BT_DBG("pchan %p", pchan); - - chan = l2cap_chan_create(); - if (!chan) - return NULL; - - chan->chan_type = pchan->chan_type; - chan->ops = &smp_chan_ops; - chan->scid = pchan->scid; - chan->dcid = chan->scid; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->mode = pchan->mode; + new_chan->ops = &smp_chan_ops; /* Other L2CAP channels may request SMP routines in order to * change the security level. This means that the SMP channel * lock must be considered in its own category to avoid lockdep * warnings. */ - atomic_set(&chan->nesting, L2CAP_NESTING_SMP); - - BT_DBG("created chan %p", chan); + atomic_set(&new_chan->nesting, L2CAP_NESTING_SMP); - return chan; + return 0; } static const struct l2cap_ops smp_root_chan_ops = { @@ -3332,7 +3317,7 @@ static struct l2cap_chan *smp_add_cid(struct hci_dev *hdev, u16 cid) l2cap_add_scid(chan, cid); - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); if (cid == L2CAP_CID_SMP) { u8 bdaddr_type; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 2bc04feadfabe..f15c613aaa4ee 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -453,12 +453,8 @@ static int bpf_test_finish(const union bpf_attr *kattr, } if (data_out) { - int len = sinfo ? copy_size - frag_size : copy_size; - - if (len < 0) { - err = -ENOSPC; - goto out; - } + u32 head_len = size - frag_size; + u32 len = min(copy_size, head_len); if (copy_to_user(data_out, data, len)) goto out; diff --git a/net/bridge/br_cfm.c b/net/bridge/br_cfm.c index 118c7ea48c351..dea56fffa1c19 100644 --- a/net/bridge/br_cfm.c +++ b/net/bridge/br_cfm.c @@ -805,6 +805,12 @@ int br_cfm_cc_ccm_tx(struct net_bridge *br, const u32 instance, goto save; } + if (!interval_to_us(mep->cc_config.exp_interval)) { + NL_SET_ERR_MSG_MOD(extack, + "Invalid CCM interval"); + return -EINVAL; + } + /* Start delayed work to transmit CCM frames. It is done with zero delay * to send first frame immediately */ diff --git a/net/bridge/br_cfm_netlink.c b/net/bridge/br_cfm_netlink.c index 2faab44652e7c..91b9922dc3f25 100644 --- a/net/bridge/br_cfm_netlink.c +++ b/net/bridge/br_cfm_netlink.c @@ -34,7 +34,9 @@ br_cfm_cc_config_policy[IFLA_BRIDGE_CFM_CC_CONFIG_MAX + 1] = { [IFLA_BRIDGE_CFM_CC_CONFIG_UNSPEC] = { .type = NLA_REJECT }, [IFLA_BRIDGE_CFM_CC_CONFIG_INSTANCE] = { .type = NLA_U32 }, [IFLA_BRIDGE_CFM_CC_CONFIG_ENABLE] = { .type = NLA_U32 }, - [IFLA_BRIDGE_CFM_CC_CONFIG_EXP_INTERVAL] = { .type = NLA_U32 }, + [IFLA_BRIDGE_CFM_CC_CONFIG_EXP_INTERVAL] = + NLA_POLICY_RANGE(NLA_U32, BR_CFM_CCM_INTERVAL_3_3_MS, + BR_CFM_CCM_INTERVAL_10_MIN), [IFLA_BRIDGE_CFM_CC_CONFIG_EXP_MAID] = { .type = NLA_BINARY, .len = CFM_MAID_LENGTH }, }; diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index d39571e137444..ec773e4ab92ec 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -391,6 +391,9 @@ void br_dev_delete(struct net_device *dev, struct list_head *head) br_fdb_delete_by_port(br, NULL, 0, 1); + timer_shutdown_sync(&br->hello_timer); + timer_shutdown_sync(&br->topology_change_timer); + timer_shutdown_sync(&br->tcn_timer); cancel_delayed_work_sync(&br->gc_work); br_sysfs_delbr(br->dev); diff --git a/net/bridge/br_stp.c b/net/bridge/br_stp.c index 024210f954683..76a2df165ffc8 100644 --- a/net/bridge/br_stp.c +++ b/net/bridge/br_stp.c @@ -371,7 +371,8 @@ void br_topology_change_detection(struct net_bridge *br) { int isroot = br_is_root_bridge(br); - if (br->stp_enabled != BR_KERNEL_STP) + if (br->stp_enabled != BR_KERNEL_STP || + !(br->dev->flags & IFF_UP)) return; br_info(br, "topology change detected, %s\n", diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index 8a6a069329d21..1147ae4fee971 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -403,6 +403,9 @@ ebt_check_match(struct ebt_entry_match *m, struct xt_mtchk_param *par, left - sizeof(struct ebt_entry_match) < m->match_size) return -EINVAL; + if (strnlen(m->u.name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN) + return -EINVAL; + match = xt_find_match(NFPROTO_BRIDGE, m->u.name, m->u.revision); if (IS_ERR(match) || match->family != NFPROTO_BRIDGE) { if (!IS_ERR(match)) @@ -921,8 +924,7 @@ static int translate_table(struct net *net, const char *name, * if an error occurs */ newinfo->chainstack = - vmalloc_array(nr_cpu_ids, - sizeof(*(newinfo->chainstack))); + vcalloc(nr_cpu_ids, sizeof(*(newinfo->chainstack))); if (!newinfo->chainstack) return -ENOMEM; for_each_possible_cpu(i) { @@ -1434,6 +1436,8 @@ static int update_counters(struct net *net, sockptr_t arg, unsigned int len) if (copy_from_sockptr(&hlp, arg, sizeof(hlp))) return -EFAULT; + hlp.name[sizeof(hlp.name) - 1] = '\0'; + if (len != sizeof(hlp) + hlp.num_counters * sizeof(struct ebt_counter)) return -EINVAL; @@ -2273,6 +2277,8 @@ static int compat_copy_ebt_replace_from_user(struct ebt_replace *repl, memcpy(repl, &tmp, offsetof(struct ebt_replace, hook_entry)); + repl->name[sizeof(repl->name) - 1] = '\0'; + /* starting with hook_entry, 32 vs. 64 bit structures are different */ for (i = 0; i < NF_BR_NUMHOOKS; i++) repl->hook_entry[i] = compat_ptr(tmp.hook_entry[i]); @@ -2395,6 +2401,8 @@ static int compat_update_counters(struct net *net, sockptr_t arg, if (copy_from_sockptr(&hlp, arg, sizeof(hlp))) return -EFAULT; + hlp.name[sizeof(hlp.name) - 1] = '\0'; + /* try real handler in case userland supplied needed padding */ if (len != sizeof(hlp) + hlp.num_counters * sizeof(struct ebt_counter)) return update_counters(net, arg, len); diff --git a/net/bridge/netfilter/nft_meta_bridge.c b/net/bridge/netfilter/nft_meta_bridge.c index 219c406802602..e4c9aa1f64e25 100644 --- a/net/bridge/netfilter/nft_meta_bridge.c +++ b/net/bridge/netfilter/nft_meta_bridge.c @@ -44,7 +44,9 @@ static void nft_meta_bridge_get_eval(const struct nft_expr *expr, if (!br_dev || !br_vlan_enabled(br_dev)) goto err; - br_vlan_get_pvid_rcu(in, &p_pvid); + if (br_vlan_get_pvid_rcu(in, &p_pvid)) + goto err; + nft_reg_store16(dest, p_pvid); return; } @@ -107,12 +109,30 @@ static int nft_meta_bridge_get_init(const struct nft_ctx *ctx, NULL, NFT_DATA_VALUE, len); } +static int nft_meta_bridge_get_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct nft_meta *priv = nft_expr_priv(expr); + unsigned int hooks; + + switch (priv->key) { + case NFT_META_BRI_IIFHWADDR: + hooks = 1 << NF_BR_PRE_ROUTING; + break; + default: + return nft_meta_get_validate(ctx, expr); + } + + return nft_chain_validate_hooks(ctx->chain, hooks); +} + static struct nft_expr_type nft_meta_bridge_type; static const struct nft_expr_ops nft_meta_bridge_get_ops = { .type = &nft_meta_bridge_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_meta)), .eval = nft_meta_bridge_get_eval, .init = nft_meta_bridge_get_init, + .validate = nft_meta_bridge_get_validate, .dump = nft_meta_get_dump, }; @@ -168,7 +188,6 @@ static int nft_meta_bridge_set_validate(const struct nft_ctx *ctx, switch (priv->key) { case NFT_META_BRI_BROUTE: - case NFT_META_BRI_IIFHWADDR: hooks = 1 << NF_BR_PRE_ROUTING; break; default: diff --git a/net/can/bcm.c b/net/can/bcm.c index a4bef2c48a559..3d637a1e0ac1a 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,8 @@ MODULE_ALIAS("can-proto-2"); #define BCM_MIN_NAMELEN CAN_REQUIRED_SIZE(struct sockaddr_can, can_ifindex) +static struct workqueue_struct *bcm_wq; + /* * easy access to the first 64 bit of can(fd)_frame payload. cp->data is * 64 bit aligned so the offset has to be multiples of 8 which is ensured @@ -105,14 +108,16 @@ static inline u64 get_u64(const struct canfd_frame *cp, int offset) struct bcm_op { struct list_head list; struct rcu_head rcu; + struct work_struct work; int ifindex; canid_t can_id; u32 flags; - unsigned long frames_abs, frames_filtered; + atomic_long_t frames_abs, frames_filtered; struct bcm_timeval ival1, ival2; struct hrtimer timer, thrtimer; ktime_t rx_stamp, kt_ival1, kt_ival2, kt_lastmsg; int rx_ifindex; + int if_detected; /* first received ifindex in ANYDEV rx_op mode */ int cfsiz; u32 count; u32 nframes; @@ -124,7 +129,9 @@ struct bcm_op { struct canfd_frame last_sframe; struct sock *sk; struct net_device *rx_reg_dev; - spinlock_t bcm_tx_lock; /* protect currframe/count in runtime updates */ + netdevice_tracker rx_reg_dev_tracker; + spinlock_t bcm_tx_lock; /* protect tx data and timer updates */ + spinlock_t bcm_rx_update_lock; /* protect filter/timer data updates */ }; struct bcm_sock { @@ -224,10 +231,13 @@ static int bcm_proc_show(struct seq_file *m, void *v) list_for_each_entry_rcu(op, &bo->rx_ops, list) { - unsigned long reduction; + long reduction, frames_filtered, frames_abs; + + frames_filtered = atomic_long_read(&op->frames_filtered); + frames_abs = atomic_long_read(&op->frames_abs); /* print only active entries & prevent division by zero */ - if (!op->frames_abs) + if (!frames_abs) continue; seq_printf(m, "rx_op: %03X %-5s ", op->can_id, @@ -249,15 +259,15 @@ static int bcm_proc_show(struct seq_file *m, void *v) (long long)ktime_to_us(op->kt_ival2)); seq_printf(m, "# recv %ld (%ld) => reduction: ", - op->frames_filtered, op->frames_abs); + frames_filtered, frames_abs); - reduction = 100 - (op->frames_filtered * 100) / op->frames_abs; + reduction = 100 - (frames_filtered * 100) / frames_abs; seq_printf(m, "%s%ld%%\n", (reduction == 100) ? "near " : "", reduction); } - list_for_each_entry(op, &bo->tx_ops, list) { + list_for_each_entry_rcu(op, &bo->tx_ops, list) { seq_printf(m, "tx_op: %03X %s ", op->can_id, bcm_proc_getifname(net, ifname, op->ifindex)); @@ -275,7 +285,8 @@ static int bcm_proc_show(struct seq_file *m, void *v) seq_printf(m, "t2=%lld ", (long long)ktime_to_us(op->kt_ival2)); - seq_printf(m, "# sent %ld\n", op->frames_abs); + seq_printf(m, "# sent %ld\n", + atomic_long_read(&op->frames_abs)); } seq_putc(m, '\n'); @@ -285,26 +296,50 @@ static int bcm_proc_show(struct seq_file *m, void *v) } #endif /* CONFIG_PROC_FS */ +static void bcm_update_rx_stats(struct bcm_op *op) +{ + /* prevent overflow of the reduction% calculation in bcm_proc_show() */ + if (atomic_long_inc_return(&op->frames_abs) > LONG_MAX / 100) { + atomic_long_set(&op->frames_filtered, 0); + atomic_long_set(&op->frames_abs, 0); + } +} + +static void bcm_update_tx_stats(struct bcm_op *op) +{ + /* tx_op has no reduction% calculation - use the full range and + * just keep the displayed counter non-negative on overflow + */ + if (atomic_long_inc_return(&op->frames_abs) == LONG_MAX) + atomic_long_set(&op->frames_abs, 0); +} + /* * bcm_can_tx - send the (next) CAN frame to the appropriate CAN interface * of the given bcm tx op */ -static void bcm_can_tx(struct bcm_op *op) +static void bcm_can_tx(struct bcm_op *op, struct canfd_frame *cf) { struct sk_buff *skb; struct can_skb_ext *csx; struct net_device *dev; - struct canfd_frame *cf; + struct canfd_frame cframe; + bool cyclic = !cf; + unsigned int idx = 0; int err; /* no target device? => exit */ if (!op->ifindex) return; - /* read currframe under lock protection */ - spin_lock_bh(&op->bcm_tx_lock); - cf = op->frames + op->cfsiz * op->currframe; - spin_unlock_bh(&op->bcm_tx_lock); + if (cyclic) { + /* read currframe under lock protection */ + spin_lock_bh(&op->bcm_tx_lock); + idx = op->currframe; + memcpy(&cframe, op->frames + op->cfsiz * idx, op->cfsiz); + cf = &cframe; + spin_unlock_bh(&op->bcm_tx_lock); + } dev = dev_get_by_index(sock_net(op->sk), op->ifindex); if (!dev) { @@ -335,16 +370,22 @@ static void bcm_can_tx(struct bcm_op *op) spin_lock_bh(&op->bcm_tx_lock); if (!err) - op->frames_abs++; + bcm_update_tx_stats(op); - op->currframe++; + /* only advance the cyclic sequence if nothing reset currframe while + * we were sending - a concurrent TX_RESET_MULTI_IDX means this + * frame's bookkeeping belongs to a sequence that no longer exists + */ + if (!cyclic || op->currframe == idx) { + op->currframe++; - /* reached last frame? */ - if (op->currframe >= op->nframes) - op->currframe = 0; + /* reached last frame? */ + if (op->currframe >= op->nframes) + op->currframe = 0; - if (op->count > 0) - op->count--; + if (op->count > 0) + op->count--; + } spin_unlock_bh(&op->bcm_tx_lock); out: @@ -433,12 +474,18 @@ static bool bcm_tx_set_expiry(struct bcm_op *op, struct hrtimer *hrt) { ktime_t ival; + spin_lock_bh(&op->bcm_tx_lock); + if (op->kt_ival1 && op->count) ival = op->kt_ival1; - else if (op->kt_ival2) + else if (op->kt_ival2) { ival = op->kt_ival2; - else + } else { + spin_unlock_bh(&op->bcm_tx_lock); return false; + } + + spin_unlock_bh(&op->bcm_tx_lock); hrtimer_set_expires(hrt, ktime_add(ktime_get(), ival)); return true; @@ -455,26 +502,48 @@ static enum hrtimer_restart bcm_tx_timeout_handler(struct hrtimer *hrtimer) { struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer); struct bcm_msg_head msg_head; + bool tx_ival1, tx_ival2; + + /* snapshot kt_ival1/kt_ival2/count under lock to avoid torn + * ktime_t reads racing with concurrent bcm_tx_setup() updates + */ + spin_lock_bh(&op->bcm_tx_lock); + tx_ival1 = op->kt_ival1 && (op->count > 0); + tx_ival2 = !!op->kt_ival2; + spin_unlock_bh(&op->bcm_tx_lock); - if (op->kt_ival1 && (op->count > 0)) { - bcm_can_tx(op); - if (!op->count && (op->flags & TX_COUNTEVT)) { + if (tx_ival1) { + u32 flags, count; + struct bcm_timeval ival1, ival2; + bcm_can_tx(op, NULL); + + /* snapshot variables under lock to avoid torn reads racing + * with concurrent bcm_tx_setup() updates + */ + spin_lock_bh(&op->bcm_tx_lock); + flags = op->flags; + count = op->count; + ival1 = op->ival1; + ival2 = op->ival2; + spin_unlock_bh(&op->bcm_tx_lock); + + if (!count && (flags & TX_COUNTEVT)) { /* create notification to user */ memset(&msg_head, 0, sizeof(msg_head)); msg_head.opcode = TX_EXPIRED; - msg_head.flags = op->flags; - msg_head.count = op->count; - msg_head.ival1 = op->ival1; - msg_head.ival2 = op->ival2; + msg_head.flags = flags; + msg_head.count = count; + msg_head.ival1 = ival1; + msg_head.ival2 = ival2; msg_head.can_id = op->can_id; msg_head.nframes = 0; bcm_send_to_user(op, &msg_head, NULL, 0); } - } else if (op->kt_ival2) { - bcm_can_tx(op); + } else if (tx_ival2) { + bcm_can_tx(op, NULL); } return bcm_tx_set_expiry(op, &op->timer) ? @@ -488,12 +557,9 @@ static void bcm_rx_changed(struct bcm_op *op, struct canfd_frame *data) { struct bcm_msg_head head; - /* update statistics */ - op->frames_filtered++; - - /* prevent statistics overflow */ - if (op->frames_filtered > ULONG_MAX/100) - op->frames_filtered = op->frames_abs = 0; + /* update statistics (frames_filtered <= frames_abs) */ + if (atomic_long_read(&op->frames_abs)) + atomic_long_inc(&op->frames_filtered); /* this element is not throttled anymore */ data->flags &= ~RX_THR; @@ -618,6 +684,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer) struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer); struct bcm_msg_head msg_head; + spin_lock_bh(&op->bcm_rx_update_lock); + /* if user wants to be informed, when cyclic CAN-Messages come back */ if ((op->flags & RX_ANNOUNCE_RESUME) && op->last_frames) { /* clear received CAN frames to indicate 'nothing received' */ @@ -634,6 +702,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer) msg_head.can_id = op->can_id; msg_head.nframes = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + bcm_send_to_user(op, &msg_head, NULL, 0); return HRTIMER_NORESTART; @@ -682,15 +752,26 @@ static int bcm_rx_thr_flush(struct bcm_op *op) static enum hrtimer_restart bcm_rx_thr_handler(struct hrtimer *hrtimer) { struct bcm_op *op = container_of(hrtimer, struct bcm_op, thrtimer); + enum hrtimer_restart ret; + + spin_lock_bh(&op->bcm_rx_update_lock); - if (bcm_rx_thr_flush(op)) { + /* kt_ival2 may have been concurrently cleared by bcm_rx_setup() + * before it cancels this timer - never forward with a zero + * interval in that case. + */ + if (bcm_rx_thr_flush(op) && op->kt_ival2) { hrtimer_forward_now(hrtimer, op->kt_ival2); - return HRTIMER_RESTART; + ret = HRTIMER_RESTART; } else { /* rearm throttle handling */ op->kt_lastmsg = 0; - return HRTIMER_NORESTART; + ret = HRTIMER_NORESTART; } + + spin_unlock_bh(&op->bcm_rx_update_lock); + + return ret; } /* @@ -700,8 +781,10 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) { struct bcm_op *op = (struct bcm_op *)data; const struct canfd_frame *rxframe = (struct canfd_frame *)skb->data; + struct canfd_frame rtrframe; unsigned int i; unsigned char traffic_flags; + bool rtr_frame; if (op->can_id != rxframe->can_id) return; @@ -715,22 +798,59 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) return; } + /* An ANYDEV op with an active RX timeout and/or throttle timer + * tracks a single source interface: claim the first interface that + * delivers a matching frame and reject frames from any other one, + * before hrtimer_cancel() below can touch op->timer - this avoids + * racing bcm_rx_timeout_handler() across concurrent interfaces. + * RX_RTR_FRAME ops are excluded, as kt_ival1/kt_ival2 may briefly + * hold a stale value from an earlier non-RTR configuration. + */ + if (!op->ifindex) { + spin_lock_bh(&op->bcm_rx_update_lock); + + if (!(op->flags & RX_RTR_FRAME) && + (op->kt_ival1 || op->kt_ival2)) { + /* don't claim to vanishing interface */ + if (!op->if_detected && + READ_ONCE(skb->dev->reg_state) == NETREG_REGISTERED) + op->if_detected = skb->dev->ifindex; + + if (op->if_detected != skb->dev->ifindex) { + spin_unlock_bh(&op->bcm_rx_update_lock); + return; + } + } + + spin_unlock_bh(&op->bcm_rx_update_lock); + } + /* disable timeout */ hrtimer_cancel(&op->timer); - /* save rx timestamp */ - op->rx_stamp = skb->tstamp; - /* save originator for recvfrom() */ - op->rx_ifindex = skb->dev->ifindex; - /* update statistics */ - op->frames_abs++; + /* op->flags/op->frames may be updated concurrently by bcm_rx_setup() */ + spin_lock_bh(&op->bcm_rx_update_lock); + + rtr_frame = op->flags & RX_RTR_FRAME; + if (rtr_frame) { + bcm_update_rx_stats(op); + /* snapshot RTR content under lock */ + memcpy(&rtrframe, op->frames, op->cfsiz); + spin_unlock_bh(&op->bcm_rx_update_lock); - if (op->flags & RX_RTR_FRAME) { /* send reply for RTR-request (placed in op->frames[0]) */ - bcm_can_tx(op); + bcm_can_tx(op, &rtrframe); return; } + /* update statistics in the same critical section as bcm_rx_changed() + * below: frames_filtered must never be checked/incremented against a + * frames_abs snapshot from a concurrent bcm_rx_handler() call on + * another CPU for the same (wildcard) op, or frames_filtered can end + * up larger than frames_abs. + */ + bcm_update_rx_stats(op); + /* compute flags to distinguish between own/local/remote CAN traffic */ traffic_flags = 0; if (skb->sk) { @@ -739,6 +859,13 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) traffic_flags |= RX_OWN; } + /* save rx timestamp and originator for recvfrom() under lock: an + * ANYDEV op without an active timer can still run concurrently on + * different CPUs, so content and meta data must be bundled here. + */ + op->rx_stamp = skb->tstamp; + op->rx_ifindex = skb->dev->ifindex; + if (op->flags & RX_FILTER_ID) { /* the easiest case */ bcm_rx_update_and_send(op, op->last_frames, rxframe, @@ -774,6 +901,8 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) rx_starttimer: bcm_rx_starttimer(op); + + spin_unlock_bh(&op->bcm_rx_update_lock); } /* @@ -793,9 +922,12 @@ static struct bcm_op *bcm_find_op(struct list_head *ops, return NULL; } -static void bcm_free_op_rcu(struct rcu_head *rcu_head) +static void bcm_free_op_work(struct work_struct *work) { - struct bcm_op *op = container_of(rcu_head, struct bcm_op, rcu); + struct bcm_op *op = container_of(work, struct bcm_op, work); + + hrtimer_cancel(&op->timer); + hrtimer_cancel(&op->thrtimer); if ((op->frames) && (op->frames != &op->sframe)) kfree(op->frames); @@ -803,9 +935,23 @@ static void bcm_free_op_rcu(struct rcu_head *rcu_head) if ((op->last_frames) && (op->last_frames != &op->last_sframe)) kfree(op->last_frames); + /* the last possible access to op->timer/op->thrtimer has now + * happened above via hrtimer_cancel() - op->sk is no longer + * needed by any pending timer callback, so drop our reference + */ + sock_put(op->sk); + kfree(op); } +static void bcm_free_op_rcu(struct rcu_head *rcu_head) +{ + struct bcm_op *op = container_of(rcu_head, struct bcm_op, rcu); + + INIT_WORK(&op->work, bcm_free_op_work); + queue_work(bcm_wq, &op->work); +} + static void bcm_remove_op(struct bcm_op *op) { hrtimer_cancel(&op->timer); @@ -822,6 +968,7 @@ static void bcm_rx_unreg(struct net_device *dev, struct bcm_op *op) /* mark as removed subscription */ op->rx_reg_dev = NULL; + netdev_put(dev, &op->rx_reg_dev_tracker); } else printk(KERN_ERR "can-bcm: bcm_rx_unreg: registered device " "mismatch %p %p\n", op->rx_reg_dev, dev); @@ -852,17 +999,14 @@ static int bcm_delete_rx_op(struct list_head *ops, struct bcm_msg_head *mh, * Only remove subscriptions that had not * been removed due to NETDEV_UNREGISTER * in bcm_notifier() + * + * op->rx_reg_dev is a tracked reference taken + * when the subscription was registered, so it + * stays valid here even if a concurrent + * NETDEV_UNREGISTER already unlisted the dev. */ - if (op->rx_reg_dev) { - struct net_device *dev; - - dev = dev_get_by_index(sock_net(op->sk), - op->ifindex); - if (dev) { - bcm_rx_unreg(dev, op); - dev_put(dev); - } - } + if (op->rx_reg_dev) + bcm_rx_unreg(op->rx_reg_dev, op); } else can_rx_unregister(sock_net(op->sk), NULL, op->can_id, @@ -930,6 +1074,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, struct bcm_sock *bo = bcm_sk(sk); struct bcm_op *op; struct canfd_frame *cf; + bool add_op_to_list = false; unsigned int i; int err; @@ -948,6 +1093,8 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* check the given can_id */ op = bcm_find_op(&bo->tx_ops, msg_head, ifindex); if (op) { + void *new_frames; + /* update existing BCM operation */ /* @@ -958,11 +1105,23 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (msg_head->nframes > op->nframes) return -E2BIG; - /* update CAN frames content */ + /* get new CAN frames content into a staging buffer before + * locking: validate and normalize the frames there so that + * bcm_can_tx() / bcm_tx_timeout_handler() never observe a + * partially updated or unvalidated frame in op->frames + */ + new_frames = kmalloc(msg_head->nframes * op->cfsiz, GFP_KERNEL); + if (!new_frames) + return -ENOMEM; + for (i = 0; i < msg_head->nframes; i++) { - cf = op->frames + op->cfsiz * i; + cf = new_frames + op->cfsiz * i; err = memcpy_from_msg((u8 *)cf, msg, op->cfsiz); + if (err < 0) { + kfree(new_frames); + return err; + } if (op->flags & CAN_FD_FRAME) { if (cf->len > 64) @@ -972,36 +1131,38 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, err = -EINVAL; } - if (err < 0) + if (err < 0) { + kfree(new_frames); return err; + } if (msg_head->flags & TX_CP_CAN_ID) { /* copy can_id into frame */ cf->can_id = msg_head->can_id; } } + + spin_lock_bh(&op->bcm_tx_lock); + + /* update CAN frames content */ + memcpy(op->frames, new_frames, msg_head->nframes * op->cfsiz); + op->flags = msg_head->flags; - /* only lock for unlikely count/nframes/currframe changes */ if (op->nframes != msg_head->nframes || - op->flags & TX_RESET_MULTI_IDX || - op->flags & SETTIMER) { - - spin_lock_bh(&op->bcm_tx_lock); + op->flags & TX_RESET_MULTI_IDX) { + /* potentially update changed nframes */ + op->nframes = msg_head->nframes; + /* restart multiple frame transmission */ + op->currframe = 0; + } - if (op->nframes != msg_head->nframes || - op->flags & TX_RESET_MULTI_IDX) { - /* potentially update changed nframes */ - op->nframes = msg_head->nframes; - /* restart multiple frame transmission */ - op->currframe = 0; - } + if (op->flags & SETTIMER) + op->count = msg_head->count; - if (op->flags & SETTIMER) - op->count = msg_head->count; + spin_unlock_bh(&op->bcm_tx_lock); - spin_unlock_bh(&op->bcm_tx_lock); - } + kfree(new_frames); } else { /* insert new BCM operation for the given can_id */ @@ -1060,6 +1221,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* bcm_can_tx / bcm_tx_timeout_handler needs this */ op->sk = sk; + sock_hold(sk); op->ifindex = ifindex; /* initialize uninitialized (kzalloc) structure */ @@ -1070,17 +1232,18 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, hrtimer_setup(&op->thrtimer, hrtimer_dummy_timeout, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - /* add this bcm_op to the list of the tx_ops */ - list_add(&op->list, &bo->tx_ops); + add_op_to_list = true; } /* if ((op = bcm_find_op(&bo->tx_ops, msg_head->can_id, ifindex))) */ if (op->flags & SETTIMER) { /* set timer values */ + spin_lock_bh(&op->bcm_tx_lock); op->ival1 = msg_head->ival1; op->ival2 = msg_head->ival2; op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + spin_unlock_bh(&op->bcm_tx_lock); /* disable an active timer due to zero values? */ if (!op->kt_ival1 && !op->kt_ival2) @@ -1093,8 +1256,12 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, op->flags |= TX_ANNOUNCE; } + /* add this bcm_op to the list of the tx_ops? */ + if (add_op_to_list) + list_add_rcu(&op->list, &bo->tx_ops); + if (op->flags & TX_ANNOUNCE) - bcm_can_tx(op); + bcm_can_tx(op, NULL); if (op->flags & STARTTIMER) bcm_tx_start_timer(op); @@ -1108,6 +1275,39 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return err; } +static int bcm_rx_setup_rtr_check(struct bcm_msg_head *msg_head, + struct bcm_op *op, void *new_frames) +{ + struct canfd_frame *frame0 = new_frames; + + if (!(msg_head->flags & RX_RTR_FRAME)) + return 0; + + /* this frame is sent out as-is by bcm_can_tx() whenever a matching + * remote request is received, so validate its length the same way + * bcm_tx_setup() validates TX_SETUP frames before installing it + */ + if (msg_head->flags & CAN_FD_FRAME) { + if (frame0->len > 64) + return -EINVAL; + } else { + if (frame0->len > 8) + return -EINVAL; + } + + /* funny feature in RX(!)_SETUP only for RTR-mode: + * copy can_id into frame BUT without RTR-flag to + * prevent a full-load-loopback-test ... ;-] + * normalize this on the staged buffer, before it is + * ever installed into op->frames. + */ + if ((msg_head->flags & TX_CP_CAN_ID) || + frame0->can_id == op->can_id) + frame0->can_id = op->can_id & ~CAN_RTR_FLAG; + + return 0; +} + /* * bcm_rx_setup - create or update a bcm rx op (for bcm_sendmsg) */ @@ -1117,6 +1317,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, struct bcm_sock *bo = bcm_sk(sk); struct bcm_op *op; int do_rx_register; + int new_op = 0; int err = 0; if ((msg_head->flags & RX_FILTER_ID) || (!(msg_head->nframes))) { @@ -1142,6 +1343,8 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* check the given can_id */ op = bcm_find_op(&bo->rx_ops, msg_head, ifindex); if (op) { + void *new_frames = NULL; + /* update existing BCM operation */ /* @@ -1153,21 +1356,62 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return -E2BIG; if (msg_head->nframes) { - /* update CAN frames content */ - err = memcpy_from_msg(op->frames, msg, + /* get new CAN frames content before locking */ + new_frames = kmalloc(msg_head->nframes * op->cfsiz, + GFP_KERNEL); + if (!new_frames) + return -ENOMEM; + + err = memcpy_from_msg(new_frames, msg, msg_head->nframes * op->cfsiz); - if (err < 0) + if (err < 0) { + kfree(new_frames); return err; + } - /* clear last_frames to indicate 'nothing received' */ - memset(op->last_frames, 0, msg_head->nframes * op->cfsiz); + err = bcm_rx_setup_rtr_check(msg_head, op, new_frames); + if (err < 0) { + kfree(new_frames); + return err; + } } + spin_lock_bh(&op->bcm_rx_update_lock); op->nframes = msg_head->nframes; op->flags = msg_head->flags; - /* Only an update -> do not call can_rx_register() */ - do_rx_register = 0; + if (msg_head->nframes) { + /* update CAN frames content */ + memcpy(op->frames, new_frames, + msg_head->nframes * op->cfsiz); + + /* clear last_frames to indicate 'nothing received' */ + memset(op->last_frames, 0, + msg_head->nframes * op->cfsiz); + } + + if (msg_head->flags & SETTIMER) { + op->ival1 = msg_head->ival1; + op->ival2 = msg_head->ival2; + op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); + op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + op->kt_lastmsg = 0; + op->if_detected = 0; /* reclaim ifindex in ANYDEV mode */ + } + spin_unlock_bh(&op->bcm_rx_update_lock); + + /* free temporary frames / kfree(NULL) is safe */ + kfree(new_frames); + + /* Don't register a new CAN filter for the rx_op update unless + * a concurrent NETDEV_UNREGISTER notifier already tore down + * the previous registration. In this case the receiver needs + * to be re-registered here so that this update doesn't + * silently stop delivering frames for the given ifindex. + * Ops with ifindex = 0 (all CAN interfaces) never carry a + * tracked rx_reg_dev and stay registered as-is. + */ + do_rx_register = (ifindex && !op->rx_reg_dev) ? 1 : 0; } else { /* insert new BCM operation for the given can_id */ @@ -1176,6 +1420,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return -ENOMEM; spin_lock_init(&op->bcm_tx_lock); + spin_lock_init(&op->bcm_rx_update_lock); op->can_id = msg_head->can_id; op->nframes = msg_head->nframes; op->cfsiz = CFSIZ(msg_head->flags); @@ -1209,18 +1454,17 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (msg_head->nframes) { err = memcpy_from_msg(op->frames, msg, msg_head->nframes * op->cfsiz); - if (err < 0) { - if (op->frames != &op->sframe) - kfree(op->frames); - if (op->last_frames != &op->last_sframe) - kfree(op->last_frames); - kfree(op); - return err; - } + if (err < 0) + goto free_op; + + err = bcm_rx_setup_rtr_check(msg_head, op, op->frames); + if (err < 0) + goto free_op; } /* bcm_can_tx / bcm_tx_timeout_handler needs this */ op->sk = sk; + sock_hold(sk); op->ifindex = ifindex; /* ifindex for timeout events w/o previous frame reception */ @@ -1232,40 +1476,31 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, hrtimer_setup(&op->thrtimer, bcm_rx_thr_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - /* add this bcm_op to the list of the rx_ops */ - list_add(&op->list, &bo->rx_ops); - /* call can_rx_register() */ do_rx_register = 1; + new_op = 1; } /* if ((op = bcm_find_op(&bo->rx_ops, msg_head->can_id, ifindex))) */ /* check flags */ if (op->flags & RX_RTR_FRAME) { - struct canfd_frame *frame0 = op->frames; - /* no timers in RTR-mode */ hrtimer_cancel(&op->thrtimer); hrtimer_cancel(&op->timer); - - /* - * funny feature in RX(!)_SETUP only for RTR-mode: - * copy can_id into frame BUT without RTR-flag to - * prevent a full-load-loopback-test ... ;-] - */ - if ((op->flags & TX_CP_CAN_ID) || - (frame0->can_id == op->can_id)) - frame0->can_id = op->can_id & ~CAN_RTR_FLAG; - } else { if (op->flags & SETTIMER) { - /* set timer value */ - op->ival1 = msg_head->ival1; - op->ival2 = msg_head->ival2; - op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); - op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + /* set timers (locked) for newly created op */ + if (new_op) { + spin_lock_bh(&op->bcm_rx_update_lock); + op->ival1 = msg_head->ival1; + op->ival2 = msg_head->ival2; + op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); + op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + op->kt_lastmsg = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + } /* disable an active timer due to zero value? */ if (!op->kt_ival1) @@ -1275,9 +1510,11 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, * In any case cancel the throttle timer, flush * potentially blocked msgs and reset throttle handling */ - op->kt_lastmsg = 0; hrtimer_cancel(&op->thrtimer); + + spin_lock_bh(&op->bcm_rx_update_lock); bcm_rx_thr_flush(op); + spin_unlock_bh(&op->bcm_rx_update_lock); } if ((op->flags & STARTTIMER) && op->kt_ival1) @@ -1285,7 +1522,10 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, HRTIMER_MODE_REL_SOFT); } - /* now we can register for can_ids, if we added a new bcm_op */ + /* now we can register for can_ids, if we added a new bcm_op + * or need to re-register after a NETDEV_UNREGISTER tore down + * the previous registration of an existing op + */ if (do_rx_register) { if (ifindex) { struct net_device *dev; @@ -1298,23 +1538,62 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, bcm_rx_handler, op, "bcm", sk); - op->rx_reg_dev = dev; + /* keep a tracked reference so that a later + * unregister can safely reach the device even + * if a concurrent NETDEV_UNREGISTER has + * already unlisted it by ifindex + */ + if (!err) { + op->rx_reg_dev = dev; + netdev_hold(dev, + &op->rx_reg_dev_tracker, + GFP_KERNEL); + } dev_put(dev); + } else { + /* the requested device is gone - do not + * silently succeed without registering + */ + err = -ENODEV; } - } else + } else { err = can_rx_register(sock_net(sk), NULL, op->can_id, REGMASK(op->can_id), bcm_rx_handler, op, "bcm", sk); + } + if (err) { - /* this bcm rx op is broken -> remove it */ - list_del_rcu(&op->list); - bcm_remove_op(op); + /* newly created bcm rx op is broken -> remove it */ + if (new_op) { + bcm_remove_op(op); + return err; + } + + /* an existing op just stays unregistered. + * Cancel op->timer and (defensively) op->thrtimer. + * Other settings can't be reached until the next + * successful RX_SETUP. + */ + hrtimer_cancel(&op->timer); + hrtimer_cancel(&op->thrtimer); return err; } + + /* add a new bcm_op to the list of the rx_ops */ + if (new_op) + list_add_rcu(&op->list, &bo->rx_ops); } return msg_head->nframes * op->cfsiz + MHSIZ; + +free_op: + if (op->frames != &op->sframe) + kfree(op->frames); + if (op->last_frames != &op->last_sframe) + kfree(op->last_frames); + kfree(op); + return err; } /* @@ -1373,12 +1652,13 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) { struct sock *sk = sock->sk; struct bcm_sock *bo = bcm_sk(sk); - int ifindex = bo->ifindex; /* default ifindex for this bcm_op */ + int ifindex; struct bcm_msg_head msg_head; int cfsiz; int ret; /* read bytes or error codes as return value */ - if (!bo->bound) + /* Lockless fast-path check for bound socket */ + if (!READ_ONCE(bo->bound)) return -ENOTCONN; /* check for valid message length from userspace */ @@ -1394,17 +1674,38 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) if ((size - MHSIZ) % cfsiz) return -EINVAL; + lock_sock(sk); + + /* Re-validate under the socket lock: a concurrent bcm_notify() + * may have unbound this socket (device removal) after the + * lockless fast-path check above. bo->ifindex is only ever + * mutated under lock_sock(), so reading it here - instead of + * before taking the lock - guarantees it can't be observed + * torn against bo->bound. + */ + if (!bo->bound) { + ret = -ENOTCONN; + goto out_release; + } + + /* default ifindex for this bcm_op */ + ifindex = bo->ifindex; + /* check for alternative ifindex for this bcm_op */ if (!ifindex && msg->msg_name) { /* no bound device as default => check msg_name */ DECLARE_SOCKADDR(struct sockaddr_can *, addr, msg->msg_name); - if (msg->msg_namelen < BCM_MIN_NAMELEN) - return -EINVAL; + if (msg->msg_namelen < BCM_MIN_NAMELEN) { + ret = -EINVAL; + goto out_release; + } - if (addr->can_family != AF_CAN) - return -EINVAL; + if (addr->can_family != AF_CAN) { + ret = -EINVAL; + goto out_release; + } /* ifindex from sendto() */ ifindex = addr->can_ifindex; @@ -1413,20 +1714,21 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) struct net_device *dev; dev = dev_get_by_index(sock_net(sk), ifindex); - if (!dev) - return -ENODEV; + if (!dev) { + ret = -ENODEV; + goto out_release; + } if (dev->type != ARPHRD_CAN) { dev_put(dev); - return -ENODEV; + ret = -ENODEV; + goto out_release; } dev_put(dev); } } - lock_sock(sk); - switch (msg_head.opcode) { case TX_SETUP: @@ -1476,6 +1778,7 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) break; } +out_release: release_sock(sk); return ret; @@ -1499,11 +1802,30 @@ static void bcm_notify(struct bcm_sock *bo, unsigned long msg, case NETDEV_UNREGISTER: lock_sock(sk); - /* remove device specific receive entries */ - list_for_each_entry(op, &bo->rx_ops, list) + /* rx_ops: remove device specific receive entries */ + list_for_each_entry(op, &bo->rx_ops, list) { if (op->rx_reg_dev == dev) bcm_rx_unreg(dev, op); + /* release an ANYDEV op's claim (see bcm_rx_handler()) + * on this now confirmed-gone interface. + */ + if (!op->ifindex) { + spin_lock_bh(&op->bcm_rx_update_lock); + if (op->if_detected == dev->ifindex) + op->if_detected = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + } + } + + /* tx_ops: stop device specific cyclic transmissions on the + * vanishing ifindex. Cancelling the timer is enough to stop + * cyclic bcm_can_tx() calls as there is no re-arming. + */ + list_for_each_entry(op, &bo->tx_ops, list) + if (op->ifindex == dev->ifindex) + hrtimer_cancel(&op->timer); + /* remove device reference, if this is our bound device */ if (bo->bound && bo->ifindex == dev->ifindex) { #if IS_ENABLED(CONFIG_PROC_FS) @@ -1512,7 +1834,12 @@ static void bcm_notify(struct bcm_sock *bo, unsigned long msg, bo->bcm_proc_read = NULL; } #endif - bo->bound = 0; + /* Paired with the lockless fast-path check in + * bcm_sendmsg(); bo->ifindex itself is only ever + * accessed under lock_sock() so it needs no + * annotation. + */ + WRITE_ONCE(bo->bound, 0); bo->ifindex = 0; notify_enodev = 1; } @@ -1616,8 +1943,10 @@ static int bcm_release(struct socket *sock) remove_proc_entry(bo->procname, net->can.bcmproc_dir); #endif /* CONFIG_PROC_FS */ - list_for_each_entry_safe(op, next, &bo->tx_ops, list) + list_for_each_entry_safe(op, next, &bo->tx_ops, list) { + list_del_rcu(&op->list); bcm_remove_op(op); + } list_for_each_entry_safe(op, next, &bo->rx_ops, list) { /* @@ -1629,16 +1958,14 @@ static int bcm_release(struct socket *sock) * Only remove subscriptions that had not * been removed due to NETDEV_UNREGISTER * in bcm_notifier() + * + * op->rx_reg_dev is a tracked reference taken + * when the subscription was registered, so it + * stays valid here even if a concurrent + * NETDEV_UNREGISTER already unlisted the device. */ - if (op->rx_reg_dev) { - struct net_device *dev; - - dev = dev_get_by_index(net, op->ifindex); - if (dev) { - bcm_rx_unreg(dev, op); - dev_put(dev); - } - } + if (op->rx_reg_dev) + bcm_rx_unreg(op->rx_reg_dev, op); } else can_rx_unregister(net, NULL, op->can_id, REGMASK(op->can_id), @@ -1648,12 +1975,14 @@ static int bcm_release(struct socket *sock) synchronize_rcu(); - list_for_each_entry_safe(op, next, &bo->rx_ops, list) + list_for_each_entry_safe(op, next, &bo->rx_ops, list) { + list_del_rcu(&op->list); bcm_remove_op(op); + } /* remove device reference */ if (bo->bound) { - bo->bound = 0; + WRITE_ONCE(bo->bound, 0); bo->ifindex = 0; } @@ -1723,7 +2052,10 @@ static int bcm_connect(struct socket *sock, struct sockaddr_unsized *uaddr, int } #endif /* CONFIG_PROC_FS */ - bo->bound = 1; + /* bo->ifindex above is fully assigned before this point; pairs + * with the lockless fast-path check in bcm_sendmsg() + */ + WRITE_ONCE(bo->bound, 1); fail: release_sock(sk); @@ -1839,11 +2171,15 @@ static int __init bcm_module_init(void) { int err; + bcm_wq = alloc_workqueue("can-bcm-wq", WQ_UNBOUND, 0); + if (!bcm_wq) + return -ENOMEM; + pr_info("can: broadcast manager protocol\n"); err = register_pernet_subsys(&canbcm_pernet_ops); if (err) - return err; + goto register_pernet_failed; err = register_netdevice_notifier(&canbcm_notifier); if (err) @@ -1861,6 +2197,8 @@ static int __init bcm_module_init(void) unregister_netdevice_notifier(&canbcm_notifier); register_notifier_failed: unregister_pernet_subsys(&canbcm_pernet_ops); +register_pernet_failed: + destroy_workqueue(bcm_wq); return err; } @@ -1869,6 +2207,8 @@ static void __exit bcm_module_exit(void) can_proto_unregister(&bcm_can_proto); unregister_netdevice_notifier(&canbcm_notifier); unregister_pernet_subsys(&canbcm_pernet_ops); + rcu_barrier(); + destroy_workqueue(bcm_wq); } module_init(bcm_module_init); diff --git a/net/can/isotp.c b/net/can/isotp.c index c48b4a818297e..54becaf6898f1 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -152,11 +152,13 @@ struct isotp_sock { struct sock sk; int bound; int ifindex; + struct net_device *dev; + netdevice_tracker dev_tracker; canid_t txid; canid_t rxid; ktime_t tx_gap; ktime_t lastrxcf_tstamp; - struct hrtimer rxtimer, txtimer, txfrtimer; + struct hrtimer rxtimer, txtimer, txfrtimer, echotimer; struct can_isotp_options opt; struct can_isotp_fc_options rxfc, txfc; struct can_isotp_ll_options ll; @@ -164,6 +166,7 @@ struct isotp_sock { u32 force_tx_stmin; u32 force_rx_stmin; u32 cfecho; /* consecutive frame echo tag */ + u32 tx_gen; /* generation, bumped per new tx transfer */ struct tpcon rx, tx; struct list_head notifier; wait_queue_head_t wait; @@ -376,6 +379,15 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) hrtimer_cancel(&so->txtimer); + /* isotp_tx_timeout() may have given up on this job while + * hrtimer_cancel() above waited for it to finish; so->rx_lock + * (held by our caller isotp_rcv()) rules out a concurrent claim, + * so a plain recheck is enough here. + */ + if (so->tx.state != ISOTP_WAIT_FC && + so->tx.state != ISOTP_WAIT_FIRST_FC) + return 1; + if ((cf->len < ae + FC_CONTENT_SZ) || ((so->opt.flags & ISOTP_CHECK_PADDING) && check_pad(so, cf, ae + FC_CONTENT_SZ, so->opt.rxpad_content))) { @@ -422,7 +434,7 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) so->tx.bs = 0; so->tx.state = ISOTP_SENDING; /* send CF frame and enable echo timeout handling */ - hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), + hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); isotp_send_cframe(so); break; @@ -575,6 +587,14 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, hrtimer_cancel(&so->rxtimer); + /* isotp_rx_timer_handler() may have raced us for so->rx.state + * while hrtimer_cancel() above waited for it to finish, already + * reporting ETIMEDOUT and resetting the reception; don't process + * this CF into a reassembly that has already been given up on. + */ + if (so->rx.state != ISOTP_WAIT_DATA) + return 1; + /* CFs are never longer than the FF */ if (cf->len > so->rx.ll_dl) return 1; @@ -870,20 +890,36 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) struct canfd_frame *cf = (struct canfd_frame *)skb->data; /* only handle my own local echo CF/SF skb's (no FF!) */ - if (skb->sk != sk || so->cfecho != *(u32 *)cf->data) + if (skb->sk != sk) return; + /* unlike isotp_rcv_fc()/isotp_rcv_cf(), not already under so->rx_lock + * (no isotp_rcv() caller here), so take it ourselves + */ + spin_lock(&so->rx_lock); + + /* so->cfecho may since belong to a new transfer; recheck under lock */ + if (so->cfecho != *(u32 *)cf->data) + goto out_unlock; + /* cancel local echo timeout */ - hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->echotimer); /* local echo skb with consecutive frame has been consumed */ so->cfecho = 0; + /* claiming a transfer also takes so->rx_lock, so a plain recheck + * is enough: so->tx.state can't have flipped to ISOTP_SENDING for + * a new claim while we're still in here + */ + if (so->tx.state != ISOTP_SENDING) + goto out_unlock; + if (so->tx.idx >= so->tx.len) { /* we are done */ so->tx.state = ISOTP_IDLE; wake_up_interruptible(&so->wait); - return; + goto out_unlock; } if (so->txfc.bs && so->tx.bs >= so->txfc.bs) { @@ -891,53 +927,83 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) so->tx.state = ISOTP_WAIT_FC; hrtimer_start(&so->txtimer, ktime_set(ISOTP_FC_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); - return; + goto out_unlock; } /* no gap between data frames needed => use burst mode */ if (!so->tx_gap) { /* enable echo timeout handling */ - hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), + hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); isotp_send_cframe(so); - return; + goto out_unlock; } /* start timer to send next consecutive frame with correct delay */ hrtimer_start(&so->txfrtimer, so->tx_gap, HRTIMER_MODE_REL_SOFT); + +out_unlock: + spin_unlock(&so->rx_lock); } -static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) +/* shared by so->txtimer's and so->echotimer's callbacks. Both timers get + * cancelled under so->rx_lock elsewhere, so this must stay lock-free to + * avoid deadlocking with that; uses so->tx_gen instead to avoid tainting + * a new transfer with an error from the one that just timed out. + */ +static enum hrtimer_restart isotp_tx_timeout(struct isotp_sock *so) { - struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, - txtimer); struct sock *sk = &so->sk; + u32 gen = READ_ONCE(so->tx_gen); + u32 old_state = READ_ONCE(so->tx.state); /* don't handle timeouts in IDLE or SHUTDOWN state */ - if (so->tx.state == ISOTP_IDLE || so->tx.state == ISOTP_SHUTDOWN) + if (old_state == ISOTP_IDLE || old_state == ISOTP_SHUTDOWN) + return HRTIMER_NORESTART; + + /* only claim the timeout if the state is still unchanged */ + if (cmpxchg(&so->tx.state, old_state, ISOTP_IDLE) != old_state) return HRTIMER_NORESTART; /* we did not get any flow control or echo frame in time */ - /* report 'communication error on send' */ - sk->sk_err = ECOMM; - if (!sock_flag(sk, SOCK_DEAD)) - sk_error_report(sk); + if (READ_ONCE(so->tx_gen) == gen) { + /* report 'communication error on send' */ + sk->sk_err = ECOMM; + if (!sock_flag(sk, SOCK_DEAD)) + sk_error_report(sk); + } - /* reset tx state */ - so->tx.state = ISOTP_IDLE; wake_up_interruptible(&so->wait); return HRTIMER_NORESTART; } +/* so->txtimer: fires when a Flow Control frame does not arrive in time */ +static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + txtimer); + + return isotp_tx_timeout(so); +} + +/* so->echotimer: fires when a sent CF/SF's local echo does not arrive */ +static enum hrtimer_restart isotp_echo_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + echotimer); + + return isotp_tx_timeout(so); +} + static enum hrtimer_restart isotp_txfr_timer_handler(struct hrtimer *hrtimer) { struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, txfrtimer); /* start echo timeout handling and cover below protocol error */ - hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), + hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); /* cfecho should be consumed by isotp_rcv_echo() here */ @@ -958,13 +1024,24 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; int wait_tx_done = (so->opt.flags & CAN_ISOTP_WAIT_TX_DONE) ? 1 : 0; s64 hrtimer_sec = ISOTP_ECHO_TIMEOUT; + struct hrtimer *tx_hrt = &so->echotimer; + u32 new_state = ISOTP_SENDING; int off; int err; if (!so->bound || so->tx.state == ISOTP_SHUTDOWN) return -EADDRNOTAVAIL; - while (cmpxchg(&so->tx.state, ISOTP_IDLE, ISOTP_SENDING) != ISOTP_IDLE) { + /* claim the socket under so->rx_lock: this serializes the claim + * with the RX path and with sendmsg()'s own error paths below, so + * none of them can ever see a transfer mid-claim + */ + for (;;) { + spin_lock_bh(&so->rx_lock); + if (READ_ONCE(so->tx.state) == ISOTP_IDLE) + break; + spin_unlock_bh(&so->rx_lock); + /* we do not support multiple buffers - for now */ if (msg->msg_flags & MSG_DONTWAIT) return -EAGAIN; @@ -973,9 +1050,29 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) return -EADDRNOTAVAIL; /* wait for complete transmission of current pdu */ - err = wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + err = wait_event_interruptible(so->wait, + so->tx.state == ISOTP_IDLE); if (err) - goto err_event_drop; + return err; + } + + /* new transfer: bump so->tx_gen and drain the old one's timers, + * still under the so->rx_lock we just claimed the socket with + */ + WRITE_ONCE(so->tx.state, ISOTP_SENDING); + WRITE_ONCE(so->tx_gen, READ_ONCE(so->tx_gen) + 1); + hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->echotimer); + hrtimer_cancel(&so->txfrtimer); + so->cfecho = 0; + spin_unlock_bh(&so->rx_lock); + + /* so->bound is only checked once above - a wakeup may have + * unbound/rebound the socket meanwhile, so re-validate it + */ + if (!so->bound) { + err = -EADDRNOTAVAIL; + goto err_out_drop; } /* PDU size > default => try max_pdu_size */ @@ -1086,18 +1183,33 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) so->cfecho = *(u32 *)cf->data; } else { /* standard flow control check */ - so->tx.state = ISOTP_WAIT_FIRST_FC; + new_state = ISOTP_WAIT_FIRST_FC; /* start timeout for FC */ hrtimer_sec = ISOTP_FC_TIMEOUT; + tx_hrt = &so->txtimer; /* no CF echo tag for isotp_rcv_echo() (FF-mode) */ so->cfecho = 0; } } - hrtimer_start(&so->txtimer, ktime_set(hrtimer_sec, 0), + spin_lock_bh(&so->rx_lock); + if (so->tx.state == ISOTP_SHUTDOWN) { + /* isotp_release() has since taken over and already drained + * our timers - don't send into a socket that's going away + */ + spin_unlock_bh(&so->rx_lock); + kfree_skb(skb); + dev_put(dev); + wake_up_interruptible(&so->wait); + return -EADDRNOTAVAIL; + } + /* WAIT_FIRST_FC for standard FF, else stays ISOTP_SENDING */ + so->tx.state = new_state; + hrtimer_start(tx_hrt, ktime_set(hrtimer_sec, 0), HRTIMER_MODE_REL_SOFT); + spin_unlock_bh(&so->rx_lock); /* send the first or only CAN frame */ cf->flags = so->ll.tx_flags; @@ -1110,13 +1222,10 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) pr_notice_once("can-isotp: %s: can_send_ret %pe\n", __func__, ERR_PTR(err)); + spin_lock_bh(&so->rx_lock); /* no transmission -> no timeout monitoring */ - hrtimer_cancel(&so->txtimer); - - /* reset consecutive frame echo tag */ - so->cfecho = 0; - - goto err_out_drop; + hrtimer_cancel(tx_hrt); + goto err_out_drop_locked; } if (wait_tx_done) { @@ -1132,14 +1241,21 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) return size; +err_out_drop: + /* claimed but nothing sent yet - no timer to cancel */ + spin_lock_bh(&so->rx_lock); + goto err_out_drop_locked; err_event_drop: - /* got signal: force tx state machine to be idle */ - so->tx.state = ISOTP_IDLE; + /* interrupted waiting on our own transfer - drain its timers */ + spin_lock_bh(&so->rx_lock); hrtimer_cancel(&so->txfrtimer); hrtimer_cancel(&so->txtimer); -err_out_drop: - /* drop this PDU and unlock a potential wait queue */ + hrtimer_cancel(&so->echotimer); +err_out_drop_locked: + /* release the claim; so->rx_lock still held from above */ + so->cfecho = 0; so->tx.state = ISOTP_IDLE; + spin_unlock_bh(&so->rx_lock); wake_up_interruptible(&so->wait); return err; @@ -1201,13 +1317,20 @@ static int isotp_release(struct socket *sock) so = isotp_sk(sk); net = sock_net(sk); - /* wait for complete transmission of current pdu */ - while (wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE) == 0 && - cmpxchg(&so->tx.state, ISOTP_IDLE, ISOTP_SHUTDOWN) != ISOTP_IDLE) + /* best-effort: wait for a running pdu to finish, but don't block on + * it forever - give up after the first signal + */ + while (so->tx.state != ISOTP_IDLE && + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE) == 0) ; - /* force state machines to be idle also when a signal occurred */ + /* claim the socket under so->rx_lock like sendmsg() does, so its + * claim can't race the forced ISOTP_SHUTDOWN below; force it + * unconditionally, even when a signal cut the wait above short + */ + spin_lock_bh(&so->rx_lock); so->tx.state = ISOTP_SHUTDOWN; + spin_unlock_bh(&so->rx_lock); so->rx.state = ISOTP_IDLE; spin_lock(&isotp_notifier_lock); @@ -1219,36 +1342,43 @@ static int isotp_release(struct socket *sock) list_del(&so->notifier); spin_unlock(&isotp_notifier_lock); + rtnl_lock(); lock_sock(sk); - /* remove current filters & unregister */ - if (so->bound) { - if (so->ifindex) { - struct net_device *dev; - - dev = dev_get_by_index(net, so->ifindex); - if (dev) { - if (isotp_register_rxid(so)) - can_rx_unregister(net, dev, so->rxid, - SINGLE_MASK(so->rxid), - isotp_rcv, sk); - - can_rx_unregister(net, dev, so->txid, - SINGLE_MASK(so->txid), - isotp_rcv_echo, sk); - dev_put(dev); - synchronize_rcu(); - } - } + /* remove current filters & unregister + * tracked reference so->dev is taken at bind() time with rtnl_lock + */ + if (so->bound && so->dev) { + if (isotp_register_rxid(so)) + can_rx_unregister(net, so->dev, so->rxid, + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + + can_rx_unregister(net, so->dev, so->txid, + SINGLE_MASK(so->txid), + isotp_rcv_echo, sk); + netdev_put(so->dev, &so->dev_tracker); } + so->ifindex = 0; + so->bound = 0; + so->dev = NULL; + + rtnl_unlock(); + + /* Always wait for a grace period before touching the timers below. + * A concurrent NETDEV_UNREGISTER may have already unregistered our + * filters and cleared so->bound in isotp_notify() without waiting + * for in-flight isotp_rcv() callers to finish, so this call must not + * be skipped just because so->bound is already 0 here. + */ + synchronize_rcu(); + hrtimer_cancel(&so->txfrtimer); hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->echotimer); hrtimer_cancel(&so->rxtimer); - so->ifindex = 0; - so->bound = 0; - sock_orphan(sk); sock->sk = NULL; @@ -1303,6 +1433,7 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l if (!addr->can_ifindex) return -ENODEV; + rtnl_lock(); lock_sock(sk); if (so->bound) { @@ -1310,6 +1441,17 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l goto out; } + /* A transmission or reception that outlived a previous binding + * (unbound by NETDEV_UNREGISTER) may still be draining; the FC/echo + * and RX watchdog timers bound how long this takes. Checked together + * with so->bound in the same lock_sock() section above, so there is + * no window in which a concurrent isotp_notify() could be missed. + */ + if (so->tx.state != ISOTP_IDLE || so->rx.state != ISOTP_IDLE) { + err = -EAGAIN; + goto out; + } + /* ensure different CAN IDs when the rx_id is to be registered */ if (isotp_register_rxid(so) && rx_id == tx_id) { err = -EADDRNOTAVAIL; @@ -1322,14 +1464,12 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l goto out; } if (dev->type != ARPHRD_CAN) { - dev_put(dev); err = -ENODEV; - goto out; + goto out_put_dev; } if (READ_ONCE(dev->mtu) < so->ll.mtu) { - dev_put(dev); err = -EINVAL; - goto out; + goto out_put_dev; } if (!(dev->flags & IFF_UP)) notify_enetdown = 1; @@ -1347,16 +1487,25 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l can_rx_register(net, dev, tx_id, SINGLE_MASK(tx_id), isotp_rcv_echo, sk, "isotpe", sk); - dev_put(dev); - /* switch to new settings */ so->ifindex = ifindex; so->rxid = rx_id; so->txid = tx_id; so->bound = 1; + /* bind() ok -> hold a reference for so->dev so that isotp_release() + * can safely reach the device later, even if a concurrent + * NETDEV_UNREGISTER has already unlisted it by ifindex. + */ + so->dev = dev; + netdev_hold(so->dev, &so->dev_tracker, GFP_KERNEL); + +out_put_dev: + /* remove potential reference from dev_get_by_index() */ + dev_put(dev); out: release_sock(sk); + rtnl_unlock(); if (notify_enetdown) { sk->sk_err = ENETDOWN; @@ -1559,7 +1708,7 @@ static void isotp_notify(struct isotp_sock *so, unsigned long msg, if (!net_eq(dev_net(dev), sock_net(sk))) return; - if (so->ifindex != dev->ifindex) + if (so->dev != dev) return; switch (msg) { @@ -1575,10 +1724,12 @@ static void isotp_notify(struct isotp_sock *so, unsigned long msg, can_rx_unregister(dev_net(dev), dev, so->txid, SINGLE_MASK(so->txid), isotp_rcv_echo, sk); + netdev_put(so->dev, &so->dev_tracker); } so->ifindex = 0; so->bound = 0; + so->dev = NULL; release_sock(sk); sk->sk_err = ENODEV; @@ -1638,6 +1789,7 @@ static int isotp_init(struct sock *sk) so->ifindex = 0; so->bound = 0; + so->dev = NULL; so->opt.flags = CAN_ISOTP_DEFAULT_FLAGS; so->opt.ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; @@ -1664,10 +1816,14 @@ static int isotp_init(struct sock *sk) so->rx.buflen = ARRAY_SIZE(so->rx.sbuf); so->tx.buflen = ARRAY_SIZE(so->tx.sbuf); - hrtimer_setup(&so->rxtimer, isotp_rx_timer_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - hrtimer_setup(&so->txtimer, isotp_tx_timer_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - hrtimer_setup(&so->txfrtimer, isotp_txfr_timer_handler, CLOCK_MONOTONIC, - HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->rxtimer, isotp_rx_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->txtimer, isotp_tx_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->echotimer, isotp_echo_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->txfrtimer, isotp_txfr_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); init_waitqueue_head(&so->wait); spin_lock_init(&so->rx_lock); diff --git a/net/core/dev.c b/net/core/dev.c index 0c6c270d9f7d1..f81ce83fb3250 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -5381,8 +5381,6 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, u32 tail; reason = SKB_DROP_REASON_DEV_READY; - if (unlikely(!netif_running(skb->dev))) - goto bad_dev; sd = &per_cpu(softnet_data, cpu); @@ -5394,6 +5392,10 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, backlog_lock_irq_save(sd, &flags); qlen = skb_queue_len(&sd->input_pkt_queue); if (likely(qlen <= max_backlog)) { + if (unlikely(!netif_running(skb->dev))) { + backlog_unlock_irq_restore(sd, flags); + goto bad_dev; + } if (!qlen) { /* Schedule NAPI for backlog device. We can use * non atomic operation as we own the queue lock. @@ -11217,7 +11219,8 @@ static int netif_alloc_netdev_queues(struct net_device *dev) netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); spin_lock_init(&dev->tx_global_lock); - + spin_lock_init(&dev->watchdog_lock); + dev->watchdog_ref_held = false; return 0; } diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c index 8ca634964e363..cf374c2087325 100644 --- a/net/core/fib_rules.c +++ b/net/core/fib_rules.c @@ -349,7 +349,7 @@ int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl, if (err != -EAGAIN) { if ((arg->flags & FIB_LOOKUP_NOREF) || - likely(refcount_inc_not_zero(&rule->refcnt))) { + likely(fib_rule_get_safe(rule))) { arg->rule = rule; goto out; } @@ -410,8 +410,12 @@ int fib_rules_dump(struct net *net, struct notifier_block *nb, int family, if (!ops) return -EAFNOSUPPORT; list_for_each_entry_rcu(rule, &ops->rules_list, list) { + if (!fib_rule_get_safe(rule)) + continue; + err = call_fib_rule_notifier(nb, FIB_EVENT_RULE_ADD, rule, family, extack); + fib_rule_put(rule); if (err) break; } diff --git a/net/core/filter.c b/net/core/filter.c index 80439767e0eea..f21aeff99cc98 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -2733,11 +2733,13 @@ BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start, poffset += len; sge->length = 0; put_page(sg_page(sge)); + __clear_bit(i, msg->sg.copy); sk_msg_iter_var_next(i); } while (i != last_sge); sg_set_page(&msg->sg.data[first_sge], page, copy, 0); + __clear_bit(first_sge, msg->sg.copy); /* To repair sg ring we need to shift entries. If we only * had a single entry though we can just replace it and @@ -2763,9 +2765,11 @@ BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start, break; msg->sg.data[i] = msg->sg.data[move_from]; + sk_msg_sg_copy_assign(msg, i, msg, move_from); msg->sg.data[move_from].length = 0; msg->sg.data[move_from].page_link = 0; msg->sg.data[move_from].offset = 0; + __clear_bit(move_from, msg->sg.copy); sk_msg_iter_var_next(i); } while (1); @@ -2794,6 +2798,7 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, { struct scatterlist sge, nsge, nnsge, rsge = {0}, *psge; u32 new, i = 0, l = 0, space, copy = 0, offset = 0; + bool sge_copy, nsge_copy, nnsge_copy, rsge_copy = false; u8 *raw, *to, *from; struct page *page; @@ -2829,6 +2834,9 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, if (!space || (space == 1 && start != offset)) copy = msg->sg.data[i].length; + if (unlikely(copy + len < copy)) + return -EINVAL; + page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP, get_order(copy + len)); if (unlikely(!page)) @@ -2866,6 +2874,7 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, sk_msg_iter_var_prev(i); psge = sk_msg_elem(msg, i); rsge = sk_msg_elem_cpy(msg, i); + rsge_copy = test_bit(i, msg->sg.copy); psge->length = start - offset; rsge.length -= psge->length; @@ -2890,24 +2899,32 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, /* Shift one or two slots as needed */ sge = sk_msg_elem_cpy(msg, new); + sge_copy = test_bit(new, msg->sg.copy); sg_unmark_end(&sge); nsge = sk_msg_elem_cpy(msg, i); + nsge_copy = test_bit(i, msg->sg.copy); if (rsge.length) { sk_msg_iter_var_next(i); nnsge = sk_msg_elem_cpy(msg, i); + nnsge_copy = test_bit(i, msg->sg.copy); sk_msg_iter_next(msg, end); } while (i != msg->sg.end) { msg->sg.data[i] = sge; + __assign_bit(i, msg->sg.copy, sge_copy); sge = nsge; + sge_copy = nsge_copy; sk_msg_iter_var_next(i); if (rsge.length) { nsge = nnsge; + nsge_copy = nnsge_copy; nnsge = sk_msg_elem_cpy(msg, i); + nnsge_copy = test_bit(i, msg->sg.copy); } else { nsge = sk_msg_elem_cpy(msg, i); + nsge_copy = test_bit(i, msg->sg.copy); } } @@ -2921,6 +2938,7 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, get_page(sg_page(&rsge)); sk_msg_iter_var_next(new); msg->sg.data[new] = rsge; + __assign_bit(new, msg->sg.copy, rsge_copy); } sk_msg_reset_curr(msg); @@ -2948,25 +2966,33 @@ static void sk_msg_shift_left(struct sk_msg *msg, int i) prev = i; sk_msg_iter_var_next(i); msg->sg.data[prev] = msg->sg.data[i]; + sk_msg_sg_copy_assign(msg, prev, msg, i); } while (i != msg->sg.end); sk_msg_iter_prev(msg, end); + __clear_bit(msg->sg.end, msg->sg.copy); } static void sk_msg_shift_right(struct sk_msg *msg, int i) { struct scatterlist tmp, sge; + bool tmp_copy, sge_copy; sk_msg_iter_next(msg, end); sge = sk_msg_elem_cpy(msg, i); + sge_copy = test_bit(i, msg->sg.copy); sk_msg_iter_var_next(i); tmp = sk_msg_elem_cpy(msg, i); + tmp_copy = test_bit(i, msg->sg.copy); while (i != msg->sg.end) { msg->sg.data[i] = sge; + __assign_bit(i, msg->sg.copy, sge_copy); sk_msg_iter_var_next(i); sge = tmp; + sge_copy = tmp_copy; tmp = sk_msg_elem_cpy(msg, i); + tmp_copy = test_bit(i, msg->sg.copy); } } @@ -2974,8 +3000,8 @@ BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, u32, len, u64, flags) { u32 i = 0, l = 0, space, offset = 0; - u64 last = start + len; - int pop; + u64 last = (u64)start + len; + u32 pop; if (unlikely(flags)) return -EINVAL; @@ -3026,6 +3052,8 @@ BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, struct scatterlist *nsge, *sge = sk_msg_elem(msg, i); int a = start - offset; int b = sge->length - pop - a; + u32 sge_i = i; + bool sge_copy = test_bit(i, msg->sg.copy); sk_msg_iter_var_next(i); @@ -3038,6 +3066,7 @@ BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, sg_set_page(nsge, sg_page(sge), b, sge->offset + pop + a); + __assign_bit(i, msg->sg.copy, sge_copy); } else { struct page *page, *orig; u8 *to, *from; @@ -3054,6 +3083,7 @@ BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, memcpy(to, from, a); memcpy(to + a, from + a + pop, b); sg_set_page(sge, page, a + b, 0); + __clear_bit(sge_i, msg->sg.copy); put_page(orig); } pop = 0; @@ -4403,7 +4433,7 @@ u32 xdp_master_redirect(struct xdp_buff *xdp) struct net_device *master, *slave; master = netdev_master_upper_dev_get_rcu(xdp->rxq->dev); - if (unlikely(!(master->flags & IFF_UP))) + if (unlikely(!master || !(master->flags & IFF_UP))) return XDP_ABORTED; slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp); if (slave && slave != xdp->rxq->dev) { @@ -5544,11 +5574,24 @@ static int sol_tcp_sockopt(struct sock *sk, int optname, KERNEL_SOCKPTR(optval), *optlen); } +static bool sk_allows_sol_ip_sockopt(struct sock *sk) +{ + switch (sk->sk_family) { + case AF_INET: + return true; + case AF_INET6: + /* Allow getting/setting sockopt for possible ipv4-mapped ipv6 socket. */ + return sk->sk_type != SOCK_RAW && !ipv6_only_sock(sk); + default: + return false; + } +} + static int sol_ip_sockopt(struct sock *sk, int optname, char *optval, int *optlen, bool getopt) { - if (sk->sk_family != AF_INET) + if (!sk_allows_sol_ip_sockopt(sk)) return -EINVAL; switch (optname) { @@ -6139,7 +6182,7 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, struct in_device *in_dev; struct net_device *dev; struct fib_result res; - struct flowi4 fl4; + struct flowi4 fl4 = {}; u32 mtu = 0; int err; @@ -6279,7 +6322,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, struct neighbour *neigh; struct net_device *dev; struct inet6_dev *idev; - struct flowi6 fl6; + struct flowi6 fl6 = {}; int strict = 0; int oif, err; u32 mtu = 0; diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 2a98f5fa74eb0..8aa4f9b4df810 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -1173,13 +1173,21 @@ bool __skb_flow_dissect(const struct net *net, if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { - struct ethhdr *eth = eth_hdr(skb); struct flow_dissector_key_eth_addrs *key_eth_addrs; key_eth_addrs = skb_flow_dissector_target(flow_dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS, target_container); - memcpy(key_eth_addrs, eth, sizeof(*key_eth_addrs)); + /* TC filter blocks can be shared across devices with + * different link types, so we cannot validate this + * when the filter is installed -- check at dissect time. + */ + if (skb && skb->dev && + skb->dev->type == ARPHRD_ETHER && + skb_mac_header_was_set(skb)) + memcpy(key_eth_addrs, eth_hdr(skb), sizeof(*key_eth_addrs)); + else + memset(key_eth_addrs, 0, sizeof(*key_eth_addrs)); } if (dissector_uses_key(flow_dissector, diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index f71ef82a5f3d3..bf588f508b79e 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -599,6 +599,7 @@ static int handle_gso_encap(struct sk_buff *skb, bool ipv4, int encap_len) int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress) { + bool is_udp_tunnel; struct iphdr *iph; bool ipv4; int err; @@ -612,10 +613,16 @@ int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress) ipv4 = true; if (unlikely(len < iph->ihl * 4)) return -EINVAL; + is_udp_tunnel = iph->protocol == IPPROTO_UDP; + if (unlikely(is_udp_tunnel && len < iph->ihl * 4 + sizeof(struct udphdr))) + return -EINVAL; } else if (iph->version == 6) { ipv4 = false; if (unlikely(len < sizeof(struct ipv6hdr))) return -EINVAL; + is_udp_tunnel = ((struct ipv6hdr *)iph)->nexthdr == NEXTHDR_UDP; + if (unlikely(is_udp_tunnel && len < sizeof(struct ipv6hdr) + sizeof(struct udphdr))) + return -EINVAL; } else { return -EINVAL; } @@ -637,6 +644,11 @@ int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress) if (ingress) skb_postpush_rcsum(skb, iph, len); skb_reset_network_header(skb); + if (is_udp_tunnel) { + size_t iph_sz = ipv4 ? iph->ihl * 4 : sizeof(struct ipv6hdr); + + skb_set_transport_header(skb, skb_network_offset(skb) + iph_sz); + } memcpy(skb_network_header(skb), hdr, len); bpf_compute_data_pointers(skb); skb_clear_hash(skb); diff --git a/net/core/lwtunnel.c b/net/core/lwtunnel.c index f9d76d85d04fd..b01a395d9a966 100644 --- a/net/core/lwtunnel.c +++ b/net/core/lwtunnel.c @@ -350,6 +350,8 @@ int lwtunnel_output(struct net *net, struct sock *sk, struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->output)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->output(net, sk, skb); dev_xmit_recursion_dec(); @@ -404,6 +406,8 @@ int lwtunnel_xmit(struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->xmit)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->xmit(skb); dev_xmit_recursion_dec(); @@ -455,6 +459,8 @@ int lwtunnel_input(struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->input)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->input(skb); dev_xmit_recursion_dec(); diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index 119eaa6501d5e..ed76544b2d45c 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -189,7 +190,8 @@ netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, goto nla_put_failure; if (napi->thread) { - pid = task_pid_nr(napi->thread); + pid = task_pid_nr_ns(napi->thread, + task_active_pid_ns(current)); if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) goto nla_put_failure; } diff --git a/net/core/netdev_rx_queue.c b/net/core/netdev_rx_queue.c index de4dac4c88b36..00a7011eb4d57 100644 --- a/net/core/netdev_rx_queue.c +++ b/net/core/netdev_rx_queue.c @@ -338,12 +338,12 @@ void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq, void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq, struct netdev_rx_queue *virt_rxq) { - struct pp_memory_provider_params *p = &phys_rxq->mp_params; unsigned int rxq_idx = get_netdev_rx_queue_index(phys_rxq); + struct pp_memory_provider_params p = phys_rxq->mp_params; - if (!p->mp_ops) + if (!p.mp_ops) return; - __netif_mp_uninstall_rxq(virt_rxq, p); - __netif_mp_close_rxq(phys_rxq->dev, rxq_idx, p); + __netif_mp_close_rxq(phys_rxq->dev, rxq_idx, &p); + __netif_mp_uninstall_rxq(virt_rxq, &p); } diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 3f4a17fa5713a..9004e3145c11b 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -814,14 +814,6 @@ static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head) container_of(rcu_head, struct netpoll_info, rcu); skb_queue_purge(&npinfo->txq); - - /* we can't call cancel_delayed_work_sync here, as we are in softirq */ - cancel_delayed_work(&npinfo->tx_work); - - /* clean after last, unfinished work */ - __skb_queue_purge(&npinfo->txq); - /* now cancel it again */ - cancel_delayed_work(&npinfo->tx_work); kfree(npinfo); } @@ -845,6 +837,7 @@ static void __netpoll_cleanup(struct netpoll *np) ops->ndo_netpoll_cleanup(np->dev); RCU_INIT_POINTER(np->dev->npinfo, NULL); + disable_delayed_work_sync(&npinfo->tx_work); call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info); } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 511c25bf6f2a2..3be5f264f5ad9 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2421,6 +2421,14 @@ struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid) } EXPORT_SYMBOL_GPL(rtnl_get_net_ns_capable); +bool rtnl_dev_link_net_capable(const struct net_device *dev, + const struct net *link_net) +{ + return net_eq(link_net, dev_net(dev)) || + ns_capable(link_net->user_ns, CAP_NET_ADMIN); +} +EXPORT_SYMBOL_GPL(rtnl_dev_link_net_capable); + static int rtnl_valid_dump_ifinfo_req(const struct nlmsghdr *nlh, bool strict_check, struct nlattr **tb, struct netlink_ext_ack *extack) diff --git a/net/core/skmsg.c b/net/core/skmsg.c index e1850caf1a71a..30c3b9a2681c4 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -66,6 +66,7 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, sge = &msg->sg.data[msg->sg.end]; sg_unmark_end(sge); sg_set_page(sge, pfrag->page, use, orig_offset); + __clear_bit(msg->sg.end, msg->sg.copy); get_page(pfrag->page); sk_msg_iter_next(msg, end); } @@ -186,6 +187,7 @@ static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i, sk_mem_uncharge(sk, len); put_page(sg_page(sge)); } + __clear_bit(i, msg->sg.copy); memset(sge, 0, sizeof(*sge)); return len; } diff --git a/net/devlink/rate.c b/net/devlink/rate.c index 41be2d6c2954e..533d21b028a7b 100644 --- a/net/devlink/rate.c +++ b/net/devlink/rate.c @@ -486,16 +486,19 @@ static int devlink_nl_rate_set(struct devlink_rate *devlink_rate, devlink_rate->tx_weight = weight; } - nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME]; - if (nla_parent) { - err = devlink_nl_rate_parent_node_set(devlink_rate, info, - nla_parent); + if (attrs[DEVLINK_ATTR_RATE_TC_BWS]) { + err = devlink_nl_rate_tc_bw_set(devlink_rate, info); if (err) return err; } - if (attrs[DEVLINK_ATTR_RATE_TC_BWS]) { - err = devlink_nl_rate_tc_bw_set(devlink_rate, info); + /* Keep parent setting last because it takes a reference. This function + * has no rollback, so failing after taking the ref would leak it. + */ + nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME]; + if (nla_parent) { + err = devlink_nl_rate_parent_node_set(devlink_rate, info, + nla_parent); if (err) return err; } @@ -725,11 +728,6 @@ devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name, if (!rate_node) return ERR_PTR(-ENOMEM); - if (parent) { - rate_node->parent = parent; - refcount_inc(&rate_node->parent->refcnt); - } - rate_node->type = DEVLINK_RATE_TYPE_NODE; rate_node->devlink = devlink; rate_node->priv = priv; @@ -740,6 +738,11 @@ devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name, return ERR_PTR(-ENOMEM); } + if (parent) { + rate_node->parent = parent; + refcount_inc(&rate_node->parent->refcnt); + } + refcount_set(&rate_node->refcnt, 1); list_add(&rate_node->list, &devlink->rate_list); devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW); diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index fd2198e45d2bb..733fb3c3a63a5 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -115,6 +115,34 @@ static inline void ethnl_update_u8(u8 *dst, const struct nlattr *attr, *mod = true; } +/** + * ethnl_update_u8_u32() - update u8 value from an NLA_U32 attribute + * @dst: value to update + * @attr: netlink attribute with new value or null + * @mod: pointer to bool for modification tracking + * + * Some attributes are NLA_U32 on the wire but are stored in a u8. Read the + * full 32-bit value from NLA_U32 netlink attribute @attr and narrow it into + * the u8 pointed to by @dst; do nothing if @attr is null. + * Bool pointed to by @mod is set to true if this function changed the value + * of *dst, otherwise it is left as is. + */ +static inline void ethnl_update_u8_u32(u8 *dst, const struct nlattr *attr, + bool *mod) +{ + u32 val; + + if (!attr) + return; + val = nla_get_u32(attr); + DEBUG_NET_WARN_ON_ONCE(val > U8_MAX); + if (*dst == val) + return; + + *dst = val; + *mod = true; +} + /** * ethnl_update_bool32() - update u32 used as bool from NLA_U8 attribute * @dst: value to update diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c index 53792f53f9229..ca6ed152c2be3 100644 --- a/net/ethtool/rss.c +++ b/net/ethtool/rss.c @@ -574,7 +574,7 @@ static const struct nla_policy ethnl_rss_flows_policy[] = { const struct nla_policy ethnl_rss_set_policy[ETHTOOL_A_RSS_FLOW_HASH + 1] = { [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32, }, - [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1), + [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_RANGE(NLA_U32, 1, U8_MAX), [ETHTOOL_A_RSS_INDIR] = { .type = NLA_BINARY, }, [ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1), [ETHTOOL_A_RSS_INPUT_XFRM] = @@ -855,7 +855,7 @@ ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info) indir_mod = !!tb[ETHTOOL_A_RSS_INDIR]; rxfh.hfunc = data.hfunc; - ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); + ethnl_update_u8_u32(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); if (rxfh.hfunc == data.hfunc) rxfh.hfunc = ETH_RSS_HASH_NO_CHANGE; @@ -864,7 +864,8 @@ ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info) goto exit_free_indir; rxfh.input_xfrm = data.input_xfrm; - ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod); + ethnl_update_u8_u32(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], + &mod); xfrm_sym = rxfh.input_xfrm || data.input_xfrm; if (rxfh.input_xfrm == data.input_xfrm) rxfh.input_xfrm = RXH_XFRM_NO_CHANGE; @@ -938,7 +939,7 @@ const struct ethnl_request_ops ethnl_rss_request_ops = { const struct nla_policy ethnl_rss_create_policy[ETHTOOL_A_RSS_INPUT_XFRM + 1] = { [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), [ETHTOOL_A_RSS_CONTEXT] = NLA_POLICY_MIN(NLA_U32, 1), - [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1), + [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_RANGE(NLA_U32, 1, U8_MAX), [ETHTOOL_A_RSS_INDIR] = NLA_POLICY_MIN(NLA_BINARY, 1), [ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1), [ETHTOOL_A_RSS_INPUT_XFRM] = @@ -1054,14 +1055,15 @@ int ethnl_rss_create_doit(struct sk_buff *skb, struct genl_info *info) goto exit_clean_data; indir_user_size = ret; - ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); + ethnl_update_u8_u32(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); ret = rss_set_prep_hkey(dev, info, &data, &rxfh, &mod); if (ret) goto exit_free_indir; rxfh.input_xfrm = RXH_XFRM_NO_CHANGE; - ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod); + ethnl_update_u8_u32(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], + &mod); ctx = ethtool_rxfh_ctx_alloc(ops, data.indir_size, data.hkey_size); if (!ctx) { diff --git a/net/handshake/genl.c b/net/handshake/genl.c index 4b20cd9cdd0e0..feac1ad063ee7 100644 --- a/net/handshake/genl.c +++ b/net/handshake/genl.c @@ -38,7 +38,7 @@ static const struct genl_split_ops handshake_nl_ops[] = { .doit = handshake_nl_done_doit, .policy = handshake_done_nl_policy, .maxattr = HANDSHAKE_A_DONE_REMOTE_AUTH, - .flags = GENL_CMD_CAP_DO, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, }; diff --git a/net/ieee802154/core.c b/net/ieee802154/core.c index 89b671b12600f..c0b8712018a16 100644 --- a/net/ieee802154/core.c +++ b/net/ieee802154/core.c @@ -228,36 +228,43 @@ int cfg802154_switch_netns(struct cfg802154_registered_device *rdev, continue; wpan_dev->netdev->netns_immutable = false; err = dev_change_net_namespace(wpan_dev->netdev, net, "wpan%d"); - if (err) + if (err) { + WARN_ON(err && err != -ENOMEM); break; + } wpan_dev->netdev->netns_immutable = true; } - if (err) { - /* failed -- clean up to old netns */ - net = wpan_phy_net(&rdev->wpan_phy); - - list_for_each_entry_continue_reverse(wpan_dev, - &rdev->wpan_dev_list, - list) { - if (!wpan_dev->netdev) - continue; - wpan_dev->netdev->netns_immutable = false; - err = dev_change_net_namespace(wpan_dev->netdev, net, - "wpan%d"); - WARN_ON(err); - wpan_dev->netdev->netns_immutable = true; - } + if (err) + goto errout; - return err; - } + err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev)); + WARN_ON(err && err != -ENOMEM); - wpan_phy_net_set(&rdev->wpan_phy, net); + if (err) + goto errout; - err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev)); - WARN_ON(err); + wpan_phy_net_set(&rdev->wpan_phy, net); return 0; + +errout: + /* failed -- clean up to old netns */ + net = wpan_phy_net(&rdev->wpan_phy); + + list_for_each_entry_continue_reverse(wpan_dev, + &rdev->wpan_dev_list, + list) { + if (!wpan_dev->netdev) + continue; + wpan_dev->netdev->netns_immutable = false; + err = dev_change_net_namespace(wpan_dev->netdev, net, + "wpan%d"); + WARN_ON(err && err != -ENOMEM); + wpan_dev->netdev->netns_immutable = true; + } + + return err; } void cfg802154_dev_free(struct cfg802154_registered_device *rdev) @@ -351,7 +358,7 @@ static void __net_exit cfg802154_pernet_exit(struct net *net) rtnl_lock(); list_for_each_entry(rdev, &cfg802154_rdev_list, list) { if (net_eq(wpan_phy_net(&rdev->wpan_phy), net)) - WARN_ON(cfg802154_switch_netns(rdev, &init_net)); + cfg802154_switch_netns(rdev, &init_net); } rtnl_unlock(); } diff --git a/net/ieee802154/header_ops.c b/net/ieee802154/header_ops.c index 41a556be10179..a9f0c8df5ae4a 100644 --- a/net/ieee802154/header_ops.c +++ b/net/ieee802154/header_ops.c @@ -173,10 +173,13 @@ ieee802154_hdr_get_addr(const u8 *buf, int mode, bool omit_pan, { int pos = 0; - addr->mode = mode; - - if (mode == IEEE802154_ADDR_NONE) + if (mode == IEEE802154_ADDR_NONE) { + memset(addr, 0, sizeof(*addr)); + addr->mode = IEEE802154_ADDR_NONE; return 0; + } + + addr->mode = mode; if (!omit_pan) { memcpy(&addr->pan_id, buf + pos, 2); diff --git a/net/ieee802154/ieee802154.h b/net/ieee802154/ieee802154.h index c5d91f78301ad..e765adc4b88f2 100644 --- a/net/ieee802154/ieee802154.h +++ b/net/ieee802154/ieee802154.h @@ -16,6 +16,15 @@ void ieee802154_nl_exit(void); .flags = GENL_ADMIN_PERM, \ } +#define IEEE802154_OP_RELAXED(_cmd, _func) \ + { \ + .cmd = _cmd, \ + .doit = _func, \ + .dumpit = NULL, \ + .flags = GENL_ADMIN_PERM, \ + .validate = GENL_DONT_VALIDATE_STRICT,\ + } + #define IEEE802154_DUMP(_cmd, _func, _dump) \ { \ .cmd = _cmd, \ @@ -23,6 +32,14 @@ void ieee802154_nl_exit(void); .dumpit = _dump, \ } +#define IEEE802154_DUMP_PRIV(_cmd, _func, _dump) \ + { \ + .cmd = _cmd, \ + .doit = _func, \ + .dumpit = _dump, \ + .flags = GENL_ADMIN_PERM, \ + } + struct genl_info; struct sk_buff *ieee802154_nl_create(int flags, u8 req); diff --git a/net/ieee802154/netlink.c b/net/ieee802154/netlink.c index 7d2de4ee6992b..cacad21347eca 100644 --- a/net/ieee802154/netlink.c +++ b/net/ieee802154/netlink.c @@ -98,24 +98,24 @@ static const struct genl_small_ops ieee802154_ops[] = { IEEE802154_OP(IEEE802154_SET_MACPARAMS, ieee802154_set_macparams), IEEE802154_OP(IEEE802154_LLSEC_GETPARAMS, ieee802154_llsec_getparams), IEEE802154_OP(IEEE802154_LLSEC_SETPARAMS, ieee802154_llsec_setparams), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_KEY, NULL, - ieee802154_llsec_dump_keys), - IEEE802154_OP(IEEE802154_LLSEC_ADD_KEY, ieee802154_llsec_add_key), - IEEE802154_OP(IEEE802154_LLSEC_DEL_KEY, ieee802154_llsec_del_key), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_DEV, NULL, - ieee802154_llsec_dump_devs), - IEEE802154_OP(IEEE802154_LLSEC_ADD_DEV, ieee802154_llsec_add_dev), - IEEE802154_OP(IEEE802154_LLSEC_DEL_DEV, ieee802154_llsec_del_dev), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_DEVKEY, NULL, - ieee802154_llsec_dump_devkeys), - IEEE802154_OP(IEEE802154_LLSEC_ADD_DEVKEY, ieee802154_llsec_add_devkey), - IEEE802154_OP(IEEE802154_LLSEC_DEL_DEVKEY, ieee802154_llsec_del_devkey), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_SECLEVEL, NULL, - ieee802154_llsec_dump_seclevels), - IEEE802154_OP(IEEE802154_LLSEC_ADD_SECLEVEL, - ieee802154_llsec_add_seclevel), - IEEE802154_OP(IEEE802154_LLSEC_DEL_SECLEVEL, - ieee802154_llsec_del_seclevel), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_KEY, NULL, + ieee802154_llsec_dump_keys), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_KEY, ieee802154_llsec_add_key), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_KEY, ieee802154_llsec_del_key), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_DEV, NULL, + ieee802154_llsec_dump_devs), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_DEV, ieee802154_llsec_add_dev), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_DEV, ieee802154_llsec_del_dev), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_DEVKEY, NULL, + ieee802154_llsec_dump_devkeys), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_DEVKEY, ieee802154_llsec_add_devkey), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_DEVKEY, ieee802154_llsec_del_devkey), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_SECLEVEL, NULL, + ieee802154_llsec_dump_seclevels), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_SECLEVEL, + ieee802154_llsec_add_seclevel), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_SECLEVEL, + ieee802154_llsec_del_seclevel), }; static const struct genl_multicast_group ieee802154_mcgrps[] = { diff --git a/net/ife/ife.c b/net/ife/ife.c index be05b690b9ef2..7a75947a31e31 100644 --- a/net/ife/ife.c +++ b/net/ife/ife.c @@ -79,7 +79,7 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) if (unlikely(ifehdrln < 2)) return NULL; - if (unlikely(!pskb_may_pull(skb, total_pull))) + if (unlikely(!pskb_may_pull(skb, total_pull + ETH_HLEN))) return NULL; ifehdr = (struct ifeheadr *)(skb->data + skb->dev->hard_header_len); diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 1308213791f19..dac543c1d6867 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -2172,10 +2172,14 @@ static int fib_leaf_notify(struct key_vector *l, struct fib_table *tb, if (fa->fa_slen == last_slen) continue; + if (!fib_info_hold_safe(fa->fa_info)) + continue; + last_slen = fa->fa_slen; err = call_fib_entry_notifier(nb, FIB_EVENT_ENTRY_REPLACE, l->key, KEYLENGTH - fa->fa_slen, fa, extack); + fib_info_put(fa->fa_info); if (err) return err; } diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c index 5bae3cf7fe769..6bed0e1dbe0e9 100644 --- a/net/ipv4/fou_core.c +++ b/net/ipv4/fou_core.c @@ -558,11 +558,8 @@ static int fou_add_to_port_list(struct net *net, struct fou *fou, static void fou_release(struct fou *fou) { - struct socket *sock = fou->sock; - list_del(&fou->list); - udp_tunnel_sock_release(sock); - + udp_tunnel_sock_release(fou->sock->sk); kfree_rcu(fou, rcu); } @@ -618,7 +615,7 @@ static int fou_create(struct net *net, struct fou_cfg *cfg, goto error; } - setup_udp_tunnel_sock(net, sock, &tunnel_cfg); + setup_udp_tunnel_sock(net, sk, &tunnel_cfg); sk->sk_allocation = GFP_ATOMIC; @@ -634,7 +631,7 @@ static int fou_create(struct net *net, struct fou_cfg *cfg, error: kfree(fou); if (sock) - udp_tunnel_sock_release(sock); + udp_tunnel_sock_release(sock->sk); return err; } diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index a9ad39064f3bb..f22fbc17f15b3 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -217,13 +217,18 @@ static void ip_sf_list_clear_all(struct ip_sf_list *psf) static void igmp_stop_timer(struct ip_mc_list *im) { + bool put = false; + spin_lock_bh(&im->lock); if (timer_delete(&im->timer)) - refcount_dec(&im->refcnt); - im->tm_running = 0; - im->reporter = 0; + put = true; + WRITE_ONCE(im->tm_running, 0); + WRITE_ONCE(im->reporter, 0); im->unsolicit_count = 0; spin_unlock_bh(&im->lock); + + if (put) + ip_ma_put(im); } /* It must be called with locked im->lock */ @@ -231,7 +236,7 @@ static void igmp_start_timer(struct ip_mc_list *im, int max_delay) { int tv = get_random_u32_below(max_delay); - im->tm_running = 1; + WRITE_ONCE(im->tm_running, 1); if (refcount_inc_not_zero(&im->refcnt)) { if (mod_timer(&im->timer, jiffies + tv + 2)) ip_ma_put(im); @@ -248,33 +253,42 @@ static void igmp_gq_start_timer(struct in_device *in_dev) return; in_dev->mr_gq_running = 1; - if (!mod_timer(&in_dev->mr_gq_timer, exp)) - in_dev_hold(in_dev); + if (in_dev_hold_safe(in_dev)) { + if (mod_timer(&in_dev->mr_gq_timer, exp)) + in_dev_put(in_dev); + } } static void igmp_ifc_start_timer(struct in_device *in_dev, int delay) { - int tv = get_random_u32_below(delay); + if (in_dev_hold_safe(in_dev)) { + int tv = get_random_u32_below(delay); - if (!mod_timer(&in_dev->mr_ifc_timer, jiffies+tv+2)) - in_dev_hold(in_dev); + if (mod_timer(&in_dev->mr_ifc_timer, jiffies + tv + 2)) + in_dev_put(in_dev); + } } static void igmp_mod_timer(struct ip_mc_list *im, int max_delay) { + bool put = false; + spin_lock_bh(&im->lock); im->unsolicit_count = 0; if (timer_delete(&im->timer)) { if ((long)(im->timer.expires-jiffies) < max_delay) { add_timer(&im->timer); - im->tm_running = 1; + WRITE_ONCE(im->tm_running, 1); spin_unlock_bh(&im->lock); return; } - refcount_dec(&im->refcnt); + put = true; } igmp_start_timer(im, max_delay); spin_unlock_bh(&im->lock); + + if (put) + ip_ma_put(im); } @@ -857,12 +871,12 @@ static void igmp_timer_expire(struct timer_list *t) struct in_device *in_dev = im->interface; spin_lock(&im->lock); - im->tm_running = 0; + WRITE_ONCE(im->tm_running, 0); if (im->unsolicit_count && --im->unsolicit_count) igmp_start_timer(im, unsolicited_report_interval(in_dev)); - im->reporter = 1; + WRITE_ONCE(im->reporter, 1); spin_unlock(&im->lock); if (IGMP_V1_SEEN(in_dev)) @@ -1325,7 +1339,7 @@ static void __igmp_group_dropped(struct ip_mc_list *im, gfp_t gfp) !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) return; - reporter = im->reporter; + reporter = READ_ONCE(im->reporter); igmp_stop_timer(im); if (!in_dev->dead) { @@ -1541,7 +1555,7 @@ static void ____ip_mc_inc_group(struct in_device *in_dev, __be32 addr, } if (im) { - im->users++; + WRITE_ONCE(im->users, im->users + 1); ip_mc_add_src(in_dev, &addr, mode, 0, NULL, 0); goto out; } @@ -1550,7 +1564,7 @@ static void ____ip_mc_inc_group(struct in_device *in_dev, __be32 addr, if (!im) goto out; - im->users = 1; + WRITE_ONCE(im->users, 1); im->interface = in_dev; in_dev_hold(in_dev); im->multiaddr = addr; @@ -1784,7 +1798,10 @@ void __ip_mc_dec_group(struct in_device *in_dev, __be32 addr, gfp_t gfp) (i = rtnl_dereference(*ip)) != NULL; ip = &i->next_rcu) { if (i->multiaddr == addr) { - if (--i->users == 0) { + int new_users = i->users - 1; + + WRITE_ONCE(i->users, new_users); + if (new_users == 0) { ip_mc_hash_remove(in_dev, i); *ip = i->next_rcu; in_dev->mc_count--; @@ -1918,6 +1935,7 @@ void ip_mc_destroy_dev(struct in_device *in_dev) #endif while ((i = rtnl_dereference(in_dev->mc_list)) != NULL) { + ip_mc_hash_remove(in_dev, i); in_dev->mc_list = i->next_rcu; in_dev->mc_count--; ip_mc_clear_src(i); @@ -2959,6 +2977,7 @@ static int igmp_mc_seq_show(struct seq_file *seq, void *v) struct ip_mc_list *im = v; struct igmp_mc_iter_state *state = igmp_mc_seq_private(seq); char *querier; + int tm_running; long delta; #ifdef CONFIG_IP_MULTICAST @@ -2974,13 +2993,14 @@ static int igmp_mc_seq_show(struct seq_file *seq, void *v) state->dev->ifindex, state->dev->name, state->in_dev->mc_count, querier); } - delta = im->timer.expires - jiffies; + tm_running = READ_ONCE(im->tm_running); + delta = READ_ONCE(im->timer.expires) - jiffies; seq_printf(seq, "\t\t\t\t%08X %5d %d:%08lX\t\t%d\n", - im->multiaddr, im->users, - im->tm_running, - im->tm_running ? jiffies_delta_to_clock_t(delta) : 0, - im->reporter); + im->multiaddr, READ_ONCE(im->users), + tm_running, + tm_running ? jiffies_delta_to_clock_t(delta) : 0, + READ_ONCE(im->reporter)); } return 0; } diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 5b934ce8d98a8..690f7fb3f029e 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -1285,6 +1285,7 @@ EXPORT_SYMBOL(inet_csk_destroy_sock); void inet_csk_prepare_for_destroy_sock(struct sock *sk) { /* The below has to be done to allow calling inet_csk_destroy_sock */ + tcp_clear_sock_ops_cb_flags(sk); sock_set_flag(sk, SOCK_DEAD); tcp_orphan_count_inc(); } diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 169e2921a851d..0ebed1438f6c5 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -1457,6 +1457,9 @@ static int ipgre_changelink(struct net_device *dev, struct nlattr *tb[], __u32 fwmark = t->fwmark; int err; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; @@ -1486,6 +1489,9 @@ static int erspan_changelink(struct net_device *dev, struct nlattr *tb[], __u32 fwmark = t->fwmark; int err; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 5bcd73cbdb41c..ec790bad1679a 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1117,8 +1117,8 @@ static int __ip_append_data(struct sock *sk, !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; else { - alloclen = fragheaderlen + transhdrlen; - pagedlen = datalen - transhdrlen; + alloclen = fragheaderlen + transhdrlen + fraggap; + pagedlen = datalen - transhdrlen - fraggap; } alloclen += alloc_extra; @@ -1165,9 +1165,6 @@ static int __ip_append_data(struct sock *sk, } copy = datalen - transhdrlen - fraggap - pagedlen; - /* [!] NOTE: copy will be negative if pagedlen>0 - * because then the equation reduces to -fraggap. - */ if (copy > 0 && INDIRECT_CALL_1(getfrag, ip_generic_getfrag, from, data + transhdrlen, offset, diff --git a/net/ipv4/ip_vti.c b/net/ipv4/ip_vti.c index 95b6bb78fcd27..3b80929994a00 100644 --- a/net/ipv4/ip_vti.c +++ b/net/ipv4/ip_vti.c @@ -596,6 +596,9 @@ static int vti_changelink(struct net_device *dev, struct nlattr *tb[], struct ip_tunnel_parm_kern p; __u32 fwmark = t->fwmark; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + vti_netlink_parms(data, &p, &fwmark); return ip_tunnel_changelink(dev, tb, &p, fwmark); } diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index ff95b1b9908e9..e7378569bd5b6 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -494,6 +494,9 @@ static int ipip_changelink(struct net_device *dev, struct nlattr *tb[], bool collect_md; __u32 fwmark = t->fwmark; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { int err = ip_tunnel_encap_setup(t, &ipencap); diff --git a/net/ipv4/netfilter/nf_nat_h323.c b/net/ipv4/netfilter/nf_nat_h323.c index 10e1b0837731b..183e8a3ff2bab 100644 --- a/net/ipv4/netfilter/nf_nat_h323.c +++ b/net/ipv4/netfilter/nf_nat_h323.c @@ -100,6 +100,9 @@ static int set_sig_addr(struct sk_buff *skb, struct nf_conn *ct, __be16 port; union nf_inet_addr addr; + if (!info) + return -1; + for (i = 0; i < count; i++) { if (get_h225_addr(ct, *data, &taddr[i], &addr, &port)) { if (addr.ip == ct->tuplehash[dir].tuple.src.u3.ip && @@ -184,6 +187,9 @@ static int nat_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, int i; u_int16_t nated_port; + if (!info) + return -1; + /* Set expectations for NAT */ rtp_exp->saved_proto.udp.port = rtp_exp->tuple.dst.u.udp.port; rtp_exp->expectfn = nf_nat_follow_master; @@ -325,6 +331,9 @@ static int nat_h245(struct sk_buff *skb, struct nf_conn *ct, int dir = CTINFO2DIR(ctinfo); u_int16_t nated_port = ntohs(port); + if (!info) + return -1; + /* Set expectations for NAT */ exp->saved_proto.tcp.port = exp->tuple.dst.u.tcp.port; exp->expectfn = nf_nat_follow_master; @@ -404,6 +413,9 @@ static int nat_q931(struct sk_buff *skb, struct nf_conn *ct, u_int16_t nated_port = ntohs(port); union nf_inet_addr addr; + if (!info) + return -1; + /* Set expectations for NAT */ exp->saved_proto.tcp.port = exp->tuple.dst.u.tcp.port; exp->expectfn = ip_nat_q931_expect; diff --git a/net/ipv4/netfilter/nf_nat_pptp.c b/net/ipv4/netfilter/nf_nat_pptp.c index fab357cc85594..fed5249001a49 100644 --- a/net/ipv4/netfilter/nf_nat_pptp.c +++ b/net/ipv4/netfilter/nf_nat_pptp.c @@ -53,11 +53,13 @@ static void pptp_nat_expected(struct nf_conn *ct, struct nf_conn_nat *nat; nat = nf_ct_nat_ext_add(ct); - if (WARN_ON_ONCE(!nat)) + if (!nat) return; nat_pptp_info = &nat->help.nat_pptp_info; ct_pptp_info = nfct_help_data(master); + if (!ct_pptp_info) + return; /* And here goes the grand finale of corrosion... */ if (exp->dir == IP_CT_DIR_ORIGINAL) { @@ -132,11 +134,13 @@ pptp_outbound_pkt(struct sk_buff *skb, __be16 new_callid; unsigned int cid_off; - if (WARN_ON_ONCE(!nat)) + if (!nat) return NF_DROP; nat_pptp_info = &nat->help.nat_pptp_info; ct_pptp_info = nfct_help_data(ct); + if (!ct_pptp_info) + return NF_DROP; new_callid = ct_pptp_info->pns_call_id; @@ -204,11 +208,13 @@ pptp_exp_gre(struct nf_conntrack_expect *expect_orig, struct nf_ct_pptp_master *ct_pptp_info; struct nf_nat_pptp *nat_pptp_info; - if (WARN_ON_ONCE(!nat)) + if (!nat) return; nat_pptp_info = &nat->help.nat_pptp_info; ct_pptp_info = nfct_help_data(ct); + if (!ct_pptp_info) + return; /* save original PAC call ID in nat_info */ nat_pptp_info->pac_call_id = ct_pptp_info->pac_call_id; @@ -241,7 +247,7 @@ pptp_inbound_pkt(struct sk_buff *skb, __be16 new_pcid; unsigned int pcid_off; - if (WARN_ON_ONCE(!nat)) + if (!nat) return NF_DROP; nat_pptp_info = &nat->help.nat_pptp_info; diff --git a/net/ipv4/netfilter/nf_nat_snmp_basic_main.c b/net/ipv4/netfilter/nf_nat_snmp_basic_main.c index 717b726504feb..0ede138dfd29e 100644 --- a/net/ipv4/netfilter/nf_nat_snmp_basic_main.c +++ b/net/ipv4/netfilter/nf_nat_snmp_basic_main.c @@ -202,29 +202,34 @@ static const struct nf_conntrack_expect_policy snmp_exp_policy = { .timeout = 180, }; -static struct nf_conntrack_helper snmp_trap_helper __read_mostly = { - .me = THIS_MODULE, - .help = help, - .expect_policy = &snmp_exp_policy, - .name = "snmp_trap", - .tuple.src.l3num = AF_INET, - .tuple.src.u.udp.port = cpu_to_be16(SNMP_TRAP_PORT), - .tuple.dst.protonum = IPPROTO_UDP, -}; +static struct nf_conntrack_helper snmp_trap_helper __read_mostly; +static struct nf_conntrack_helper *snmp_trap_helper_ptr __read_mostly; static int __init nf_nat_snmp_basic_init(void) { + int err; + BUG_ON(nf_nat_snmp_hook != NULL); RCU_INIT_POINTER(nf_nat_snmp_hook, help); - return nf_conntrack_helper_register(&snmp_trap_helper); + nf_ct_helper_init(&snmp_trap_helper, AF_INET, IPPROTO_UDP, + "snmp_trap", SNMP_TRAP_PORT, SNMP_TRAP_PORT, SNMP_TRAP_PORT, + &snmp_exp_policy, 0, help, NULL, THIS_MODULE); + + err = nf_conntrack_helper_register(&snmp_trap_helper, &snmp_trap_helper_ptr); + if (err < 0) { + RCU_INIT_POINTER(nf_nat_snmp_hook, NULL); + return err; + } + + return 0; } static void __exit nf_nat_snmp_basic_fini(void) { RCU_INIT_POINTER(nf_nat_snmp_hook, NULL); synchronize_rcu(); - nf_conntrack_helper_unregister(&snmp_trap_helper); + nf_conntrack_helper_unregister(snmp_trap_helper_ptr); } module_init(nf_nat_snmp_basic_init); diff --git a/net/ipv4/netfilter/nf_reject_ipv4.c b/net/ipv4/netfilter/nf_reject_ipv4.c index fecf6621f679f..4626dc46808f3 100644 --- a/net/ipv4/netfilter/nf_reject_ipv4.c +++ b/net/ipv4/netfilter/nf_reject_ipv4.c @@ -89,7 +89,7 @@ static bool nf_skb_is_icmp_unreach(const struct sk_buff *skb) if (iph->protocol != IPPROTO_ICMP) return false; - thoff = skb_network_offset(skb) + sizeof(*iph); + thoff = skb_network_offset(skb) + ip_hdrlen(skb); tp = skb_header_pointer(skb, thoff + offsetof(struct icmphdr, type), diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index c0e85cc171aec..ca1180dba1dea 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -1058,7 +1058,9 @@ static struct ctl_table ipv4_net_table[] = { .data = &init_net.ipv4.sysctl_tcp_reordering, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &init_net.ipv4.sysctl_tcp_max_reordering, }, { .procname = "tcp_retries1", @@ -1293,7 +1295,8 @@ static struct ctl_table ipv4_net_table[] = { .data = &init_net.ipv4.sysctl_tcp_max_reordering, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, }, { .procname = "tcp_dsack", @@ -1676,6 +1679,9 @@ static __net_init int ipv4_sysctl_init_net(struct net *net) */ table[i].mode &= ~0222; } + if (table[i].extra2 >= (void *)&init_net.ipv4 && + table[i].extra2 < (void *)(&init_net.ipv4 + 1)) + table[i].extra2 += (void *)net - (void *)&init_net; } } diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c index 0a4b38b315fed..69c11bfd8f2ba 100644 --- a/net/ipv4/tcp_ao.c +++ b/net/ipv4/tcp_ao.c @@ -270,8 +270,9 @@ static void tcp_ao_key_free_rcu(struct rcu_head *head) kfree_sensitive(key); } -static void tcp_ao_info_free(struct tcp_ao_info *ao) +static void tcp_ao_info_free_rcu(struct rcu_head *head) { + struct tcp_ao_info *ao = container_of(head, struct tcp_ao_info, rcu); struct tcp_ao_key *key; struct hlist_node *n; @@ -311,7 +312,7 @@ void tcp_ao_destroy_sock(struct sock *sk, bool twsk) if (!twsk) tcp_ao_sk_omem_free(sk, ao); - tcp_ao_info_free(ao); + call_rcu(&ao->rcu, tcp_ao_info_free_rcu); } void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp) @@ -1776,6 +1777,10 @@ static int tcp_ao_delete_key(struct sock *sk, struct tcp_ao_info *ao_info, * them and we can just free all resources in RCU fashion. */ if (del_async) { + if (ao_info->current_key == key) + WRITE_ONCE(ao_info->current_key, NULL); + if (ao_info->rnext_key == key) + WRITE_ONCE(ao_info->rnext_key, NULL); atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc); call_rcu(&key->rcu, tcp_ao_key_free_rcu); return 0; diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index aec7805b1d376..82378a2bfd1e1 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* Bottleneck Bandwidth and RTT (BBR) congestion control * * BBR congestion control computes the sending rate based on the delivery diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index fdc81150ff6cf..bfdabad01fa76 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1467,9 +1467,9 @@ void tcp_clear_md5_list(struct sock *sk) md5sig = rcu_dereference_protected(tp->md5sig_info, 1); hlist_for_each_entry_safe(key, n, &md5sig->head, node) { - hlist_del(&key->node); + hlist_del_rcu(&key->node); atomic_sub(sizeof(*key), &sk->sk_omem_alloc); - kfree(key); + kfree_rcu(key, rcu); } } diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 6e4bb411dc041..a88c7eb72b965 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -2687,7 +2687,7 @@ static int tcp_mtu_probe(struct sock *sk) struct sk_buff *skb, *nskb, *next; struct net *net = sock_net(sk); int probe_size; - int size_needed; + u64 size_needed; int copy, len; int mss_now; int interval; @@ -2711,7 +2711,7 @@ static int tcp_mtu_probe(struct sock *sk) mss_now = tcp_current_mss(sk); probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high + icsk->icsk_mtup.search_low) >> 1); - size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache; + size_needed = probe_size + (tp->reordering + 1) * (u64)tp->mss_cache; interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low; /* When misfortune happens, we are reprobing actively, * and then reprobe timer has expired. We stick with current @@ -4328,9 +4328,13 @@ int tcp_connect(struct sock *sk) if (needs_md5) { tcp_ao_destroy_sock(sk, false); } else if (needs_ao) { + struct tcp_md5sig_info *md5sig; + tcp_clear_md5_list(sk); - kfree(rcu_replace_pointer(tp->md5sig_info, NULL, - lockdep_sock_is_held(sk))); + md5sig = rcu_replace_pointer(tp->md5sig_info, NULL, + lockdep_sock_is_held(sk)); + kfree_rcu(md5sig, rcu); + static_branch_slow_dec_deferred(&tcp_md5_needed); } } #endif diff --git a/net/ipv4/udp_bpf.c b/net/ipv4/udp_bpf.c index 9f33b07b14813..ad57c4c9eaab6 100644 --- a/net/ipv4/udp_bpf.c +++ b/net/ipv4/udp_bpf.c @@ -50,7 +50,9 @@ static int udp_msg_wait_data(struct sock *sk, struct sk_psock *psock, sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); ret = udp_msg_has_data(sk, psock); if (!ret) { + release_sock(sk); wait_woken(&wait, TASK_INTERRUPTIBLE, timeo); + lock_sock(sk); ret = udp_msg_has_data(sk, psock); } sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); @@ -79,6 +81,7 @@ static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, goto out; } + lock_sock(sk); msg_bytes_ready: copied = sk_msg_recvmsg(sk, psock, msg, len, flags); if (!copied) { @@ -90,11 +93,17 @@ static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, if (data) { if (psock_has_data(psock)) goto msg_bytes_ready; + + release_sock(sk); + ret = sk_udp_recvmsg(sk, msg, len, flags); goto out; } copied = -EAGAIN; } + + release_sock(sk); + ret = copied; out: sk_psock_put(sk, psock); diff --git a/net/ipv4/udp_tunnel_core.c b/net/ipv4/udp_tunnel_core.c index b1f667c52cb21..3090b4745d470 100644 --- a/net/ipv4/udp_tunnel_core.c +++ b/net/ipv4/udp_tunnel_core.c @@ -68,11 +68,9 @@ static bool sk_saddr_any(struct sock *sk) #endif } -void setup_udp_tunnel_sock(struct net *net, struct socket *sock, +void setup_udp_tunnel_sock(struct net *net, struct sock *sk, struct udp_tunnel_sock_cfg *cfg) { - struct sock *sk = sock->sk; - /* Disable multicast loopback */ inet_clear_bit(MC_LOOP, sk); @@ -195,9 +193,11 @@ void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb } EXPORT_SYMBOL_GPL(udp_tunnel_xmit_skb); -void udp_tunnel_sock_release(struct socket *sock) +void udp_tunnel_sock_release(struct sock *sk) { - rcu_assign_sk_user_data(sock->sk, NULL); + struct socket *sock = sk->sk_socket; + + rcu_assign_sk_user_data(sk, NULL); synchronize_rcu(); kernel_sock_shutdown(sock, SHUT_RDWR); sock_release(sock); diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c index 9944ed923ddfd..3b32a0afa9798 100644 --- a/net/ipv4/udp_tunnel_nic.c +++ b/net/ipv4/udp_tunnel_nic.c @@ -301,7 +301,7 @@ __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) static void udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) { - if (!utn->need_sync) + if (!utn->need_sync || utn->work_pending) return; queue_work(udp_tunnel_nic_workqueue, &utn->work); diff --git a/net/ipv4/xfrm4_input.c b/net/ipv4/xfrm4_input.c index c2eac844bcdb4..f6f2a8ef3f884 100644 --- a/net/ipv4/xfrm4_input.c +++ b/net/ipv4/xfrm4_input.c @@ -76,8 +76,6 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async) NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING, dev_net(dev), NULL, skb, dev, NULL, xfrm4_rcv_encap_finish); - if (async) - dev_put(dev); return 0; } diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index c9e5d3e48ab98..defa4277e783f 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -913,7 +913,7 @@ static int addrconf_fixup_forwarding(const struct ctl_table *table, int *p, int if (newf) rt6_purge_dflt_routers(net); - return 1; + return 0; } static void addrconf_linkdown_change(struct net *net, __s32 newf) @@ -955,11 +955,7 @@ static int addrconf_fixup_linkdown(const struct ctl_table *table, int *p, int ne NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN, NETCONFA_IFINDEX_DEFAULT, net->ipv6.devconf_dflt); - rtnl_net_unlock(net); - return 0; - } - - if (p == &net->ipv6.devconf_all->ignore_routes_with_linkdown) { + } else if (p == &net->ipv6.devconf_all->ignore_routes_with_linkdown) { WRITE_ONCE(net->ipv6.devconf_dflt->ignore_routes_with_linkdown, newf); addrconf_linkdown_change(net, newf); if ((!newf) ^ (!old)) @@ -968,11 +964,21 @@ static int addrconf_fixup_linkdown(const struct ctl_table *table, int *p, int ne NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN, NETCONFA_IFINDEX_ALL, net->ipv6.devconf_all); + } else { + if (!newf ^ !old) { + struct inet6_dev *idev = table->extra1; + + inet6_netconf_notify_devconf(net, + RTM_NEWNETCONF, + NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN, + idev->dev->ifindex, + &idev->cnf); + } } rtnl_net_unlock(net); - return 1; + return 0; } #endif @@ -2168,16 +2174,18 @@ void addrconf_dad_failure(struct sk_buff *skb, struct inet6_ifaddr *ifp) struct net *net = dev_net(idev->dev); int max_addresses; - if (addrconf_dad_end(ifp)) { + spin_lock_bh(&ifp->lock); + + if (ifp->state != INET6_IFADDR_STATE_DAD) { + spin_unlock_bh(&ifp->lock); in6_ifa_put(ifp); return; } + ifp->state = INET6_IFADDR_STATE_POSTDAD; net_info_ratelimited("%s: IPv6 duplicate address %pI6c used by %pM detected!\n", ifp->idev->dev->name, &ifp->addr, eth_hdr(skb)->h_source); - spin_lock_bh(&ifp->lock); - if (ifp->flags & IFA_F_STABLE_PRIVACY) { struct in6_addr new_addr; struct inet6_ifaddr *ifp2; @@ -2225,6 +2233,11 @@ void addrconf_dad_failure(struct sk_buff *skb, struct inet6_ifaddr *ifp) in6_ifa_put(ifp2); lock_errdad: spin_lock_bh(&ifp->lock); + if (ifp->state != INET6_IFADDR_STATE_POSTDAD) { + spin_unlock_bh(&ifp->lock); + in6_ifa_put(ifp); + return; + } } errdad: @@ -6355,6 +6368,8 @@ static int addrconf_sysctl_forward(const struct ctl_table *ctl, int write, lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write) ret = addrconf_fixup_forwarding(ctl, valp, val); @@ -6452,6 +6467,8 @@ static int addrconf_sysctl_disable(const struct ctl_table *ctl, int write, lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write) ret = addrconf_disable_ipv6(ctl, valp, val); @@ -6463,20 +6480,19 @@ static int addrconf_sysctl_disable(const struct ctl_table *ctl, int write, static int addrconf_sysctl_proxy_ndp(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos) { + struct net *net = ctl->extra2; int *valp = ctl->data; - int ret; int old, new; + int ret; + + if (write && !rtnl_net_trylock(net)) + return restart_syscall(); old = *valp; ret = proc_dointvec(ctl, write, buffer, lenp, ppos); new = *valp; if (write && old != new) { - struct net *net = ctl->extra2; - - if (!rtnl_net_trylock(net)) - return restart_syscall(); - if (valp == &net->ipv6.devconf_dflt->proxy_ndp) { inet6_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_PROXY_NEIGH, @@ -6495,8 +6511,9 @@ static int addrconf_sysctl_proxy_ndp(const struct ctl_table *ctl, int write, idev->dev->ifindex, &idev->cnf); } - rtnl_net_unlock(net); } + if (write) + rtnl_net_unlock(net); return ret; } @@ -6654,6 +6671,8 @@ int addrconf_sysctl_ignore_routes_with_linkdown(const struct ctl_table *ctl, lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write) ret = addrconf_fixup_linkdown(ctl, valp, val); @@ -6748,6 +6767,8 @@ static int addrconf_sysctl_disable_policy(const struct ctl_table *ctl, int write lctl = *ctl; lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write && (*valp != val)) ret = addrconf_disable_policy(ctl, valp, val); diff --git a/net/ipv6/ioam6_iptunnel.c b/net/ipv6/ioam6_iptunnel.c index b9f6d892a566c..cfb2c41634a0b 100644 --- a/net/ipv6/ioam6_iptunnel.c +++ b/net/ipv6/ioam6_iptunnel.c @@ -35,7 +35,7 @@ struct ioam6_lwt_freq { }; struct ioam6_lwt { - struct dst_entry null_dst; + struct rt6_info null_rt; struct dst_cache cache; struct ioam6_lwt_freq freq; atomic_t pkt_cnt; @@ -176,7 +176,7 @@ static int ioam6_build_state(struct net *net, struct nlattr *nla, * it is stored in the cache. Then, +1/-1 each time we read the cache * and release it. Long story short, we're fine. */ - dst_init(&ilwt->null_dst, NULL, NULL, DST_OBSOLETE_NONE, DST_NOCOUNT); + dst_init(&ilwt->null_rt.dst, NULL, NULL, DST_OBSOLETE_NONE, DST_NOCOUNT); atomic_set(&ilwt->pkt_cnt, 0); ilwt->freq.k = freq_k; @@ -360,7 +360,7 @@ static int ioam6_output(struct net *net, struct sock *sk, struct sk_buff *skb) /* This is how we notify that the destination does not change after * transformation and that we need to use orig_dst instead of the cache */ - if (dst == &ilwt->null_dst) { + if (dst == &ilwt->null_rt.dst) { dst_release(dst); dst = orig_dst; @@ -429,7 +429,7 @@ static int ioam6_output(struct net *net, struct sock *sk, struct sk_buff *skb) local_bh_disable(); if (orig_dst->lwtstate == dst->lwtstate) dst_cache_set_ip6(&ilwt->cache, - &ilwt->null_dst, &fl6.saddr); + &ilwt->null_rt.dst, &fl6.saddr); else dst_cache_set_ip6(&ilwt->cache, dst, &fl6.saddr); local_bh_enable(); diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index b897b3c5023b2..414fc3c567360 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -637,12 +637,12 @@ static int inet6_dump_fib(struct sk_buff *skb, struct netlink_callback *cb) }; const struct nlmsghdr *nlh = cb->nlh; struct net *net = sock_net(skb->sk); - unsigned int e = 0, s_e; struct hlist_head *head; struct fib6_walker *w; struct fib6_table *tb; unsigned int h, s_h; int err = 0; + u32 s_id; rcu_read_lock(); if (cb->strict_check) { @@ -702,23 +702,22 @@ static int inet6_dump_fib(struct sk_buff *skb, struct netlink_callback *cb) } s_h = cb->args[0]; - s_e = cb->args[1]; + s_id = cb->args[1]; - for (h = s_h; h < FIB6_TABLE_HASHSZ; h++, s_e = 0) { - e = 0; + for (h = s_h; h < FIB6_TABLE_HASHSZ; h++, s_id = 0) { head = &net->ipv6.fib_table_hash[h]; hlist_for_each_entry_rcu(tb, head, tb6_hlist) { - if (e < s_e) - goto next; + if (s_id && tb->tb6_id != s_id) + continue; + + s_id = 0; + cb->args[1] = tb->tb6_id; err = fib6_dump_table(tb, skb, cb); if (err != 0) goto out; -next: - e++; } } out: - cb->args[1] = e; cb->args[0] = h; unlock: diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 365b4059eb203..8ebc99a299c95 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -2047,6 +2047,9 @@ static int ip6gre_changelink(struct net_device *dev, struct nlattr *tb[], struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id); struct __ip6_tnl_parm p; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + t = ip6gre_changelink_common(dev, tb, data, &p, extack); if (IS_ERR(t)) return PTR_ERR(t); @@ -2266,6 +2269,9 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[], struct __ip6_tnl_parm p; struct ip6gre_net *ign; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + ign = net_generic(t->net, ip6gre_net_id); t = ip6gre_changelink_common(dev, tb, data, &p, extack); if (IS_ERR(t)) diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index c14adcdd43960..b5d9c57fe439f 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1668,8 +1668,8 @@ static int __ip6_append_data(struct sock *sk, !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; else { - alloclen = fragheaderlen + transhdrlen; - pagedlen = datalen - transhdrlen; + alloclen = fragheaderlen + transhdrlen + fraggap; + pagedlen = datalen - transhdrlen - fraggap; } alloclen += alloc_extra; @@ -1684,10 +1684,7 @@ static int __ip6_append_data(struct sock *sk, fraglen = datalen + fragheaderlen; copy = datalen - transhdrlen - fraggap - pagedlen; - /* [!] NOTE: copy may be negative if pagedlen>0 - * because then the equation may reduces to -fraggap. - */ - if (copy < 0 && !(flags & MSG_SPLICE_PAGES)) { + if (copy < 0) { err = -EINVAL; goto error; } diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 9d1037ac082f6..95da1f087294e 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1850,6 +1850,13 @@ static int ip6_tnl_fill_forward_path(struct net_device_path_ctx *ctx, struct dst_entry *dst; int err; + if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT)) { + /* encaplimit option is currently not supported is + * sw-acceleration path. + */ + return -EOPNOTSUPP; + } + dst = ip6_route_output(dev_net(ctx->dev), NULL, &fl6); if (!dst->error) { path->type = DEV_PATH_TUN; @@ -2102,6 +2109,9 @@ static int ip6_tnl_changelink(struct net_device *dev, struct nlattr *tb[], struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id); struct ip_tunnel_encap ipencap; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + if (dev == ip6n->fb_tnl_dev) { if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { /* iproute2 always sets TUNNEL_ENCAP_FLAG_CSUM6, so diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c index d871cab6938d3..ab94b3a4ba9c3 100644 --- a/net/ipv6/ip6_vti.c +++ b/net/ipv6/ip6_vti.c @@ -1046,6 +1046,9 @@ static int vti6_changelink(struct net_device *dev, struct nlattr *tb[], struct __ip6_tnl_parm p; struct vti6_net *ip6n; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + ip6n = net_generic(net, vti6_net_id); if (dev == ip6n->fb_tnl_dev) return -EINVAL; diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index d9b855d5191bf..2a5e76a38002a 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -1080,8 +1080,10 @@ static void mld_gq_start_work(struct inet6_dev *idev) mc_assert_locked(idev); idev->mc_gq_running = 1; - if (!mod_delayed_work(mld_wq, &idev->mc_gq_work, tv + 2)) - in6_dev_hold(idev); + if (in6_dev_hold_safe(idev)) { + if (mod_delayed_work(mld_wq, &idev->mc_gq_work, tv + 2)) + in6_dev_put(idev); + } } static void mld_gq_stop_work(struct inet6_dev *idev) @@ -1099,8 +1101,10 @@ static void mld_ifc_start_work(struct inet6_dev *idev, unsigned long delay) mc_assert_locked(idev); - if (!mod_delayed_work(mld_wq, &idev->mc_ifc_work, tv + 2)) - in6_dev_hold(idev); + if (in6_dev_hold_safe(idev)) { + if (mod_delayed_work(mld_wq, &idev->mc_ifc_work, tv + 2)) + in6_dev_put(idev); + } } static void mld_ifc_stop_work(struct inet6_dev *idev) @@ -1118,8 +1122,10 @@ static void mld_dad_start_work(struct inet6_dev *idev, unsigned long delay) mc_assert_locked(idev); - if (!mod_delayed_work(mld_wq, &idev->mc_dad_work, tv + 2)) - in6_dev_hold(idev); + if (in6_dev_hold_safe(idev)) { + if (mod_delayed_work(mld_wq, &idev->mc_dad_work, tv + 2)) + in6_dev_put(idev); + } } static void mld_dad_stop_work(struct inet6_dev *idev) @@ -1405,18 +1411,23 @@ static void mld_process_v2(struct inet6_dev *idev, struct mld2_query *mld, void igmp6_event_query(struct sk_buff *skb) { struct inet6_dev *idev = __in6_dev_get(skb->dev); + bool put = false; if (!idev || idev->dead) goto out; spin_lock_bh(&idev->mc_query_lock); - if (skb_queue_len(&idev->mc_query_queue) < MLD_MAX_SKBS) { + if (skb_queue_len(&idev->mc_query_queue) < MLD_MAX_SKBS && + in6_dev_hold_safe(idev)) { __skb_queue_tail(&idev->mc_query_queue, skb); - if (!mod_delayed_work(mld_wq, &idev->mc_query_work, 0)) - in6_dev_hold(idev); + if (mod_delayed_work(mld_wq, &idev->mc_query_work, 0)) + put = true; skb = NULL; } spin_unlock_bh(&idev->mc_query_lock); + + if (put) + in6_dev_put(idev); out: kfree_skb(skb); } @@ -1573,18 +1584,23 @@ static void mld_query_work(struct work_struct *work) void igmp6_event_report(struct sk_buff *skb) { struct inet6_dev *idev = __in6_dev_get(skb->dev); + bool put = false; if (!idev || idev->dead) goto out; spin_lock_bh(&idev->mc_report_lock); - if (skb_queue_len(&idev->mc_report_queue) < MLD_MAX_SKBS) { + if (skb_queue_len(&idev->mc_report_queue) < MLD_MAX_SKBS && + in6_dev_hold_safe(idev)) { __skb_queue_tail(&idev->mc_report_queue, skb); - if (!mod_delayed_work(mld_wq, &idev->mc_report_work, 0)) - in6_dev_hold(idev); + if (mod_delayed_work(mld_wq, &idev->mc_report_work, 0)) + put = true; skb = NULL; } spin_unlock_bh(&idev->mc_report_lock); + + if (put) + in6_dev_put(idev); out: kfree_skb(skb); } diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index e7ad13c5bd267..f867ec8d3d905 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -967,10 +967,8 @@ static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb) return reason; } -static int accept_untracked_na(struct net_device *dev, struct in6_addr *saddr) +static int accept_untracked_na(struct inet6_dev *idev, struct in6_addr *saddr) { - struct inet6_dev *idev = __in6_dev_get(dev); - switch (READ_ONCE(idev->cnf.accept_untracked_na)) { case 0: /* Don't accept untracked na (absent in neighbor cache) */ return 0; @@ -980,7 +978,7 @@ static int accept_untracked_na(struct net_device *dev, struct in6_addr *saddr) * same subnet as an address configured on the interface that * received the na */ - return !!ipv6_chk_prefix(saddr, dev); + return !!ipv6_chk_prefix(saddr, idev->dev); default: return 0; } @@ -1078,7 +1076,7 @@ static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb) */ new_state = msg->icmph.icmp6_solicited ? NUD_REACHABLE : NUD_STALE; if (!neigh && lladdr && idev && READ_ONCE(idev->cnf.forwarding)) { - if (accept_untracked_na(dev, saddr)) { + if (accept_untracked_na(idev, saddr)) { neigh = neigh_create(&nd_tbl, &msg->target, dev); new_state = NUD_STALE; } diff --git a/net/ipv6/netfilter.c b/net/ipv6/netfilter.c index 6d80f85e55fa7..a7025ec870359 100644 --- a/net/ipv6/netfilter.c +++ b/net/ipv6/netfilter.c @@ -120,7 +120,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, ktime_t tstamp = skb->tstamp; struct ip6_frag_state state; u8 *prevhdr, nexthdr = 0; - unsigned int mtu, hlen; + unsigned int mtu, hlen, nexthdr_offset; int hroom, err = 0; __be32 frag_id; @@ -129,6 +129,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, goto blackhole; hlen = err; nexthdr = *prevhdr; + nexthdr_offset = prevhdr - skb_network_header(skb); mtu = skb->dev->mtu; if (frag_max_size > mtu || @@ -147,6 +148,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, (err = skb_checksum_help(skb))) goto blackhole; + prevhdr = skb_network_header(skb) + nexthdr_offset; hroom = LL_RESERVED_SPACE(skb->dev); if (skb_has_frag_list(skb)) { unsigned int first_len = skb_pagelen(skb); diff --git a/net/ipv6/netfilter/ip6t_ah.c b/net/ipv6/netfilter/ip6t_ah.c index 70da2f2ce064c..1258783ed8762 100644 --- a/net/ipv6/netfilter/ip6t_ah.c +++ b/net/ipv6/netfilter/ip6t_ah.c @@ -56,6 +56,11 @@ static bool ah_mt6(const struct sk_buff *skb, struct xt_action_param *par) } hdrlen = ipv6_authlen(ah); + if (skb->len - ptr < hdrlen) { + /* Packet smaller than its length field */ + par->hotdrop = true; + return false; + } pr_debug("IPv6 AH LEN %u %u ", hdrlen, ah->hdrlen); pr_debug("RES %04X ", ah->reserved); diff --git a/net/ipv6/netfilter/ip6t_hbh.c b/net/ipv6/netfilter/ip6t_hbh.c index 450dd53846a2f..6d1a5d2026a67 100644 --- a/net/ipv6/netfilter/ip6t_hbh.c +++ b/net/ipv6/netfilter/ip6t_hbh.c @@ -75,6 +75,7 @@ hbh_mt6(const struct sk_buff *skb, struct xt_action_param *par) hdrlen = ipv6_optlen(oh); if (skb->len - ptr < hdrlen) { /* Packet smaller than it's length field */ + par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 5561bd9cea818..278b52752f364 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -56,7 +56,8 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par) hdrlen = ipv6_optlen(rh); if (skb->len - ptr < hdrlen) { - /* Pcket smaller than its length field */ + /* Packet smaller than its length field */ + par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/nf_conntrack_reasm.c b/net/ipv6/netfilter/nf_conntrack_reasm.c index 64ab23ff559bb..599c49bf0a0af 100644 --- a/net/ipv6/netfilter/nf_conntrack_reasm.c +++ b/net/ipv6/netfilter/nf_conntrack_reasm.c @@ -348,7 +348,8 @@ static int nf_ct_frag6_reasm(struct frag_queue *fq, struct sk_buff *skb, skb_network_header(skb)[fq->nhoffset] = skb_transport_header(skb)[0]; memmove(skb->head + sizeof(struct frag_hdr), skb->head, (skb->data - skb->head) - sizeof(struct frag_hdr)); - skb->mac_header += sizeof(struct frag_hdr); + if (skb_mac_header_was_set(skb)) + skb->mac_header += sizeof(struct frag_hdr); skb->network_header += sizeof(struct frag_hdr); skb_reset_transport_header(skb); @@ -418,7 +419,7 @@ find_prev_fhdr(struct sk_buff *skb, u8 *prevhdrp, int *prevhoff, int *fhoff) return -1; } if (skb_copy_bits(skb, start, &hdr, sizeof(hdr))) - BUG(); + return -1; if (nexthdr == NEXTHDR_AUTH) hdrlen = ipv6_authlen(&hdr); else diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 636f0120d7e38..88b46967bb67d 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -3275,11 +3275,11 @@ static unsigned int ip6_default_advmss(const struct dst_entry *dst) /* * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and * corresponding MSS is IPV6_MAXPLEN - tcp_header_size. - * IPV6_MAXPLEN is also valid and means: "any MSS, - * rely only on pmtu discovery" + * Limit the default MSS to GSO_BY_FRAGS - 1 to avoid + * collision with the GSO_BY_FRAGS magic value (0xFFFF). */ if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr)) - mtu = IPV6_MAXPLEN; + mtu = min_t(unsigned int, IPV6_MAXPLEN, GSO_BY_FRAGS - 1); return mtu; } @@ -5053,6 +5053,9 @@ static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg) struct inet6_dev *idev = __in6_dev_get(arg->dev); u32 mtu = f6i->fib6_pmtu; + if (!idev) + return 0; + if (mtu >= arg->mtu || (mtu < arg->mtu && mtu == idev->cnf.mtu6)) fib6_metric_set(f6i, RTAX_MTU, arg->mtu); diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c index 1c3ad25700c4c..62a7eb7792026 100644 --- a/net/ipv6/seg6.c +++ b/net/ipv6/seg6.c @@ -29,6 +29,9 @@ bool seg6_validate_srh(struct ipv6_sr_hdr *srh, int len, bool reduced) int max_last_entry; int trailing; + if (len < sizeof(*srh)) + return false; + if (srh->type != IPV6_SRCRT_TYPE_4) return false; diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index b41e231a669be..34db8b886bd25 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -1611,6 +1611,9 @@ static int ipip6_changelink(struct net_device *dev, struct nlattr *tb[], __u32 fwmark = t->fwmark; int err; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + if (dev == sitn->fb_tunnel_dev) return -EINVAL; diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 699a001ac1662..89d0443b53073 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -71,8 +71,6 @@ int xfrm6_transport_finish(struct sk_buff *skb, int async) NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING, dev_net(dev), NULL, skb, dev, NULL, xfrm6_transport_finish2); - if (async) - dev_put(dev); return 0; } diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 3912e75079f5e..a998336840c33 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -1304,8 +1304,8 @@ static int kcm_attach(struct socket *sock, struct socket *csock, psock->save_write_space = csk->sk_write_space; psock->save_state_change = csk->sk_state_change; csk->sk_user_data = psock; - csk->sk_data_ready = psock_data_ready; - csk->sk_write_space = psock_write_space; + WRITE_ONCE(csk->sk_data_ready, psock_data_ready); + WRITE_ONCE(csk->sk_write_space, psock_write_space); csk->sk_state_change = psock_state_change; write_unlock_bh(&csk->sk_callback_lock); @@ -1381,8 +1381,8 @@ static void kcm_unattach(struct kcm_psock *psock) */ write_lock_bh(&csk->sk_callback_lock); csk->sk_user_data = NULL; - csk->sk_data_ready = psock->save_data_ready; - csk->sk_write_space = psock->save_write_space; + WRITE_ONCE(csk->sk_data_ready, psock->save_data_ready); + WRITE_ONCE(csk->sk_write_space, psock->save_write_space); csk->sk_state_change = psock->save_state_change; strp_stop(&psock->strp); diff --git a/net/key/af_key.c b/net/key/af_key.c index 9cffeef18cd91..3216f897a305e 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -1218,6 +1218,7 @@ static struct xfrm_state * pfkey_msg2xfrm_state(struct net *net, goto out; } strcpy(x->calg->alg_name, a->name); + x->calg->alg_key_len = 0; x->props.calgo = sa->sadb_sa_encrypt; } else { int keysize = 0; diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c index 9419c8555d229..f940914959b18 100644 --- a/net/l2tp/l2tp_core.c +++ b/net/l2tp/l2tp_core.c @@ -1684,7 +1684,7 @@ int l2tp_tunnel_register(struct l2tp_tunnel *tunnel, struct net *net, .encap_destroy = l2tp_udp_encap_destroy, }; - setup_udp_tunnel_sock(net, sock, &udp_cfg); + setup_udp_tunnel_sock(net, sock->sk, &udp_cfg); } sk->sk_allocation = GFP_ATOMIC; diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index 1b210db3119e8..f4e20d1218e91 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -317,6 +317,7 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr) /* assign new connection to its SAP */ llc_sap_add_socket(sap, sk); sock_reset_flag(sk, SOCK_ZAPPED); + llc_sap_put(sap); rc = 0; out: dev_put(dev); diff --git a/net/llc/llc_conn.c b/net/llc/llc_conn.c index 5c0ac243b248f..b4d732baafb89 100644 --- a/net/llc/llc_conn.c +++ b/net/llc/llc_conn.c @@ -773,7 +773,6 @@ static struct sock *llc_create_incoming_sock(struct sock *sk, newllc->dev = dev; dev_hold(dev); llc_sap_add_socket(llc->sap, newsk); - llc_sap_hold(llc->sap); out: return newsk; } diff --git a/net/mac80211/main.c b/net/mac80211/main.c index f47dd58770adb..dba66dd964af3 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -1599,7 +1599,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) sband = kmemdup(sband, sizeof(*sband), GFP_KERNEL); if (!sband) { result = -ENOMEM; - goto fail_rate; + goto fail_band; } wiphy_dbg(hw->wiphy, "copying sband (band %d) due to VHT EXT NSS BW flag\n", @@ -1675,6 +1675,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) #endif wiphy_unregister(local->hw.wiphy); fail_wiphy_register: + fail_band: rtnl_lock(); rate_control_deinitialize(local); ieee80211_remove_interfaces(local); diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 3fb40449c6c5c..fe583636d326e 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -1526,6 +1526,9 @@ ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx) if (status->flag & RX_FLAG_DUP_VALIDATED) return RX_CONTINUE; + if (ieee80211_is_ext(hdr->frame_control)) + return RX_CONTINUE; + /* * Drop duplicate 802.11 retransmissions * (IEEE 802.11-2012: 9.3.2.10 "Duplicate detection and recovery") @@ -4487,12 +4490,16 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx) struct ieee80211_hdr *hdr = (void *)skb->data; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); u8 *bssid = ieee80211_get_bssid(hdr, skb->len, sdata->vif.type); - bool multicast = is_multicast_ether_addr(hdr->addr1) || - ieee80211_is_s1g_beacon(hdr->frame_control); + bool multicast; static const u8 nan_network_id[ETH_ALEN] __aligned(2) = { 0x51, 0x6F, 0x9A, 0x01, 0x00, 0x00 }; + if (ieee80211_is_s1g_beacon(hdr->frame_control)) + return sdata->vif.type == NL80211_IFTYPE_STATION && bssid; + + multicast = is_multicast_ether_addr(hdr->addr1); + switch (sdata->vif.type) { case NL80211_IFTYPE_STATION: if (!bssid && !sdata->u.mgd.use_4addr) @@ -5174,6 +5181,11 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx, hdr = (struct ieee80211_hdr *)rx->skb->data; } + if (ieee80211_is_s1g_beacon(hdr->frame_control)) { + ieee80211_invoke_rx_handlers(rx); + return true; + } + /* Store a copy of the pre-translated link addresses for SW crypto */ if (unlikely(is_unicast_ether_addr(hdr->addr1) && !ieee80211_is_data(hdr->frame_control))) @@ -5263,6 +5275,13 @@ static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx, struct sta_info *sta; int link_id = -1; + if (ieee80211_is_s1g_beacon(hdr->frame_control)) { + if (!ieee80211_rx_data_set_sta(rx, NULL, -1)) + return false; + + return ieee80211_prepare_and_rx_handle(rx, skb, consume); + } + /* * Look up link station first, in case there's a * chance that they might have a link address that @@ -5338,6 +5357,17 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, err = -ENOBUFS; else err = skb_linearize(skb); + } else if (ieee80211_is_s1g_beacon(fc)) { + size_t s1g_hdr_len = offsetof(struct ieee80211_ext, + u.s1g_beacon.variable) + + ieee80211_s1g_optional_len(fc); + + if (skb->len < s1g_hdr_len) + err = -ENOBUFS; + else + err = skb_linearize(skb); + } else if (ieee80211_is_ext(fc)) { + err = -EINVAL; } else { err = !pskb_may_pull(skb, ieee80211_hdrlen(fc)); } diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index ea7f63e1fc17e..8e7f43ee10e5c 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2597,6 +2597,18 @@ static u16 ieee80211_store_ack_skb(struct ieee80211_local *local, return info_id; } +static void ieee80211_remove_ack_skb(struct ieee80211_local *local, u16 info_id) +{ + struct sk_buff *ack_skb; + unsigned long flags; + + spin_lock_irqsave(&local->ack_status_lock, flags); + ack_skb = idr_remove(&local->ack_status_frames, info_id); + spin_unlock_irqrestore(&local->ack_status_lock, flags); + + kfree_skb(ack_skb); +} + /** * ieee80211_build_hdr - build 802.11 header in the given frame * @sdata: virtual interface to build the header for @@ -2972,7 +2984,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, if (ieee80211_skb_resize(sdata, skb, head_need, ENCRYPT_DATA)) { ieee80211_free_txskb(&local->hw, skb); skb = NULL; - return ERR_PTR(-ENOMEM); + ret = -ENOMEM; + goto free; } } @@ -3040,6 +3053,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, return skb; free: + if (info_id) + ieee80211_remove_ack_skb(local, info_id); kfree_skb(skb); return ERR_PTR(ret); } diff --git a/net/mac80211/util.c b/net/mac80211/util.c index 2529b01e2cd55..5bc719222a87d 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -73,6 +73,9 @@ u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len, if (ieee80211_is_s1g_beacon(fc)) { struct ieee80211_ext *ext = (void *) hdr; + if (len < offsetofend(struct ieee80211_ext, u.s1g_beacon.sa)) + return NULL; + return ext->u.s1g_beacon.sa; } diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index 000be60d95803..b823720630e72 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -703,7 +703,7 @@ void ieee802154_remove_interfaces(struct ieee802154_local *local) mutex_lock(&local->iflist_mtx); list_for_each_entry_safe(sdata, tmp, &local->interfaces, list) { - list_del(&sdata->list); + list_del_rcu(&sdata->list); unregister_netdevice(sdata->dev); } diff --git a/net/mac802154/llsec.c b/net/mac802154/llsec.c index e8512578398e5..5e7cc11fab3a4 100644 --- a/net/mac802154/llsec.c +++ b/net/mac802154/llsec.c @@ -710,6 +710,7 @@ int mac802154_llsec_encrypt(struct mac802154_llsec *sec, struct sk_buff *skb) { struct ieee802154_hdr hdr; int rc, authlen, hlen; + struct sk_buff *trailer; struct mac802154_llsec_key *key; u32 frame_ctr; @@ -769,6 +770,12 @@ int mac802154_llsec_encrypt(struct mac802154_llsec *sec, struct sk_buff *skb) skb->mac_len = ieee802154_hdr_push(skb, &hdr); skb_reset_mac_header(skb); + rc = skb_cow_data(skb, 0, &trailer); + if (rc < 0) { + llsec_key_put(key); + return rc; + } + rc = llsec_do_encrypt(skb, sec, &hdr, key); llsec_key_put(key); @@ -908,6 +915,13 @@ llsec_do_decrypt(struct sk_buff *skb, const struct mac802154_llsec *sec, const struct ieee802154_hdr *hdr, struct mac802154_llsec_key *key, __le64 dev_addr) { + struct sk_buff *trailer; + int err; + + err = skb_cow_data(skb, 0, &trailer); + if (err < 0) + return err; + if (hdr->sec.level == IEEE802154_SCF_SECLEVEL_ENC) return llsec_do_decrypt_unauth(skb, sec, hdr, key, dev_addr); else diff --git a/net/mac802154/scan.c b/net/mac802154/scan.c index 0a31ac8d84155..300d4584533e6 100644 --- a/net/mac802154/scan.c +++ b/net/mac802154/scan.c @@ -594,6 +594,7 @@ int mac802154_perform_association(struct ieee802154_sub_if_data *sdata, "Negative ASSOC RESP received from %8phC: %s\n", &ceaddr, local->assoc_status == IEEE802154_PAN_AT_CAPACITY ? "PAN at capacity" : "access denied"); + goto clear_assoc; } ret = 0; diff --git a/net/netfilter/ipset/ip_set_bitmap_gen.h b/net/netfilter/ipset/ip_set_bitmap_gen.h index 798c7993635e6..bb9b5bed10e19 100644 --- a/net/netfilter/ipset/ip_set_bitmap_gen.h +++ b/net/netfilter/ipset/ip_set_bitmap_gen.h @@ -165,6 +165,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, ip_set_init_skbinfo(ext_skbinfo(x, set), ext); /* Activate element */ + smp_mb__before_atomic(); set_bit(e->id, map->members); set->elements++; @@ -219,7 +220,7 @@ mtype_list(const struct ip_set *set, cond_resched_rcu(); id = cb->args[IPSET_CB_ARG0]; x = get_ext(set, map, id); - if (!test_bit(id, map->members) || + if (!test_bit_acquire(id, map->members) || (SET_WITH_TIMEOUT(set) && #ifdef IP_SET_BITMAP_STORED_TIMEOUT mtype_is_filled(x) && @@ -278,6 +279,7 @@ mtype_gc(struct timer_list *t) x = get_ext(set, map, id); if (ip_set_timeout_expired(ext_timeout(x, set))) { clear_bit(id, map->members); + smp_mb__after_atomic(); ip_set_ext_destroy(set, x); set->elements--; } diff --git a/net/netfilter/ipset/ip_set_bitmap_ip.c b/net/netfilter/ipset/ip_set_bitmap_ip.c index 5988b9bb9029d..ac7febce074f1 100644 --- a/net/netfilter/ipset/ip_set_bitmap_ip.c +++ b/net/netfilter/ipset/ip_set_bitmap_ip.c @@ -67,7 +67,7 @@ static int bitmap_ip_do_test(const struct bitmap_ip_adt_elem *e, struct bitmap_ip *map, size_t dsize) { - return !!test_bit(e->id, map->members); + return !!test_bit_acquire(e->id, map->members); } static int diff --git a/net/netfilter/ipset/ip_set_bitmap_ipmac.c b/net/netfilter/ipset/ip_set_bitmap_ipmac.c index 752f59ef87442..5921fd9d2dca0 100644 --- a/net/netfilter/ipset/ip_set_bitmap_ipmac.c +++ b/net/netfilter/ipset/ip_set_bitmap_ipmac.c @@ -86,7 +86,7 @@ bitmap_ipmac_do_test(const struct bitmap_ipmac_adt_elem *e, { const struct bitmap_ipmac_elem *elem; - if (!test_bit(e->id, map->members)) + if (!test_bit_acquire(e->id, map->members)) return 0; elem = get_const_elem(map->extensions, e->id, dsize); if (e->add_mac && elem->filled == MAC_FILLED) diff --git a/net/netfilter/ipset/ip_set_bitmap_port.c b/net/netfilter/ipset/ip_set_bitmap_port.c index 7138e080def4c..ca875c9824245 100644 --- a/net/netfilter/ipset/ip_set_bitmap_port.c +++ b/net/netfilter/ipset/ip_set_bitmap_port.c @@ -58,7 +58,7 @@ static int bitmap_port_do_test(const struct bitmap_port_adt_elem *e, const struct bitmap_port *map, size_t dsize) { - return !!test_bit(e->id, map->members); + return !!test_bit_acquire(e->id, map->members); } static int diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index 3706b4a85a0f1..6cfad152d7d1b 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -351,8 +351,8 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment, if (unlikely(c)) { set->ext_size -= sizeof(*c) + strlen(c->str) + 1; - kfree_rcu(c, rcu); rcu_assign_pointer(comment->c, NULL); + kfree_rcu(c, rcu); } if (!len) return; @@ -393,8 +393,8 @@ ip_set_comment_free(struct ip_set *set, void *ptr) if (unlikely(!c)) return; set->ext_size -= sizeof(*c) + strlen(c->str) + 1; - kfree_rcu(c, rcu); rcu_assign_pointer(comment->c, NULL); + kfree_rcu(c, rcu); } typedef void (*destroyer)(struct ip_set *, void *); @@ -1480,7 +1480,11 @@ ip_set_dump_done(struct netlink_callback *cb) struct ip_set_net *inst = (struct ip_set_net *)cb->args[IPSET_CB_NET]; ip_set_id_t index = (ip_set_id_t)cb->args[IPSET_CB_INDEX]; - struct ip_set *set = ip_set_ref_netlink(inst, index); + struct ip_set *set; + + rcu_read_lock(); + set = ip_set_ref_netlink(inst, index); + rcu_read_unlock(); if (set->variant->uref) set->variant->uref(set, cb, false); @@ -1686,7 +1690,9 @@ ip_set_dump_do(struct sk_buff *skb, struct netlink_callback *cb) release_refcount: /* If there was an error or set is done, release set */ if (ret || !cb->args[IPSET_CB_ARG0]) { + rcu_read_lock(); set = ip_set_ref_netlink(inst, index); + rcu_read_unlock(); if (set->variant->uref) set->variant->uref(set, cb, false); pr_debug("release set %s\n", set->name); diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index 04e4627ddfc10..dedf59b661ddf 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -606,7 +606,7 @@ mtype_cancel_gc(struct ip_set *set) struct htype *h = set->data; if (SET_WITH_TIMEOUT(set)) - cancel_delayed_work_sync(&h->gc.dwork); + disable_delayed_work_sync(&h->gc.dwork); } static int @@ -689,7 +689,7 @@ mtype_resize(struct ip_set *set, bool retried) continue; pos = smp_load_acquire(&n->pos); for (j = 0; j < pos; j++) { - if (!test_bit(j, n->used)) + if (!test_bit_acquire(j, n->used)) continue; data = ahash_data(n, j, dsize); if (SET_ELEM_EXPIRED(set, data)) @@ -826,7 +826,7 @@ mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size) continue; pos = smp_load_acquire(&n->pos); for (j = 0; j < pos; j++) { - if (!test_bit(j, n->used)) + if (!test_bit_acquire(j, n->used)) continue; data = ahash_data(n, j, set->dsize); if (!SET_ELEM_EXPIRED(set, data)) @@ -1201,7 +1201,7 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, continue; pos = smp_load_acquire(&n->pos); for (i = 0; i < pos; i++) { - if (!test_bit(i, n->used)) + if (!test_bit_acquire(i, n->used)) continue; data = ahash_data(n, i, set->dsize); if (!mtype_data_equal(data, d, &multi)) @@ -1259,7 +1259,7 @@ mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext, } pos = smp_load_acquire(&n->pos); for (i = 0; i < pos; i++) { - if (!test_bit(i, n->used)) + if (!test_bit_acquire(i, n->used)) continue; data = ahash_data(n, i, set->dsize); if (!mtype_data_equal(data, d, &multi)) @@ -1396,7 +1396,7 @@ mtype_list(const struct ip_set *set, continue; pos = smp_load_acquire(&n->pos); for (i = 0; i < pos; i++) { - if (!test_bit(i, n->used)) + if (!test_bit_acquire(i, n->used)) continue; e = ahash_data(n, i, set->dsize); if (SET_ELEM_EXPIRED(set, e)) diff --git a/net/netfilter/ipvs/ip_vs_app.c b/net/netfilter/ipvs/ip_vs_app.c index d54d7da583346..b0e00be85cb12 100644 --- a/net/netfilter/ipvs/ip_vs_app.c +++ b/net/netfilter/ipvs/ip_vs_app.c @@ -361,14 +361,13 @@ static inline int app_tcp_pkt_out(struct ip_vs_conn *cp, struct sk_buff *skb, struct ip_vs_iphdr *ipvsh) { int diff; - const unsigned int tcp_offset = ip_hdrlen(skb); struct tcphdr *th; __u32 seq; - if (skb_ensure_writable(skb, tcp_offset + sizeof(*th))) + if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th))) return 0; - th = (struct tcphdr *)(skb_network_header(skb) + tcp_offset); + th = (struct tcphdr *)(skb_network_header(skb) + ipvsh->len); /* * Remember seq number in case this pkt gets resized @@ -438,14 +437,13 @@ static inline int app_tcp_pkt_in(struct ip_vs_conn *cp, struct sk_buff *skb, struct ip_vs_iphdr *ipvsh) { int diff; - const unsigned int tcp_offset = ip_hdrlen(skb); struct tcphdr *th; __u32 seq; - if (skb_ensure_writable(skb, tcp_offset + sizeof(*th))) + if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th))) return 0; - th = (struct tcphdr *)(skb_network_header(skb) + tcp_offset); + th = (struct tcphdr *)(skb_network_header(skb) + ipvsh->len); /* * Remember seq number in case this pkt gets resized diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 9ea6b4fa78bf0..b457dd2f6bc89 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -1411,8 +1411,8 @@ ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, cp->app = NULL; cp->app_data = NULL; /* reset struct ip_vs_seq */ - cp->in_seq.delta = 0; - cp->out_seq.delta = 0; + memset(&cp->in_seq, 0, sizeof(cp->in_seq)); + memset(&cp->out_seq, 0, sizeof(cp->out_seq)); atomic_inc(&ipvs->conn_count); if (unlikely(flags & IP_VS_CONN_F_NO_CPORT)) { diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index d40b404c1bf64..bafab93451d03 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -398,10 +398,10 @@ ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc) static inline void ip_vs_set_state(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd) + struct ip_vs_proto_data *pd, unsigned int iph_len) { if (likely(pd->pp->state_transition)) - pd->pp->state_transition(cp, direction, skb, pd); + pd->pp->state_transition(cp, direction, skb, pd, iph_len); } static inline int @@ -803,7 +803,7 @@ int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb, ip_vs_in_stats(cp, skb); /* set state */ - ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd); + ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd, iph->len); /* transmit the first SYN packet */ ret = cp->packet_xmit(skb, cp, pd->pp, iph); @@ -1219,8 +1219,7 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, snet.in6 = ciph.saddr.in6; offset = ciph.len; return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp, - pp, offset, sizeof(struct ipv6hdr), - hooknum); + pp, offset, ipvsh->len, hooknum); } #endif @@ -1484,7 +1483,7 @@ handle_response(int af, struct sk_buff *skb, struct ip_vs_proto_data *pd, after_nat: ip_vs_out_stats(cp, skb); - ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd); + ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd, iph->len); skb->ipvs_property = 1; if (!(cp->flags & IP_VS_CONN_F_NFCT)) ip_vs_notrack(skb); @@ -1767,6 +1766,8 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, bool tunnel, new_cp = false; union nf_inet_addr *raddr; char *outer_proto = "IPIP"; + unsigned int hlen_ipip; + int ulen = 0; *related = 1; @@ -1803,9 +1804,10 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Now find the contained IP header */ offset += sizeof(_icmph); cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (cih == NULL) + if (!(cih && cih->version == 4 && cih->ihl >= 5)) return NF_ACCEPT; /* The packet looks wrong, ignore */ raddr = (union nf_inet_addr *)&cih->daddr; + hlen_ipip = cih->ihl * 4; /* Special case for errors for IPIP/UDP/GRE tunnel packets */ tunnel = false; @@ -1821,9 +1823,9 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Only for known tunnel */ if (!dest || dest->tun_type != IP_VS_CONN_F_TUNNEL_TYPE_IPIP) return NF_ACCEPT; - offset += cih->ihl * 4; + offset += hlen_ipip; cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (cih == NULL) + if (!(cih && cih->version == 4 && cih->ihl >= 5)) return NF_ACCEPT; /* The packet looks wrong, ignore */ tunnel = true; } else if ((cih->protocol == IPPROTO_UDP || /* Can be UDP encap */ @@ -1831,12 +1833,11 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Error for our tunnel must arrive at LOCAL_IN */ (skb_rtable(skb)->rt_flags & RTCF_LOCAL)) { __u8 iproto; - int ulen; /* Non-first fragment has no UDP/GRE header */ if (unlikely(cih->frag_off & htons(IP_OFFSET))) return NF_ACCEPT; - offset2 = offset + cih->ihl * 4; + offset2 = offset + hlen_ipip; if (cih->protocol == IPPROTO_UDP) { ulen = ipvs_udp_decap(ipvs, skb, offset2, AF_INET, raddr, &iproto); @@ -1905,6 +1906,7 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, } if (tunnel) { + unsigned int hlen_orig = cih->ihl * 4; __be32 info = ic->un.gateway; __u8 type = ic->type; __u8 code = ic->code; @@ -1921,6 +1923,9 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, goto ignore_tunnel; offset2 -= ihl + sizeof(_icmph); skb_reset_network_header(skb); + /* Ensure the IP header is present in headroom */ + if (!pskb_may_pull(skb, hlen_ipip)) + goto ignore_tunnel; IP_VS_DBG(12, "ICMP for %s %pI4->%pI4: mtu=%u\n", outer_proto, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, mtu); @@ -1936,8 +1941,8 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (dest_dst) mtu = dst_mtu(dest_dst->dst_cache); } - if (mtu > 68 + sizeof(struct iphdr)) - mtu -= sizeof(struct iphdr); + if (mtu > 68 + hlen_ipip + ulen) + mtu -= hlen_ipip + ulen; info = htonl(mtu); } /* Strip outer IP, ICMP and IPIP/UDP/GRE, go to IP header of @@ -1946,6 +1951,9 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (pskb_pull(skb, offset2) == NULL) goto ignore_tunnel; skb_reset_network_header(skb); + /* Ensure the IP header is present in headroom */ + if (!pskb_may_pull(skb, hlen_orig)) + goto ignore_tunnel; IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n", &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, type, code, ntohl(info)); @@ -2233,7 +2241,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state IP_VS_DBG_PKT(11, af, pp, skb, iph.off, "Incoming packet"); ip_vs_in_stats(cp, skb); - ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd); + ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd, iph.len); if (cp->packet_xmit) ret = cp->packet_xmit(skb, cp, pp, &iph); /* do not touch skb anymore */ diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index 63c78a1f3918a..c67317be17dfa 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -372,20 +372,15 @@ static const char *sctp_state_name(int state) static inline void set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, - int direction, const struct sk_buff *skb) + int direction, const struct sk_buff *skb, + unsigned int iph_len) { struct sctp_chunkhdr _sctpch, *sch; unsigned char chunk_type; int event, next_state; - int ihl, cofs; + int cofs; -#ifdef CONFIG_IP_VS_IPV6 - ihl = cp->af == AF_INET ? ip_hdrlen(skb) : sizeof(struct ipv6hdr); -#else - ihl = ip_hdrlen(skb); -#endif - - cofs = ihl + sizeof(struct sctphdr); + cofs = iph_len + sizeof(struct sctphdr); sch = skb_header_pointer(skb, cofs, sizeof(_sctpch), &_sctpch); if (sch == NULL) return; @@ -468,10 +463,11 @@ set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, static void sctp_state_transition(struct ip_vs_conn *cp, int direction, - const struct sk_buff *skb, struct ip_vs_proto_data *pd) + const struct sk_buff *skb, struct ip_vs_proto_data *pd, + unsigned int iph_len) { spin_lock_bh(&cp->lock); - set_sctp_state(pd, cp, direction, skb); + set_sctp_state(pd, cp, direction, skb, iph_len); spin_unlock_bh(&cp->lock); } diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index 8cc0a8ce62411..f86b763efcc4d 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -579,17 +579,12 @@ set_tcp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, static void tcp_state_transition(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd) + struct ip_vs_proto_data *pd, + unsigned int iph_len) { struct tcphdr _tcph, *th; -#ifdef CONFIG_IP_VS_IPV6 - int ihl = cp->af == AF_INET ? ip_hdrlen(skb) : sizeof(struct ipv6hdr); -#else - int ihl = ip_hdrlen(skb); -#endif - - th = skb_header_pointer(skb, ihl, sizeof(_tcph), &_tcph); + th = skb_header_pointer(skb, iph_len, sizeof(_tcph), &_tcph); if (th == NULL) return; diff --git a/net/netfilter/ipvs/ip_vs_proto_udp.c b/net/netfilter/ipvs/ip_vs_proto_udp.c index f9de632e38cdd..58f9e255927e2 100644 --- a/net/netfilter/ipvs/ip_vs_proto_udp.c +++ b/net/netfilter/ipvs/ip_vs_proto_udp.c @@ -444,7 +444,8 @@ static const char * udp_state_name(int state) static void udp_state_transition(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd) + struct ip_vs_proto_data *pd, + unsigned int iph_len) { if (unlikely(!pd)) { pr_err("UDP no ns data\n"); diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index ce542ed4b013c..9fef4335da13f 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -736,13 +736,11 @@ int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) { - struct iphdr *iph = ip_hdr(skb); - - if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, iph->daddr, + if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, ip_hdr(skb)->daddr, IP_VS_RT_MODE_NON_LOCAL, NULL, ipvsh) < 0) goto tx_error; - ip_send_check(iph); + ip_send_check(ip_hdr(skb)); /* Another hack: avoid icmp_send in ip_fragment */ skb->ignore_df = 1; diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index 00eed5b4d1b12..495c4a779d22f 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -179,17 +179,16 @@ static int __nf_conncount_add(struct net *net, return -ENOENT; if (ct && nf_ct_is_confirmed(ct)) { - /* local connections are confirmed in postrouting so confirmation - * might have happened before hitting connlimit + /* Connection is confirmed but might still be in the setup phase. + * Only skip the tracking if it is fully assured. This guarantees + * that setup packets or retransmissions are properly counted and + * deduplicated. */ - if (skb->skb_iif != LOOPBACK_IFINDEX) { + if (test_bit(IPS_ASSURED_BIT, &ct->status)) { err = -EEXIST; goto out_put; } - /* this is likely a local connection, skip optimization to avoid - * adding duplicates from a 'packet train' - */ goto check_connections; } @@ -208,8 +207,8 @@ static int __nf_conncount_add(struct net *net, /* Not found, but might be about to be confirmed */ if (PTR_ERR(found) == -EAGAIN) { if (nf_ct_tuple_equal(&conn->tuple, &tuple) && - nf_ct_zone_id(&conn->zone, conn->zone.dir) == - nf_ct_zone_id(zone, zone->dir)) + nf_ct_zone_id(&conn->zone, IP_CT_DIR_ORIGINAL) == + nf_ct_zone_id(zone, IP_CT_DIR_ORIGINAL)) goto out_put; /* already exists */ } else { collect++; @@ -220,7 +219,7 @@ static int __nf_conncount_add(struct net *net, found_ct = nf_ct_tuplehash_to_ctrack(found); if (nf_ct_tuple_equal(&conn->tuple, &tuple) && - nf_ct_zone_equal(found_ct, zone, zone->dir)) { + nf_ct_zone_equal(found_ct, zone, IP_CT_DIR_ORIGINAL)) { /* * We should not see tuples twice unless someone hooks * this into a table without "-p tcp --syn". @@ -499,7 +498,7 @@ count_tree(struct net *net, hash = jhash2(key, data->keylen, conncount_rnd) % CONNCOUNT_SLOTS; root = &data->root[hash]; - parent = rcu_dereference_raw(root->rb_node); + parent = rcu_dereference(root->rb_node); while (parent) { int diff; @@ -507,9 +506,9 @@ count_tree(struct net *net, diff = key_diff(key, rbconn->key, data->keylen); if (diff < 0) { - parent = rcu_dereference_raw(parent->rb_left); + parent = rcu_dereference(parent->rb_left); } else if (diff > 0) { - parent = rcu_dereference_raw(parent->rb_right); + parent = rcu_dereference(parent->rb_right); } else { int ret; diff --git a/net/netfilter/nf_conntrack_amanda.c b/net/netfilter/nf_conntrack_amanda.c index d2c09e8dd8726..ddafbdfc96dc3 100644 --- a/net/netfilter/nf_conntrack_amanda.c +++ b/net/netfilter/nf_conntrack_amanda.c @@ -169,35 +169,15 @@ static const struct nf_conntrack_expect_policy amanda_exp_policy = { .timeout = 180, }; -static struct nf_conntrack_helper amanda_helper[2] __read_mostly = { - { - .name = HELPER_NAME, - .me = THIS_MODULE, - .help = amanda_help, - .tuple.src.l3num = AF_INET, - .tuple.src.u.udp.port = cpu_to_be16(10080), - .tuple.dst.protonum = IPPROTO_UDP, - .expect_policy = &amanda_exp_policy, - .nat_mod_name = NF_NAT_HELPER_NAME(HELPER_NAME), - }, - { - .name = "amanda", - .me = THIS_MODULE, - .help = amanda_help, - .tuple.src.l3num = AF_INET6, - .tuple.src.u.udp.port = cpu_to_be16(10080), - .tuple.dst.protonum = IPPROTO_UDP, - .expect_policy = &amanda_exp_policy, - .nat_mod_name = NF_NAT_HELPER_NAME(HELPER_NAME), - }, -}; +static struct nf_conntrack_helper amanda_helper[2] __read_mostly; +static struct nf_conntrack_helper *amanda_helper_ptr[2] __read_mostly; static void __exit nf_conntrack_amanda_fini(void) { int i; - nf_conntrack_helpers_unregister(amanda_helper, - ARRAY_SIZE(amanda_helper)); + nf_conntrack_helpers_unregister(amanda_helper_ptr, + ARRAY_SIZE(amanda_helper_ptr)); for (i = 0; i < ARRAY_SIZE(search); i++) textsearch_destroy(search[i].ts); } @@ -217,8 +197,17 @@ static int __init nf_conntrack_amanda_init(void) goto err1; } } + + nf_ct_helper_init(&amanda_helper[0], AF_INET, IPPROTO_UDP, + HELPER_NAME, 10080, 10080, 10080, + &amanda_exp_policy, 0, amanda_help, NULL, THIS_MODULE); + nf_ct_helper_init(&amanda_helper[1], AF_INET6, IPPROTO_UDP, + HELPER_NAME, 10080, 10080, 10080, + &amanda_exp_policy, 0, amanda_help, NULL, THIS_MODULE); + ret = nf_conntrack_helpers_register(amanda_helper, - ARRAY_SIZE(amanda_helper)); + ARRAY_SIZE(amanda_helper), + amanda_helper_ptr); if (ret < 0) goto err1; return 0; diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c index 40c261cd0af38..f98d1d4b42c3d 100644 --- a/net/netfilter/nf_conntrack_bpf.c +++ b/net/netfilter/nf_conntrack_bpf.c @@ -65,6 +65,15 @@ enum { NF_BPF_CT_OPTS_SZ = 16, }; +static void *bpf_ct_opts_result(struct bpf_ct_opts *opts, u32 opts__sz, void *ret) +{ + if (!IS_ERR(ret)) + return ret; + if (opts__sz >= offsetofend(struct bpf_ct_opts, error)) + opts->error = PTR_ERR(ret); + return NULL; +} + static int bpf_nf_ct_tuple_parse(struct bpf_sock_tuple *bpf_tuple, u32 tuple_len, u8 protonum, u8 dir, struct nf_conntrack_tuple *tuple) @@ -297,12 +306,7 @@ bpf_xdp_ct_alloc(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple, nfct = __bpf_nf_ct_alloc_entry(dev_net(ctx->rxq->dev), bpf_tuple, tuple__sz, opts, opts__sz, 10); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - - return (struct nf_conn___init *)nfct; + return (struct nf_conn___init *)bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_xdp_ct_lookup - Lookup CT entry for the given tuple, and acquire a @@ -331,11 +335,7 @@ bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple, caller_net = dev_net(ctx->rxq->dev); nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - return nfct; + return bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_skb_ct_alloc - Allocate a new CT entry @@ -363,12 +363,7 @@ bpf_skb_ct_alloc(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple, net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk); nfct = __bpf_nf_ct_alloc_entry(net, bpf_tuple, tuple__sz, opts, opts__sz, 10); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - - return (struct nf_conn___init *)nfct; + return (struct nf_conn___init *)bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_skb_ct_lookup - Lookup CT entry for the given tuple, and acquire a @@ -397,11 +392,7 @@ bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple, caller_net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk); nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - return nfct; + return bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_ct_insert_entry - Add the provided entry into a CT map diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c index 75e53fde6b297..bf78828c7549d 100644 --- a/net/netfilter/nf_conntrack_broadcast.c +++ b/net/netfilter/nf_conntrack_broadcast.c @@ -29,6 +29,9 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, struct nf_conn_help *help = nfct_help(ct); __be32 mask = 0; + if (!help) + goto out; + /* we're only interested in locally generated packets */ if (skb->sk == NULL || !net_eq(nf_ct_net(ct), sock_net(skb->sk))) goto out; @@ -59,6 +62,7 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, if (exp == NULL) goto out; + exp->master_tuple = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; exp->tuple = ct->tuplehash[IP_CT_DIR_REPLY].tuple; helper = rcu_dereference(help->helper); diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index b521b5ebd6644..1f7b0d5a55ecf 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -562,9 +562,23 @@ static void destroy_gre_conntrack(struct nf_conn *ct) { #ifdef CONFIG_NF_CT_PROTO_GRE struct nf_conn *master = ct->master; + struct nf_conn_help *help; + + if (!master) + return; + + help = nfct_help(master); + if (help) { + struct nf_conntrack_helper *helper; - if (master) - nf_ct_gre_keymap_destroy(master); + rcu_read_lock(); + helper = rcu_dereference(help->helper); + /* Only pptp helper has a destroy callback. */ + if (helper && helper->destroy) + nf_ct_gre_keymap_destroy(master); + + rcu_read_unlock(); + } #endif } @@ -840,33 +854,6 @@ static void __nf_conntrack_hash_insert(struct nf_conn *ct, &nf_conntrack_hash[reply_hash]); } -static bool nf_ct_ext_valid_pre(const struct nf_ct_ext *ext) -{ - /* if ext->gen_id is not equal to nf_conntrack_ext_genid, some extensions - * may contain stale pointers to e.g. helper that has been removed. - * - * The helper can't clear this because the nf_conn object isn't in - * any hash and synchronize_rcu() isn't enough because associated skb - * might sit in a queue. - */ - return !ext || ext->gen_id == atomic_read(&nf_conntrack_ext_genid); -} - -static bool nf_ct_ext_valid_post(struct nf_ct_ext *ext) -{ - if (!ext) - return true; - - if (ext->gen_id != atomic_read(&nf_conntrack_ext_genid)) - return false; - - /* inserted into conntrack table, nf_ct_iterate_cleanup() - * will find it. Disable nf_ct_ext_find() id check. - */ - WRITE_ONCE(ext->gen_id, 0); - return true; -} - int nf_conntrack_hash_check_insert(struct nf_conn *ct) { @@ -882,9 +869,6 @@ nf_conntrack_hash_check_insert(struct nf_conn *ct) zone = nf_ct_zone(ct); - if (!nf_ct_ext_valid_pre(ct->ext)) - return -EAGAIN; - local_bh_disable(); do { sequence = read_seqcount_begin(&nf_conntrack_generation); @@ -918,18 +902,6 @@ nf_conntrack_hash_check_insert(struct nf_conn *ct) goto chaintoolong; } - /* If genid has changed, we can't insert anymore because ct - * extensions could have stale pointers and nf_ct_iterate_destroy - * might have completed its table scan already. - * - * Increment of the ext genid right after this check is fine: - * nf_ct_iterate_destroy blocks until locks are released. - */ - if (!nf_ct_ext_valid_post(ct->ext)) { - err = -EAGAIN; - goto out; - } - smp_wmb(); /* The caller holds a reference to this object */ refcount_set(&ct->ct_general.use, 2); @@ -1257,11 +1229,6 @@ __nf_conntrack_confirm(struct sk_buff *skb) return NF_DROP; } - if (!nf_ct_ext_valid_pre(ct->ext)) { - NF_CT_STAT_INC(net, insert_failed); - goto dying; - } - /* We have to check the DYING flag after unlink to prevent * a race against nf_ct_get_next_corpse() possibly called from * user context, else we insert an already 'dead' hash, blocking @@ -1324,16 +1291,6 @@ __nf_conntrack_confirm(struct sk_buff *skb) nf_conntrack_double_unlock(hash, reply_hash); local_bh_enable(); - /* ext area is still valid (rcu read lock is held, - * but will go out of scope soon, we need to remove - * this conntrack again. - */ - if (!nf_ct_ext_valid_post(ct->ext)) { - nf_ct_kill(ct); - NF_CT_STAT_INC_ATOMIC(net, drop); - return NF_DROP; - } - help = nfct_help(ct); if (help && help->helper) nf_conntrack_event_cache(IPCT_HELPER, ct); @@ -1514,6 +1471,31 @@ static bool gc_worker_can_early_drop(const struct nf_conn *ct) return false; } +static void nf_ct_help_gc(struct nf_conn *ct) +{ + struct nf_conn_help *help; + + if (!refcount_inc_not_zero(&ct->ct_general.use)) + return; + + /* load ->status after refcount increase */ + smp_acquire__after_ctrl_dep(); + + if (!nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct)) { + nf_ct_put(ct); + return; + } + + /* re-check helper due to SLAB_TYPESAFE_BY_RCU */ + if (test_bit(IPS_HELPER_BIT, &ct->status)) { + help = nfct_help(ct); + if (help) + nf_ct_expectation_gc(help); + } + + nf_ct_put(ct); +} + static void gc_worker(struct work_struct *work) { unsigned int i, hashsz, nf_conntrack_max95 = 0; @@ -1586,7 +1568,13 @@ static void gc_worker(struct work_struct *work) expires = (expires - (long)next_run) / ++count; next_run += expires; - if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp)) + if (gc_worker_skip_ct(tmp)) + continue; + + if (test_bit(IPS_HELPER_BIT, &tmp->status)) + nf_ct_help_gc(tmp); + + if (nf_conntrack_max95 == 0) continue; net = nf_ct_net(tmp); @@ -1737,16 +1725,18 @@ void nf_conntrack_free(struct nf_conn *ct) */ WARN_ON(refcount_read(&ct->ct_general.use) != 0); + rcu_read_lock(); if (ct->status & IPS_SRC_NAT_DONE) { const struct nf_nat_hook *nat_hook; - rcu_read_lock(); nat_hook = rcu_dereference(nf_nat_hook); if (nat_hook) nat_hook->remove_nat_bysrc(ct); - rcu_read_unlock(); } + nf_ct_timeout_put(ct); + rcu_read_unlock(); + kfree(ct->ext); kmem_cache_free(nf_conntrack_cachep, ct); cnet = nf_ct_pernet(net); @@ -2213,6 +2203,7 @@ static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct, { const struct nf_conntrack_helper *helper; const struct nf_conn_help *help; + unsigned int helper_flags; int protoff; help = nfct_help(ct); @@ -2223,7 +2214,8 @@ static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct, if (!helper) return NF_ACCEPT; - if (!(helper->flags & NF_CT_HELPER_F_USERSPACE)) + helper_flags = READ_ONCE(helper->flags); + if (!(helper_flags & NF_CT_HELPER_F_USERSPACE)) return NF_ACCEPT; switch (nf_ct_l3num(ct)) { @@ -2436,13 +2428,11 @@ nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data) */ synchronize_net(); - nf_ct_ext_bump_genid(); iter_data.data = data; nf_ct_iterate_cleanup(iter, &iter_data); /* Another cpu might be in a rcu read section with - * rcu protected pointer cleared in iter callback - * or hidden via nf_ct_ext_bump_genid() above. + * rcu protected pointer cleared in iter callback. * * Wait until those are done. */ diff --git a/net/netfilter/nf_conntrack_ecache.c b/net/netfilter/nf_conntrack_ecache.c index 9df159448b897..cc8d8e85169fb 100644 --- a/net/netfilter/nf_conntrack_ecache.c +++ b/net/netfilter/nf_conntrack_ecache.c @@ -77,7 +77,7 @@ static enum retry_state ecache_work_evict_list(struct nf_conntrack_net *cnet) hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode, &evicted_list); - if (time_after(stop, jiffies)) { + if (time_after(jiffies, stop)) { ret = STATE_RESTART; break; } diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index 8e943efbdf0a5..38630c5e006f0 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -43,6 +43,24 @@ unsigned int nf_ct_expect_max __read_mostly; static struct kmem_cache *nf_ct_expect_cachep __read_mostly; static siphash_aligned_key_t nf_ct_expect_hashrnd; +void nf_ct_expectation_gc(struct nf_conn_help *master_help) +{ + struct nf_conntrack_expect *exp; + struct hlist_node *next; + + if (hlist_empty(&master_help->expectations)) + return; + + spin_lock_bh(&nf_conntrack_expect_lock); + hlist_for_each_entry_safe(exp, next, &master_help->expectations, lnode) { + if (!nf_ct_exp_is_expired(exp)) + continue; + + nf_ct_unlink_expect(exp); + } + spin_unlock_bh(&nf_conntrack_expect_lock); +} + /* nf_conntrack_expect helper functions */ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, u32 portid, int report) @@ -52,8 +70,6 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, struct nf_conntrack_net *cnet; lockdep_nfct_expect_lock_held(); - WARN_ON(!master_help); - WARN_ON(timer_pending(&exp->timeout)); hlist_del_rcu(&exp->hnode); @@ -61,7 +77,8 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, cnet->expect_count--; hlist_del_rcu(&exp->lnode); - master_help->expecting[exp->class]--; + if (master_help) + master_help->expecting[exp->class]--; nf_ct_expect_event_report(IPEXP_DESTROY, exp, portid, report); nf_ct_expect_put(exp); @@ -70,16 +87,6 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, } EXPORT_SYMBOL_GPL(nf_ct_unlink_expect_report); -static void nf_ct_expectation_timed_out(struct timer_list *t) -{ - struct nf_conntrack_expect *exp = timer_container_of(exp, t, timeout); - - spin_lock_bh(&nf_conntrack_expect_lock); - nf_ct_unlink_expect(exp); - spin_unlock_bh(&nf_conntrack_expect_lock); - nf_ct_expect_put(exp); -} - static unsigned int nf_ct_expect_dst_hash(const struct net *n, const struct nf_conntrack_tuple *tuple) { struct { @@ -117,19 +124,6 @@ nf_ct_exp_equal(const struct nf_conntrack_tuple *tuple, nf_ct_exp_zone_equal_any(i, zone); } -bool nf_ct_remove_expect(struct nf_conntrack_expect *exp) -{ - lockdep_nfct_expect_lock_held(); - - if (timer_delete(&exp->timeout)) { - nf_ct_unlink_expect(exp); - nf_ct_expect_put(exp); - return true; - } - return false; -} -EXPORT_SYMBOL_GPL(nf_ct_remove_expect); - struct nf_conntrack_expect * __nf_ct_expect_find(struct net *net, const struct nf_conntrack_zone *zone, @@ -144,6 +138,8 @@ __nf_ct_expect_find(struct net *net, h = nf_ct_expect_dst_hash(net, tuple); hlist_for_each_entry_rcu(i, &nf_ct_expect_hash[h], hnode) { + if (nf_ct_exp_is_expired(i)) + continue; if (nf_ct_exp_equal(tuple, i, zone, net)) return i; } @@ -178,6 +174,7 @@ nf_ct_find_expectation(struct net *net, { struct nf_conntrack_net *cnet = nf_ct_pernet(net); struct nf_conntrack_expect *i, *exp = NULL; + struct hlist_node *next; unsigned int h; lockdep_nfct_expect_lock_held(); @@ -186,7 +183,11 @@ nf_ct_find_expectation(struct net *net, return NULL; h = nf_ct_expect_dst_hash(net, tuple); - hlist_for_each_entry(i, &nf_ct_expect_hash[h], hnode) { + hlist_for_each_entry_safe(i, next, &nf_ct_expect_hash[h], hnode) { + if (nf_ct_exp_is_expired(i)) { + nf_ct_unlink_expect(i); + continue; + } if (!(i->flags & NF_CT_EXPECT_INACTIVE) && nf_ct_exp_equal(tuple, i, zone, net)) { exp = i; @@ -196,13 +197,16 @@ nf_ct_find_expectation(struct net *net, if (!exp) return NULL; + if (!refcount_inc_not_zero(&exp->use)) + return NULL; + /* If master is not in hash table yet (ie. packet hasn't left this machine yet), how can other end know about expected? Hence these are not the droids you are looking for (if master ct never got confirmed, we'd hold a reference to it and weird things would happen to future packets). */ if (!nf_ct_is_confirmed(exp->master)) - return NULL; + goto err_release_exp; /* Avoid race with other CPUs, that for exp->master ct, is * about to invoke ->destroy(), or nf_ct_delete() via timeout @@ -214,18 +218,17 @@ nf_ct_find_expectation(struct net *net, */ if (unlikely(nf_ct_is_dying(exp->master) || !refcount_inc_not_zero(&exp->master->ct_general.use))) - return NULL; + goto err_release_exp; - if (exp->flags & NF_CT_EXPECT_PERMANENT || !unlink) { - refcount_inc(&exp->use); + if (exp->flags & NF_CT_EXPECT_PERMANENT || !unlink) return exp; - } else if (timer_delete(&exp->timeout)) { - nf_ct_unlink_expect(exp); - return exp; - } - /* Undo exp->master refcnt increase, if timer_delete() failed */ - nf_ct_put(exp->master); + nf_ct_unlink_expect(exp); + + return exp; + +err_release_exp: + nf_ct_expect_put(exp); return NULL; } @@ -241,9 +244,8 @@ void nf_ct_remove_expectations(struct nf_conn *ct) return; spin_lock_bh(&nf_conntrack_expect_lock); - hlist_for_each_entry_safe(exp, next, &help->expectations, lnode) { - nf_ct_remove_expect(exp); - } + hlist_for_each_entry_safe(exp, next, &help->expectations, lnode) + nf_ct_unlink_expect(exp); spin_unlock_bh(&nf_conntrack_expect_lock); } EXPORT_SYMBOL_GPL(nf_ct_remove_expectations); @@ -292,7 +294,7 @@ static bool master_matches(const struct nf_conntrack_expect *a, void nf_ct_unexpect_related(struct nf_conntrack_expect *exp) { spin_lock_bh(&nf_conntrack_expect_lock); - nf_ct_remove_expect(exp); + WRITE_ONCE(exp->flags, exp->flags | NF_CT_EXPECT_DEAD); spin_unlock_bh(&nf_conntrack_expect_lock); } EXPORT_SYMBOL_GPL(nf_ct_unexpect_related); @@ -308,6 +310,7 @@ struct nf_conntrack_expect *nf_ct_expect_alloc(struct nf_conn *me) if (!new) return NULL; + new->timeout = nfct_time_stamp; new->master = me; refcount_set(&new->use, 1); return new; @@ -352,6 +355,8 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, exp->tuple.src.l3num = family; exp->tuple.dst.protonum = proto; + exp->master_tuple = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; + if (saddr) { memcpy(&exp->tuple.src.u3, saddr, len); if (sizeof(exp->tuple.src.u3) > len) @@ -405,25 +410,20 @@ void nf_ct_expect_put(struct nf_conntrack_expect *exp) } EXPORT_SYMBOL_GPL(nf_ct_expect_put); -static void nf_ct_expect_insert(struct nf_conntrack_expect *exp) +static void nf_ct_expect_insert(struct nf_conntrack_expect *exp, + struct nf_conn_help *master_help) { struct nf_conntrack_net *cnet; - struct nf_conn_help *master_help = nfct_help(exp->master); struct nf_conntrack_helper *helper; struct net *net = nf_ct_exp_net(exp); unsigned int h = nf_ct_expect_dst_hash(net, &exp->tuple); - /* two references : one for hash insert, one for the timer */ - refcount_add(2, &exp->use); + refcount_inc(&exp->use); - timer_setup(&exp->timeout, nf_ct_expectation_timed_out, 0); helper = rcu_dereference_protected(master_help->helper, lockdep_is_held(&nf_conntrack_expect_lock)); - if (helper) { - exp->timeout.expires = jiffies + - helper->expect_policy[exp->class].timeout * HZ; - } - add_timer(&exp->timeout); + if (helper) + exp->timeout += helper->expect_policy[exp->class].timeout * HZ; hlist_add_head_rcu(&exp->lnode, &master_help->expectations); master_help->expecting[exp->class]++; @@ -435,30 +435,35 @@ static void nf_ct_expect_insert(struct nf_conntrack_expect *exp) NF_CT_STAT_INC(net, expect_create); } -/* Race with expectations being used means we could have none to find; OK. */ -static void evict_oldest_expect(struct nf_conn *master, - struct nf_conntrack_expect *new) +static void evict_oldest_expect(struct nf_conn_help *master_help, + struct nf_conntrack_expect *new, + const struct nf_conntrack_expect_policy *p) { - struct nf_conn_help *master_help = nfct_help(master); struct nf_conntrack_expect *exp, *last = NULL; + struct hlist_node *next; - hlist_for_each_entry(exp, &master_help->expectations, lnode) { + hlist_for_each_entry_safe(exp, next, &master_help->expectations, lnode) { + if (nf_ct_exp_is_expired(exp)) { + nf_ct_unlink_expect(exp); + continue; + } if (exp->class == new->class) last = exp; } - if (last) - nf_ct_remove_expect(last); + /* Still worth to evict oldest expectation after garbage collection? */ + if (last && + master_help->expecting[last->class] >= p->max_expected) + nf_ct_unlink_expect(last); } static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, + struct nf_conn_help *master_help, unsigned int flags) { const struct nf_conntrack_expect_policy *p; struct nf_conntrack_expect *i; struct nf_conntrack_net *cnet; - struct nf_conn *master = expect->master; - struct nf_conn_help *master_help = nfct_help(master); struct nf_conntrack_helper *helper; struct net *net = nf_ct_exp_net(expect); struct hlist_node *next; @@ -467,20 +472,20 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, lockdep_nfct_expect_lock_held(); - if (!master_help) { - ret = -ESHUTDOWN; - goto out; - } h = nf_ct_expect_dst_hash(net, &expect->tuple); hlist_for_each_entry_safe(i, next, &nf_ct_expect_hash[h], hnode) { + if (nf_ct_exp_is_expired(i)) { + nf_ct_unlink_expect(i); + continue; + } if (master_matches(i, expect, flags) && expect_matches(i, expect)) { if (i->class != expect->class || i->master != expect->master) return -EALREADY; - if (nf_ct_remove_expect(i)) - break; + nf_ct_unlink_expect(i); + break; } else if (expect_clash(i, expect)) { ret = -EBUSY; goto out; @@ -491,15 +496,15 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, lockdep_is_held(&nf_conntrack_expect_lock)); if (helper) { p = &helper->expect_policy[expect->class]; - if (p->max_expected && - master_help->expecting[expect->class] >= p->max_expected) { - evict_oldest_expect(master, expect); - if (master_help->expecting[expect->class] - >= p->max_expected) { - ret = -EMFILE; - goto out; - } - } + if (master_help->expecting[expect->class] >= p->max_expected) + evict_oldest_expect(master_help, expect, p); + } else { + const struct nf_conntrack_expect_policy default_exp_policy = { + .max_expected = NF_CT_EXPECT_MAX_CNT, + }; + + if (master_help->expecting[expect->class] >= default_exp_policy.max_expected) + evict_oldest_expect(master_help, expect, &default_exp_policy); } cnet = nf_ct_pernet(net); @@ -514,14 +519,21 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, int nf_ct_expect_related_report(struct nf_conntrack_expect *expect, u32 portid, int report, unsigned int flags) { + struct nf_conn_help *master_help; int ret; spin_lock_bh(&nf_conntrack_expect_lock); - ret = __nf_ct_expect_check(expect, flags); + master_help = nfct_help(expect->master); + if (!master_help) { + ret = -ESHUTDOWN; + goto out; + } + + ret = __nf_ct_expect_check(expect, master_help, flags); if (ret < 0) goto out; - nf_ct_expect_insert(expect); + nf_ct_expect_insert(expect, master_help); nf_ct_expect_event_report(IPEXP_NEW, expect, portid, report); spin_unlock_bh(&nf_conntrack_expect_lock); @@ -546,10 +558,8 @@ void nf_ct_expect_iterate_destroy(bool (*iter)(struct nf_conntrack_expect *e, vo hlist_for_each_entry_safe(exp, next, &nf_ct_expect_hash[i], hnode) { - if (iter(exp, data) && timer_delete(&exp->timeout)) { + if (iter(exp, data)) nf_ct_unlink_expect(exp); - nf_ct_expect_put(exp); - } } } @@ -576,10 +586,8 @@ void nf_ct_expect_iterate_net(struct net *net, if (!net_eq(nf_ct_exp_net(exp), net)) continue; - if (iter(exp, data) && timer_delete(&exp->timeout)) { + if (iter(exp, data)) nf_ct_unlink_expect_report(exp, portid, report); - nf_ct_expect_put(exp); - } } } @@ -656,17 +664,17 @@ static int exp_seq_show(struct seq_file *s, void *v) struct net *net = seq_file_net(s); struct hlist_node *n = v; char *delim = ""; + __s32 timeout; expect = hlist_entry(n, struct nf_conntrack_expect, hnode); if (!net_eq(nf_ct_exp_net(expect), net)) return 0; + if (nf_ct_exp_is_expired(expect)) + return 0; - if (expect->timeout.function) - seq_printf(s, "%ld ", timer_pending(&expect->timeout) - ? (long)(expect->timeout.expires - jiffies)/HZ : 0); - else - seq_puts(s, "- "); + timeout = (__s32)(READ_ONCE(expect->timeout) - nfct_time_stamp) / HZ; + seq_printf(s, "%d ", timeout > 0 ? timeout : 0); seq_printf(s, "l3proto = %u proto=%u ", expect->tuple.src.l3num, expect->tuple.dst.protonum); diff --git a/net/netfilter/nf_conntrack_extend.c b/net/netfilter/nf_conntrack_extend.c index dd62cc12e7750..0da105e1ded93 100644 --- a/net/netfilter/nf_conntrack_extend.c +++ b/net/netfilter/nf_conntrack_extend.c @@ -27,8 +27,6 @@ #define NF_CT_EXT_PREALLOC 128u /* conntrack events are on by default */ -atomic_t nf_conntrack_ext_genid __read_mostly = ATOMIC_INIT(1); - static const u8 nf_ct_ext_type_len[NF_CT_EXT_NUM] = { [NF_CT_EXT_HELPER] = sizeof(struct nf_conn_help), #if IS_ENABLED(CONFIG_NF_NAT) @@ -118,10 +116,8 @@ void *nf_ct_ext_add(struct nf_conn *ct, enum nf_ct_ext_id id, gfp_t gfp) if (!new) return NULL; - if (!ct->ext) { + if (!ct->ext) memset(new->offset, 0, sizeof(new->offset)); - new->gen_id = atomic_read(&nf_conntrack_ext_genid); - } new->offset[id] = newoff; new->len = newlen; @@ -131,29 +127,3 @@ void *nf_ct_ext_add(struct nf_conn *ct, enum nf_ct_ext_id id, gfp_t gfp) return (void *)new + newoff; } EXPORT_SYMBOL(nf_ct_ext_add); - -/* Use nf_ct_ext_find wrapper. This is only useful for unconfirmed entries. */ -void *__nf_ct_ext_find(const struct nf_ct_ext *ext, u8 id) -{ - unsigned int gen_id = atomic_read(&nf_conntrack_ext_genid); - unsigned int this_id = READ_ONCE(ext->gen_id); - - if (!__nf_ct_ext_exist(ext, id)) - return NULL; - - if (this_id == 0 || ext->gen_id == gen_id) - return (void *)ext + ext->offset[id]; - - return NULL; -} -EXPORT_SYMBOL(__nf_ct_ext_find); - -void nf_ct_ext_bump_genid(void) -{ - unsigned int value = atomic_inc_return(&nf_conntrack_ext_genid); - - if (value == UINT_MAX) - atomic_set(&nf_conntrack_ext_genid, 1); - - msleep(HZ); -} diff --git a/net/netfilter/nf_conntrack_ftp.c b/net/netfilter/nf_conntrack_ftp.c index de83bf9e6c61a..6e3c06de8a6eb 100644 --- a/net/netfilter/nf_conntrack_ftp.c +++ b/net/netfilter/nf_conntrack_ftp.c @@ -381,6 +381,9 @@ static int help(struct sk_buff *skb, int found = 0, ends_in_nl; nf_nat_ftp_hook_fn *nf_nat_ftp; + if (!ct_ftp_info) + return NF_DROP; + /* Until there's been traffic both ways, don't look in packets. */ if (ctinfo != IP_CT_ESTABLISHED && ctinfo != IP_CT_ESTABLISHED_REPLY) { @@ -542,6 +545,9 @@ static int nf_ct_ftp_from_nlattr(struct nlattr *attr, struct nf_conn *ct) { struct nf_ct_ftp_master *ftp = nfct_help_data(ct); + if (!ftp) + return -ENOENT; + /* This conntrack has been injected from user-space, always pick up * sequence tracking. Otherwise, the first FTP command after the * failover breaks. @@ -552,6 +558,7 @@ static int nf_ct_ftp_from_nlattr(struct nlattr *attr, struct nf_conn *ct) } static struct nf_conntrack_helper ftp[MAX_PORTS * 2] __read_mostly; +static struct nf_conntrack_helper *ftp_ptr[MAX_PORTS * 2] __read_mostly; static const struct nf_conntrack_expect_policy ftp_exp_policy = { .max_expected = 1, @@ -560,7 +567,7 @@ static const struct nf_conntrack_expect_policy ftp_exp_policy = { static void __exit nf_conntrack_ftp_fini(void) { - nf_conntrack_helpers_unregister(ftp, ports_c * 2); + nf_conntrack_helpers_unregister(ftp_ptr, ports_c * 2); } static int __init nf_conntrack_ftp_init(void) @@ -585,7 +592,7 @@ static int __init nf_conntrack_ftp_init(void) nf_ct_ftp_from_nlattr, THIS_MODULE); } - ret = nf_conntrack_helpers_register(ftp, ports_c * 2); + ret = nf_conntrack_helpers_register(ftp, ports_c * 2, ftp_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index b2fe6554b9cf4..24931e379985b 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -76,6 +76,9 @@ static int get_tpkt_data(struct sk_buff *skb, unsigned int protoff, int tpktlen; int tpktoff; + if (!info) + return 0; + /* Get TCP header */ th = skb_header_pointer(skb, protoff, sizeof(_tcph), &_tcph); if (th == NULL) @@ -577,14 +580,8 @@ static const struct nf_conntrack_expect_policy h245_exp_policy = { .timeout = 240, }; -static struct nf_conntrack_helper nf_conntrack_helper_h245 __read_mostly = { - .name = "H.245", - .me = THIS_MODULE, - .tuple.src.l3num = AF_UNSPEC, - .tuple.dst.protonum = IPPROTO_UDP, - .help = h245_help, - .expect_policy = &h245_exp_policy, -}; +static struct nf_conntrack_helper nf_conntrack_helper_h245 __read_mostly; +static struct nf_conntrack_helper *nf_conntrack_helper_h245_ptr __read_mostly; int get_h225_addr(struct nf_conn *ct, unsigned char *data, TransportAddress *taddr, @@ -643,7 +640,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - rcu_assign_pointer(exp->assign_helper, &nf_conntrack_helper_h245); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_h245_ptr); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -732,6 +729,9 @@ static int callforward_do_filter(struct net *net, } +static struct nf_conntrack_helper nf_conntrack_helper_q931[2] __read_mostly; +static struct nf_conntrack_helper *nf_conntrack_helper_q931_ptr[2] __read_mostly; + static int expect_callforwarding(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo, @@ -767,7 +767,7 @@ static int expect_callforwarding(struct sk_buff *skb, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); - rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931_ptr[0]); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -1140,27 +1140,6 @@ static const struct nf_conntrack_expect_policy q931_exp_policy = { .timeout = 240, }; -static struct nf_conntrack_helper nf_conntrack_helper_q931[] __read_mostly = { - { - .name = "Q.931", - .me = THIS_MODULE, - .tuple.src.l3num = AF_INET, - .tuple.src.u.tcp.port = cpu_to_be16(Q931_PORT), - .tuple.dst.protonum = IPPROTO_TCP, - .help = q931_help, - .expect_policy = &q931_exp_policy, - }, - { - .name = "Q.931", - .me = THIS_MODULE, - .tuple.src.l3num = AF_INET6, - .tuple.src.u.tcp.port = cpu_to_be16(Q931_PORT), - .tuple.dst.protonum = IPPROTO_TCP, - .help = q931_help, - .expect_policy = &q931_exp_policy, - }, -}; - static unsigned char *get_udp_data(struct sk_buff *skb, unsigned int protoff, int *datalen) { @@ -1215,6 +1194,9 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, union nf_inet_addr addr; struct nf_conntrack_expect *exp; + if (!info) + return -1; + /* Look for the first related address */ for (i = 0; i < count; i++) { if (get_h225_addr(ct, *data, &taddr[i], &addr, &port) && @@ -1234,7 +1216,7 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3 : NULL, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931_ptr[0]); exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */ nathook = rcu_dereference(nfct_h323_nat_hook); @@ -1275,6 +1257,9 @@ static int process_grq(struct sk_buff *skb, struct nf_conn *ct, return 0; } +static struct nf_conntrack_helper nf_conntrack_helper_ras[2] __read_mostly; +static struct nf_conntrack_helper *nf_conntrack_helper_ras_ptr[2] __read_mostly; + static int process_gcf(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo, unsigned int protoff, @@ -1306,7 +1291,7 @@ static int process_gcf(struct sk_buff *skb, struct nf_conn *ct, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_UDP, NULL, &port); - rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_ras); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_ras_ptr[0]); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect RAS "); @@ -1328,6 +1313,9 @@ static int process_rrq(struct sk_buff *skb, struct nf_conn *ct, const struct nfct_h323_nat_hooks *nathook; int ret; + if (!info) + return -1; + pr_debug("nf_ct_ras: RRQ\n"); ret = expect_q931(skb, ct, ctinfo, protoff, data, @@ -1366,6 +1354,9 @@ static int process_rcf(struct sk_buff *skb, struct nf_conn *ct, int ret; struct nf_conntrack_expect *exp; + if (!info) + return -1; + pr_debug("nf_ct_ras: RCF\n"); nathook = rcu_dereference(nfct_h323_nat_hook); @@ -1397,8 +1388,8 @@ static int process_rcf(struct sk_buff *skb, struct nf_conn *ct, "timeout to %u seconds for", info->timeout); nf_ct_dump_tuple(&exp->tuple); - mod_timer_pending(&exp->timeout, - jiffies + info->timeout * HZ); + WRITE_ONCE(exp->timeout, + nfct_time_stamp + (info->timeout * HZ)); } spin_unlock_bh(&nf_conntrack_expect_lock); } @@ -1416,6 +1407,9 @@ static int process_urq(struct sk_buff *skb, struct nf_conn *ct, int dir = CTINFO2DIR(ctinfo); int ret; + if (!info) + return -1; + pr_debug("nf_ct_ras: URQ\n"); nathook = rcu_dereference(nfct_h323_nat_hook); @@ -1450,6 +1444,9 @@ static int process_arq(struct sk_buff *skb, struct nf_conn *ct, __be16 port; union nf_inet_addr addr; + if (!info) + return 0; + pr_debug("nf_ct_ras: ARQ\n"); nathook = rcu_dereference(nfct_h323_nat_hook); @@ -1523,7 +1520,7 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931_ptr[0]); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); @@ -1577,7 +1574,7 @@ static int process_lcf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931_ptr[0]); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); @@ -1711,59 +1708,57 @@ static const struct nf_conntrack_expect_policy ras_exp_policy = { .timeout = 240, }; -static struct nf_conntrack_helper nf_conntrack_helper_ras[] __read_mostly = { - { - .name = "RAS", - .me = THIS_MODULE, - .tuple.src.l3num = AF_INET, - .tuple.src.u.udp.port = cpu_to_be16(RAS_PORT), - .tuple.dst.protonum = IPPROTO_UDP, - .help = ras_help, - .expect_policy = &ras_exp_policy, - }, - { - .name = "RAS", - .me = THIS_MODULE, - .tuple.src.l3num = AF_INET6, - .tuple.src.u.udp.port = cpu_to_be16(RAS_PORT), - .tuple.dst.protonum = IPPROTO_UDP, - .help = ras_help, - .expect_policy = &ras_exp_policy, - }, -}; - static int __init h323_helper_init(void) { int ret; - ret = nf_conntrack_helper_register(&nf_conntrack_helper_h245); + nf_ct_helper_init(&nf_conntrack_helper_ras[0], AF_INET, IPPROTO_UDP, + "RAS", RAS_PORT, RAS_PORT, RAS_PORT, + &ras_exp_policy, 0, ras_help, NULL, THIS_MODULE); + nf_ct_helper_init(&nf_conntrack_helper_ras[1], AF_INET6, IPPROTO_UDP, + "RAS", RAS_PORT, RAS_PORT, RAS_PORT, + &ras_exp_policy, 0, ras_help, NULL, THIS_MODULE); + nf_ct_helper_init(&nf_conntrack_helper_h245, AF_UNSPEC, IPPROTO_UDP, + "H.245", 0, 0, 0, + &h245_exp_policy, 0, h245_help, NULL, THIS_MODULE); + nf_ct_helper_init(&nf_conntrack_helper_q931[0], AF_INET, IPPROTO_TCP, + "Q.931", Q931_PORT, Q931_PORT, Q931_PORT, + &q931_exp_policy, 0, q931_help, NULL, THIS_MODULE); + nf_ct_helper_init(&nf_conntrack_helper_q931[1], AF_INET6, IPPROTO_TCP, + "Q.931", Q931_PORT, Q931_PORT, Q931_PORT, + &q931_exp_policy, 0, q931_help, NULL, THIS_MODULE); + + ret = nf_conntrack_helper_register(&nf_conntrack_helper_h245, + &nf_conntrack_helper_h245_ptr); if (ret < 0) return ret; ret = nf_conntrack_helpers_register(nf_conntrack_helper_q931, - ARRAY_SIZE(nf_conntrack_helper_q931)); + ARRAY_SIZE(nf_conntrack_helper_q931), + nf_conntrack_helper_q931_ptr); if (ret < 0) goto err1; ret = nf_conntrack_helpers_register(nf_conntrack_helper_ras, - ARRAY_SIZE(nf_conntrack_helper_ras)); + ARRAY_SIZE(nf_conntrack_helper_ras), + nf_conntrack_helper_ras_ptr); if (ret < 0) goto err2; return 0; err2: - nf_conntrack_helpers_unregister(nf_conntrack_helper_q931, - ARRAY_SIZE(nf_conntrack_helper_q931)); + nf_conntrack_helpers_unregister(nf_conntrack_helper_q931_ptr, + ARRAY_SIZE(nf_conntrack_helper_q931_ptr)); err1: - nf_conntrack_helper_unregister(&nf_conntrack_helper_h245); + nf_conntrack_helper_unregister(nf_conntrack_helper_h245_ptr); return ret; } static void __exit h323_helper_exit(void) { - nf_conntrack_helpers_unregister(nf_conntrack_helper_ras, + nf_conntrack_helpers_unregister(nf_conntrack_helper_ras_ptr, ARRAY_SIZE(nf_conntrack_helper_ras)); - nf_conntrack_helpers_unregister(nf_conntrack_helper_q931, + nf_conntrack_helpers_unregister(nf_conntrack_helper_q931_ptr, ARRAY_SIZE(nf_conntrack_helper_q931)); - nf_conntrack_helper_unregister(&nf_conntrack_helper_h245); + nf_conntrack_helper_unregister(nf_conntrack_helper_h245_ptr); } static void __exit nf_conntrack_h323_fini(void) diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index 2c5a717355612..d1392b8e09af8 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -180,10 +180,10 @@ nf_ct_helper_ext_add(struct nf_conn *ct, gfp_t gfp) struct nf_conn_help *help; help = nf_ct_ext_add(ct, NF_CT_EXT_HELPER, gfp); - if (help) + if (help) { + __set_bit(IPS_HELPER_BIT, &ct->status); INIT_HLIST_HEAD(&help->expectations); - else - pr_debug("failed to add helper extension area"); + } return help; } EXPORT_SYMBOL_GPL(nf_ct_helper_ext_add); @@ -202,10 +202,8 @@ int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl, return 0; help = nfct_help(tmpl); - if (help != NULL) { + if (help) helper = rcu_dereference(help->helper); - set_bit(IPS_HELPER_BIT, &ct->status); - } help = nfct_help(ct); @@ -366,22 +364,26 @@ void nf_ct_helper_log(struct sk_buff *skb, const struct nf_conn *ct, } EXPORT_SYMBOL_GPL(nf_ct_helper_log); -int nf_conntrack_helper_register(struct nf_conntrack_helper *me) +int __nf_conntrack_helper_register(struct nf_conntrack_helper *me) { struct nf_conntrack_tuple_mask mask = { .src.u.all = htons(0xFFFF) }; unsigned int h = helper_hash(&me->tuple); struct nf_conntrack_helper *cur; int ret = 0, i; - BUG_ON(me->expect_policy == NULL); BUG_ON(me->expect_class_max >= NF_CT_MAX_EXPECT_CLASSES); BUG_ON(strlen(me->name) > NF_CT_HELPER_NAME_LEN - 1); if (!nf_ct_helper_hash) return -ENOENT; - if (me->expect_policy->max_expected > NF_CT_EXPECT_MAX_CNT) - return -EINVAL; + for (i = 0; i <= me->expect_class_max; i++) { + if (!me->expect_policy[i].max_expected) + me->expect_policy[i].max_expected = NF_CT_EXPECT_MAX_CNT; + + if (me->expect_policy[i].max_expected > NF_CT_EXPECT_MAX_CNT) + return -EINVAL; + } mutex_lock(&nf_ct_helper_mutex); for (i = 0; i < nf_ct_helper_hsize; i++) { @@ -413,6 +415,33 @@ int nf_conntrack_helper_register(struct nf_conntrack_helper *me) mutex_unlock(&nf_ct_helper_mutex); return ret; } +EXPORT_SYMBOL_GPL(__nf_conntrack_helper_register); + +int nf_conntrack_helper_register(struct nf_conntrack_helper *me, + struct nf_conntrack_helper **helper_ptr) +{ + struct nf_conntrack_helper *new_helper; + int err; + + new_helper = kzalloc_obj(*new_helper, GFP_KERNEL_ACCOUNT); + if (!new_helper) + return -ENOMEM; + + memcpy(new_helper, me, sizeof(*new_helper)); + *helper_ptr = new_helper; + + err = __nf_conntrack_helper_register(new_helper); + if (err < 0) + goto err_helper; + + return 0; + +err_helper: + *helper_ptr = NULL; + kfree(new_helper); + + return err; +} EXPORT_SYMBOL_GPL(nf_conntrack_helper_register); static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data) @@ -449,6 +478,7 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) * last step, this ensures rcu readers of exp->helper are done. * No need for another synchronize_rcu() here. */ + kfree(me); } EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister); @@ -464,11 +494,12 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, struct nf_conn *ct), struct module *module) { + memset(helper, 0, sizeof(*helper)); + helper->tuple.src.l3num = l3num; helper->tuple.dst.protonum = protonum; helper->tuple.src.u.all = htons(spec_port); - helper->expect_policy = exp_pol; - helper->expect_class_max = expect_class_max; + helper->help = help; helper->from_nlattr = from_nlattr; helper->me = module; @@ -479,34 +510,57 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, snprintf(helper->name, sizeof(helper->name), "%s", name); else snprintf(helper->name, sizeof(helper->name), "%s-%u", name, id); + + if (WARN_ON_ONCE(expect_class_max >= NF_CT_MAX_EXPECT_CLASSES)) + return; + + memcpy(helper->expect_policy, exp_pol, + (expect_class_max + 1) * sizeof(*exp_pol)); + helper->expect_class_max = expect_class_max; } EXPORT_SYMBOL_GPL(nf_ct_helper_init); int nf_conntrack_helpers_register(struct nf_conntrack_helper *helper, - unsigned int n) + unsigned int n, struct nf_conntrack_helper **helper_ptr) { + struct nf_conntrack_helper *new_helper; unsigned int i; int err = 0; for (i = 0; i < n; i++) { - err = nf_conntrack_helper_register(&helper[i]); - if (err < 0) + new_helper = kzalloc_obj(*new_helper, GFP_KERNEL_ACCOUNT); + if (!new_helper) { + err = -ENOMEM; goto err; + } + + memcpy(new_helper, &helper[i], sizeof(*new_helper)); + helper_ptr[i] = new_helper; + + err = __nf_conntrack_helper_register(new_helper); + if (err < 0) { + helper_ptr[i] = NULL; + goto err_helper; + } } return err; +err_helper: + kfree(new_helper); err: if (i > 0) - nf_conntrack_helpers_unregister(helper, i); + nf_conntrack_helpers_unregister(helper_ptr, i); return err; } EXPORT_SYMBOL_GPL(nf_conntrack_helpers_register); -void nf_conntrack_helpers_unregister(struct nf_conntrack_helper *helper, - unsigned int n) +void nf_conntrack_helpers_unregister(struct nf_conntrack_helper **helper, + unsigned int n) { - while (n-- > 0) - nf_conntrack_helper_unregister(&helper[n]); + while (n-- > 0) { + nf_conntrack_helper_unregister(helper[n]); + helper[n] = NULL; + } } EXPORT_SYMBOL_GPL(nf_conntrack_helpers_unregister); diff --git a/net/netfilter/nf_conntrack_irc.c b/net/netfilter/nf_conntrack_irc.c index 2ebe4cb47cf61..0c117b8492e99 100644 --- a/net/netfilter/nf_conntrack_irc.c +++ b/net/netfilter/nf_conntrack_irc.c @@ -59,7 +59,7 @@ static const char *const dccprotos[] = { /* tries to get the ip_addr and port out of a dcc command * return value: -1 on failure, 0 on success * data pointer to first byte of DCC command data - * data_end pointer to last byte of dcc command data + * data_end one past end of data * ip returns parsed ip of dcc command * port returns parsed port of dcc command * ad_beg_p returns pointer to first byte of addr data @@ -77,10 +77,10 @@ static int parse_dcc(char *data, const char *data_end, __be32 *ip, /* Make sure we have a newline character within the packet boundaries * because simple_strtoul parses until the first invalid character. */ - for (tmp = data; tmp <= data_end; tmp++) + for (tmp = data; tmp < data_end; tmp++) if (*tmp == '\n') break; - if (tmp > data_end || *tmp != '\n') + if (tmp >= data_end || *tmp != '\n') return -1; *ad_beg_p = data; @@ -255,6 +255,7 @@ static int help(struct sk_buff *skb, unsigned int protoff, } static struct nf_conntrack_helper irc[MAX_PORTS] __read_mostly; +static struct nf_conntrack_helper *irc_ptr[MAX_PORTS] __read_mostly; static struct nf_conntrack_expect_policy irc_exp_policy; static int __init nf_conntrack_irc_init(void) @@ -289,7 +290,7 @@ static int __init nf_conntrack_irc_init(void) 0, help, NULL, THIS_MODULE); } - ret = nf_conntrack_helpers_register(&irc[0], ports_c); + ret = nf_conntrack_helpers_register(&irc[0], ports_c, irc_ptr); if (ret) { pr_err("failed to register helpers\n"); kfree(irc_buffer); @@ -301,7 +302,7 @@ static int __init nf_conntrack_irc_init(void) static void __exit nf_conntrack_irc_fini(void) { - nf_conntrack_helpers_unregister(irc, ports_c); + nf_conntrack_helpers_unregister(irc_ptr, ports_c); kfree(irc_buffer); } diff --git a/net/netfilter/nf_conntrack_netbios_ns.c b/net/netfilter/nf_conntrack_netbios_ns.c index 55415f011943d..89d1cf7d6512e 100644 --- a/net/netfilter/nf_conntrack_netbios_ns.c +++ b/net/netfilter/nf_conntrack_netbios_ns.c @@ -44,27 +44,25 @@ static int netbios_ns_help(struct sk_buff *skb, unsigned int protoff, return nf_conntrack_broadcast_help(skb, ct, ctinfo, timeout); } -static struct nf_conntrack_helper helper __read_mostly = { - .name = HELPER_NAME, - .tuple.src.l3num = NFPROTO_IPV4, - .tuple.src.u.udp.port = cpu_to_be16(NMBD_PORT), - .tuple.dst.protonum = IPPROTO_UDP, - .me = THIS_MODULE, - .help = netbios_ns_help, - .expect_policy = &exp_policy, -}; +static struct nf_conntrack_helper helper __read_mostly; +static struct nf_conntrack_helper *helper_ptr __read_mostly; static int __init nf_conntrack_netbios_ns_init(void) { NF_CT_HELPER_BUILD_BUG_ON(0); exp_policy.timeout = timeout; - return nf_conntrack_helper_register(&helper); + + nf_ct_helper_init(&helper, AF_INET, IPPROTO_UDP, HELPER_NAME, + NMBD_PORT, NMBD_PORT, NMBD_PORT, + &exp_policy, 0, netbios_ns_help, NULL, THIS_MODULE); + + return nf_conntrack_helper_register(&helper, &helper_ptr); } static void __exit nf_conntrack_netbios_ns_fini(void) { - nf_conntrack_helper_unregister(&helper); + nf_conntrack_helper_unregister(helper_ptr); } module_init(nf_conntrack_netbios_ns_init); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index befa7e83ee49f..b11b8b7f31aaa 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -3006,8 +3006,7 @@ static int ctnetlink_exp_dump_expect(struct sk_buff *skb, const struct nf_conntrack_expect *exp) { - struct nf_conn *master = exp->master; - long timeout = ((long)exp->timeout.expires - (long)jiffies) / HZ; + __s32 timeout = (__s32)(READ_ONCE(exp->timeout) - nfct_time_stamp) / HZ; struct nf_conntrack_helper *helper; #if IS_ENABLED(CONFIG_NF_NAT) struct nlattr *nest_parms; @@ -3022,9 +3021,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, goto nla_put_failure; if (ctnetlink_exp_dump_mask(skb, &exp->tuple, &exp->mask) < 0) goto nla_put_failure; - if (ctnetlink_exp_dump_tuple(skb, - &master->tuplehash[IP_CT_DIR_ORIGINAL].tuple, - CTA_EXPECT_MASTER) < 0) + if (ctnetlink_exp_dump_tuple(skb, &exp->master_tuple, CTA_EXPECT_MASTER) < 0) goto nla_put_failure; #if IS_ENABLED(CONFIG_NF_NAT) @@ -3037,9 +3034,9 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, if (nla_put_be32(skb, CTA_EXPECT_NAT_DIR, htonl(exp->dir))) goto nla_put_failure; - nat_tuple.src.l3num = nf_ct_l3num(master); + nat_tuple.src.l3num = exp->master_tuple.src.l3num; nat_tuple.src.u3 = exp->saved_addr; - nat_tuple.dst.protonum = nf_ct_protonum(master); + nat_tuple.dst.protonum = exp->master_tuple.dst.protonum; nat_tuple.src.u = exp->saved_proto; if (ctnetlink_exp_dump_tuple(skb, &nat_tuple, @@ -3170,10 +3167,13 @@ ctnetlink_exp_dump_table(struct sk_buff *skb, struct netlink_callback *cb) restart: hlist_for_each_entry_rcu(exp, &nf_ct_expect_hash[cb->args[0]], hnode) { + if (nf_ct_exp_is_expired(exp)) + continue; + if (l3proto && exp->tuple.src.l3num != l3proto) continue; - if (!net_eq(nf_ct_net(exp->master), net)) + if (!net_eq(nf_ct_exp_net(exp), net)) continue; if (cb->args[1]) { @@ -3448,11 +3448,8 @@ static int ctnetlink_del_expect(struct sk_buff *skb, } /* after list removal, usage count == 1 */ - if (timer_delete(&exp->timeout)) { - nf_ct_unlink_expect_report(exp, NETLINK_CB(skb).portid, - nlmsg_report(info->nlh)); - nf_ct_expect_put(exp); - } + nf_ct_unlink_expect_report(exp, NETLINK_CB(skb).portid, + nlmsg_report(info->nlh)); spin_unlock_bh(&nf_conntrack_expect_lock); /* have to put what we 'get' above. * after this line usage count == 0 */ @@ -3476,14 +3473,10 @@ static int ctnetlink_change_expect(struct nf_conntrack_expect *x, const struct nlattr * const cda[]) { - if (cda[CTA_EXPECT_TIMEOUT]) { - if (!timer_delete(&x->timeout)) - return -ETIME; + if (cda[CTA_EXPECT_TIMEOUT]) + WRITE_ONCE(x->timeout, nfct_time_stamp + + ntohl(nla_get_be32(cda[CTA_EXPECT_TIMEOUT])) * HZ); - x->timeout.expires = jiffies + - ntohl(nla_get_be32(cda[CTA_EXPECT_TIMEOUT])) * HZ; - add_timer(&x->timeout); - } return 0; } @@ -3585,6 +3578,7 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, #endif rcu_assign_pointer(exp->helper, helper); rcu_assign_pointer(exp->assign_helper, assign_helper); + exp->master_tuple = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; exp->tuple = *tuple; exp->mask.src.u3 = mask->src.u3; exp->mask.src.u.all = mask->src.u.all; diff --git a/net/netfilter/nf_conntrack_pptp.c b/net/netfilter/nf_conntrack_pptp.c index dc23e4181618a..776505a78e645 100644 --- a/net/netfilter/nf_conntrack_pptp.c +++ b/net/netfilter/nf_conntrack_pptp.c @@ -124,65 +124,6 @@ static void pptp_expectfn(struct nf_conn *ct, } } -static int destroy_sibling_or_exp(struct net *net, struct nf_conn *ct, - const struct nf_conntrack_tuple *t) -{ - const struct nf_conntrack_tuple_hash *h; - const struct nf_conntrack_zone *zone; - struct nf_conntrack_expect *exp; - struct nf_conn *sibling; - - pr_debug("trying to timeout ct or exp for tuple "); - nf_ct_dump_tuple(t); - - zone = nf_ct_zone(ct); - h = nf_conntrack_find_get(net, zone, t); - if (h) { - sibling = nf_ct_tuplehash_to_ctrack(h); - pr_debug("setting timeout of conntrack %p to 0\n", sibling); - sibling->proto.gre.timeout = 0; - sibling->proto.gre.stream_timeout = 0; - nf_ct_kill(sibling); - nf_ct_put(sibling); - return 1; - } else { - exp = nf_ct_expect_find_get(net, zone, t); - if (exp) { - pr_debug("unexpect_related of expect %p\n", exp); - nf_ct_unexpect_related(exp); - nf_ct_expect_put(exp); - return 1; - } - } - return 0; -} - -/* timeout GRE data connections */ -static void pptp_destroy_siblings(struct nf_conn *ct) -{ - struct net *net = nf_ct_net(ct); - const struct nf_ct_pptp_master *ct_pptp_info = nfct_help_data(ct); - struct nf_conntrack_tuple t; - - nf_ct_gre_keymap_destroy(ct); - - /* try original (pns->pac) tuple */ - memcpy(&t, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, sizeof(t)); - t.dst.protonum = IPPROTO_GRE; - t.src.u.gre.key = ct_pptp_info->pns_call_id; - t.dst.u.gre.key = ct_pptp_info->pac_call_id; - if (!destroy_sibling_or_exp(net, ct, &t)) - pr_debug("failed to timeout original pns->pac ct/exp\n"); - - /* try reply (pac->pns) tuple */ - memcpy(&t, &ct->tuplehash[IP_CT_DIR_REPLY].tuple, sizeof(t)); - t.dst.protonum = IPPROTO_GRE; - t.src.u.gre.key = ct_pptp_info->pac_call_id; - t.dst.u.gre.key = ct_pptp_info->pns_call_id; - if (!destroy_sibling_or_exp(net, ct, &t)) - pr_debug("failed to timeout reply pac->pns ct/exp\n"); -} - /* expect GRE connections (PNS->PAC and PAC->PNS direction) */ static int exp_gre(struct nf_conn *ct, __be16 callid, __be16 peer_callid) { @@ -257,6 +198,9 @@ pptp_inbound_pkt(struct sk_buff *skb, unsigned int protoff, u_int16_t msg; __be16 cid = 0, pcid = 0; + if (!info) + return NF_DROP; + msg = ntohs(ctlh->messageType); pr_debug("inbound control message %s\n", pptp_msg_name(msg)); @@ -343,7 +287,7 @@ pptp_inbound_pkt(struct sk_buff *skb, unsigned int protoff, info->cstate = PPTP_CALL_NONE; /* untrack this call id, unexpect GRE packets */ - pptp_destroy_siblings(ct); + gre_pptp_destroy_siblings(ct); break; case PPTP_WAN_ERROR_NOTIFY: @@ -384,6 +328,9 @@ pptp_outbound_pkt(struct sk_buff *skb, unsigned int protoff, u_int16_t msg; __be16 cid = 0, pcid = 0; + if (!info) + return NF_DROP; + msg = ntohs(ctlh->messageType); pr_debug("outbound control message %s\n", pptp_msg_name(msg)); @@ -502,6 +449,9 @@ conntrack_pptp_help(struct sk_buff *skb, unsigned int protoff, int ret; u_int16_t msg; + if (!info) + return NF_DROP; + #if IS_ENABLED(CONFIG_NF_NAT) if (!nf_ct_is_confirmed(ct) && (ct->status & IPS_NAT_MASK)) { struct nf_conn_nat *nat = nf_ct_ext_find(ct, NF_CT_EXT_NAT); @@ -582,27 +532,25 @@ static const struct nf_conntrack_expect_policy pptp_exp_policy = { }; /* control protocol helper */ -static struct nf_conntrack_helper pptp __read_mostly = { - .name = "pptp", - .me = THIS_MODULE, - .tuple.src.l3num = AF_INET, - .tuple.src.u.tcp.port = cpu_to_be16(PPTP_CONTROL_PORT), - .tuple.dst.protonum = IPPROTO_TCP, - .help = conntrack_pptp_help, - .destroy = pptp_destroy_siblings, - .expect_policy = &pptp_exp_policy, -}; +static struct nf_conntrack_helper pptp __read_mostly; +static struct nf_conntrack_helper *pptp_ptr __read_mostly; static int __init nf_conntrack_pptp_init(void) { NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_pptp_master)); - return nf_conntrack_helper_register(&pptp); + nf_ct_helper_init(&pptp, AF_INET, IPPROTO_TCP, + "pptp", PPTP_CONTROL_PORT, PPTP_CONTROL_PORT, PPTP_CONTROL_PORT, + &pptp_exp_policy, 0, conntrack_pptp_help, NULL, THIS_MODULE); + + pptp.destroy = gre_pptp_destroy_siblings; + + return nf_conntrack_helper_register(&pptp, &pptp_ptr); } static void __exit nf_conntrack_pptp_fini(void) { - nf_conntrack_helper_unregister(&pptp); + nf_conntrack_helper_unregister(pptp_ptr); } module_init(nf_conntrack_pptp_init); diff --git a/net/netfilter/nf_conntrack_proto_gre.c b/net/netfilter/nf_conntrack_proto_gre.c index 35e22082d65ac..616ab1e2fc5ed 100644 --- a/net/netfilter/nf_conntrack_proto_gre.c +++ b/net/netfilter/nf_conntrack_proto_gre.c @@ -131,6 +131,9 @@ bool nf_ct_gre_keymap_add(struct nf_conn *ct, struct nf_ct_gre_keymap *km_orig, *km_repl; bool ret = false; + if (!ct_pptp_info) + return false; + km_orig = kmalloc_obj(*km_orig, GFP_ATOMIC); if (!km_orig) return false; @@ -187,6 +190,9 @@ void nf_ct_gre_keymap_destroy(struct nf_conn *ct) struct nf_ct_pptp_master *ct_pptp_info = nfct_help_data(ct); enum ip_conntrack_dir dir; + if (!ct_pptp_info) + return; + pr_debug("entering for ct %p\n", ct); spin_lock_bh(&keymap_lock); @@ -349,6 +355,70 @@ gre_timeout_nla_policy[CTA_TIMEOUT_GRE_MAX+1] = { }; #endif /* CONFIG_NF_CONNTRACK_TIMEOUT */ +#if IS_ENABLED(CONFIG_NF_CONNTRACK_PPTP) +static int destroy_sibling_or_exp(struct net *net, struct nf_conn *ct, + const struct nf_conntrack_tuple *t) +{ + const struct nf_conntrack_tuple_hash *h; + const struct nf_conntrack_zone *zone; + struct nf_conntrack_expect *exp; + struct nf_conn *sibling; + + pr_debug("trying to timeout ct or exp for tuple "); + nf_ct_dump_tuple(t); + + zone = nf_ct_zone(ct); + h = nf_conntrack_find_get(net, zone, t); + if (h) { + sibling = nf_ct_tuplehash_to_ctrack(h); + pr_debug("setting timeout of conntrack %p to 0\n", sibling); + sibling->proto.gre.timeout = 0; + sibling->proto.gre.stream_timeout = 0; + nf_ct_kill(sibling); + nf_ct_put(sibling); + return 1; + } else { + exp = nf_ct_expect_find_get(net, zone, t); + if (exp) { + pr_debug("unexpect_related of expect %p\n", exp); + nf_ct_unexpect_related(exp); + nf_ct_expect_put(exp); + return 1; + } + } + return 0; +} + +void gre_pptp_destroy_siblings(struct nf_conn *ct) +{ + struct net *net = nf_ct_net(ct); + const struct nf_ct_pptp_master *ct_pptp_info = nfct_help_data(ct); + struct nf_conntrack_tuple t; + + if (!ct_pptp_info) + return; + + nf_ct_gre_keymap_destroy(ct); + + /* try original (pns->pac) tuple */ + memcpy(&t, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, sizeof(t)); + t.dst.protonum = IPPROTO_GRE; + t.src.u.gre.key = ct_pptp_info->pns_call_id; + t.dst.u.gre.key = ct_pptp_info->pac_call_id; + if (!destroy_sibling_or_exp(net, ct, &t)) + pr_debug("failed to timeout original pns->pac ct/exp\n"); + + /* try reply (pac->pns) tuple */ + memcpy(&t, &ct->tuplehash[IP_CT_DIR_REPLY].tuple, sizeof(t)); + t.dst.protonum = IPPROTO_GRE; + t.src.u.gre.key = ct_pptp_info->pac_call_id; + t.dst.u.gre.key = ct_pptp_info->pns_call_id; + if (!destroy_sibling_or_exp(net, ct, &t)) + pr_debug("failed to timeout reply pac->pns ct/exp\n"); +} +EXPORT_SYMBOL_GPL(gre_pptp_destroy_siblings); +#endif + void nf_conntrack_gre_init_net(struct net *net) { struct nf_gre_net *net_gre = gre_pernet(net); diff --git a/net/netfilter/nf_conntrack_sane.c b/net/netfilter/nf_conntrack_sane.c index 13dc421fc4f52..39085acf7a718 100644 --- a/net/netfilter/nf_conntrack_sane.c +++ b/net/netfilter/nf_conntrack_sane.c @@ -74,6 +74,9 @@ static int help(struct sk_buff *skb, struct sane_reply_net_start repl; } buf; + if (!ct_sane_info) + return NF_DROP; + /* Until there's been traffic both ways, don't look in packets. */ if (ctinfo != IP_CT_ESTABLISHED && ctinfo != IP_CT_ESTABLISHED_REPLY) @@ -167,6 +170,7 @@ static int help(struct sk_buff *skb, } static struct nf_conntrack_helper sane[MAX_PORTS * 2] __read_mostly; +static struct nf_conntrack_helper *sane_ptr[MAX_PORTS * 2] __read_mostly; static const struct nf_conntrack_expect_policy sane_exp_policy = { .max_expected = 1, @@ -175,7 +179,7 @@ static const struct nf_conntrack_expect_policy sane_exp_policy = { static void __exit nf_conntrack_sane_fini(void) { - nf_conntrack_helpers_unregister(sane, ports_c * 2); + nf_conntrack_helpers_unregister(sane_ptr, ports_c * 2); } static int __init nf_conntrack_sane_init(void) @@ -200,7 +204,7 @@ static int __init nf_conntrack_sane_init(void) THIS_MODULE); } - ret = nf_conntrack_helpers_register(sane, ports_c * 2); + ret = nf_conntrack_helpers_register(sane, ports_c * 2, sane_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_conntrack_seqadj.c b/net/netfilter/nf_conntrack_seqadj.c index 7ab2b25b57bcc..220216a4edc51 100644 --- a/net/netfilter/nf_conntrack_seqadj.c +++ b/net/netfilter/nf_conntrack_seqadj.c @@ -17,12 +17,17 @@ int nf_ct_seqadj_init(struct nf_conn *ct, enum ip_conntrack_info ctinfo, if (off == 0) return 0; - set_bit(IPS_SEQ_ADJUST_BIT, &ct->status); - + spin_lock_bh(&ct->lock); seqadj = nfct_seqadj(ct); + if (!seqadj) { + spin_unlock_bh(&ct->lock); + return 0; + } + set_bit(IPS_SEQ_ADJUST_BIT, &ct->status); this_way = &seqadj->seq[dir]; this_way->offset_before = off; this_way->offset_after = off; + spin_unlock_bh(&ct->lock); return 0; } EXPORT_SYMBOL_GPL(nf_ct_seqadj_init); @@ -37,10 +42,8 @@ int nf_ct_seqadj_set(struct nf_conn *ct, enum ip_conntrack_info ctinfo, if (off == 0) return 0; - if (unlikely(!seqadj)) { - WARN_ONCE(1, "Missing nfct_seqadj_ext_add() setup call\n"); + if (unlikely(!seqadj)) return 0; - } set_bit(IPS_SEQ_ADJUST_BIT, &ct->status); @@ -123,6 +126,9 @@ static unsigned int nf_ct_sack_adjust(struct sk_buff *skb, struct nf_conn_seqadj *seqadj = nfct_seqadj(ct); unsigned int dir, optoff, optend; + if (!seqadj) + return 0; + optoff = protoff + sizeof(struct tcphdr); optend = protoff + tcph->doff * 4; @@ -173,6 +179,9 @@ int nf_ct_seq_adjust(struct sk_buff *skb, struct nf_ct_seqadj *this_way, *other_way; int res = 1; + if (!seqadj) + return 0; + this_way = &seqadj->seq[dir]; other_way = &seqadj->seq[!dir]; diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index e69941f1a1016..f3f90a8663389 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -887,6 +887,9 @@ static int refresh_signalling_expectation(struct nf_conn *ct, struct hlist_node *next; int found = 0; + if (!help) + return 0; + spin_lock_bh(&nf_conntrack_expect_lock); hlist_for_each_entry_safe(exp, next, &help->expectations, lnode) { if (exp->class != SIP_EXPECT_SIGNALLING || @@ -894,11 +897,10 @@ static int refresh_signalling_expectation(struct nf_conn *ct, exp->tuple.dst.protonum != proto || exp->tuple.dst.u.udp.port != port) continue; - if (mod_timer_pending(&exp->timeout, jiffies + expires * HZ)) { - exp->flags &= ~NF_CT_EXPECT_INACTIVE; - found = 1; - break; - } + WRITE_ONCE(exp->timeout, nfct_time_stamp + (expires * HZ)); + WRITE_ONCE(exp->flags, exp->flags & ~NF_CT_EXPECT_INACTIVE); + found = 1; + break; } spin_unlock_bh(&nf_conntrack_expect_lock); return found; @@ -910,12 +912,14 @@ static void flush_expectations(struct nf_conn *ct, bool media) struct nf_conntrack_expect *exp; struct hlist_node *next; + if (!help) + return; + spin_lock_bh(&nf_conntrack_expect_lock); hlist_for_each_entry_safe(exp, next, &help->expectations, lnode) { if ((exp->class != SIP_EXPECT_SIGNALLING) ^ media) continue; - if (!nf_ct_remove_expect(exp)) - continue; + nf_ct_unlink_expect(exp); if (!media) break; } @@ -940,6 +944,11 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, u_int16_t base_port; __be16 rtp_port, rtcp_port; const struct nf_nat_sip_hooks *hooks; + struct nf_conn_help *help; + + help = nfct_help(ct); + if (!help) + return NF_DROP; saddr = NULL; if (sip_direct_media) { @@ -947,7 +956,6 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, return NF_ACCEPT; saddr = &ct->tuplehash[!dir].tuple.src.u3; } else if (sip_external_media) { - struct net_device *dev = skb_dst(skb)->dev; struct dst_entry *dst = NULL; struct flowi fl; @@ -969,7 +977,11 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, * through the same interface as the signalling peer. */ if (dst) { - bool external_media = (dst->dev == dev); + const struct dst_entry *this_dst = skb_dst(skb); + bool external_media = false; + + if (this_dst && dst->dev == this_dst->dev) + external_media = true; dst_release(dst); if (external_media) @@ -1002,7 +1014,7 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); if (!exp || exp->master == ct || - exp->helper != nfct_help(ct)->helper || + exp->helper != help->helper || exp->class != class) break; #if IS_ENABLED(CONFIG_NF_NAT) @@ -1227,6 +1239,9 @@ static int process_invite_response(struct sk_buff *skb, unsigned int protoff, struct nf_conn *ct = nf_ct_get(skb, &ctinfo); struct nf_ct_sip_master *ct_sip_info = nfct_help_data(ct); + if (!ct_sip_info) + return NF_DROP; + if ((code >= 100 && code <= 199) || (code >= 200 && code <= 299)) return process_sdp(skb, protoff, dataoff, dptr, datalen, cseq); @@ -1244,6 +1259,9 @@ static int process_update_response(struct sk_buff *skb, unsigned int protoff, struct nf_conn *ct = nf_ct_get(skb, &ctinfo); struct nf_ct_sip_master *ct_sip_info = nfct_help_data(ct); + if (!ct_sip_info) + return NF_DROP; + if ((code >= 100 && code <= 199) || (code >= 200 && code <= 299)) return process_sdp(skb, protoff, dataoff, dptr, datalen, cseq); @@ -1261,6 +1279,9 @@ static int process_prack_response(struct sk_buff *skb, unsigned int protoff, struct nf_conn *ct = nf_ct_get(skb, &ctinfo); struct nf_ct_sip_master *ct_sip_info = nfct_help_data(ct); + if (!ct_sip_info) + return NF_DROP; + if ((code >= 100 && code <= 199) || (code >= 200 && code <= 299)) return process_sdp(skb, protoff, dataoff, dptr, datalen, cseq); @@ -1279,6 +1300,9 @@ static int process_invite_request(struct sk_buff *skb, unsigned int protoff, struct nf_ct_sip_master *ct_sip_info = nfct_help_data(ct); unsigned int ret; + if (!ct_sip_info) + return NF_DROP; + flush_expectations(ct, true); ret = process_sdp(skb, protoff, dataoff, dptr, datalen, cseq); if (ret == NF_ACCEPT) @@ -1316,11 +1340,15 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, union nf_inet_addr *saddr, daddr; const struct nf_nat_sip_hooks *hooks; struct nf_conntrack_helper *helper; + struct nf_conn_help *help; __be16 port; u8 proto; unsigned int expires = 0; int ret; + if (!ct_sip_info) + return NF_DROP; + /* Expected connections can not register again. */ if (ct->status & IPS_EXPECTED) return NF_ACCEPT; @@ -1366,7 +1394,11 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, goto store_cseq; } - helper = rcu_dereference(nfct_help(ct)->helper); + help = nfct_help(ct); + if (!help) + return NF_DROP; + + helper = rcu_dereference(help->helper); if (!helper) return NF_DROP; @@ -1382,7 +1414,6 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), saddr, &daddr, proto, NULL, &port); - exp->timeout.expires = sip_timeout * HZ; rcu_assign_pointer(exp->assign_helper, helper); exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE; @@ -1421,6 +1452,9 @@ static int process_register_response(struct sk_buff *skb, unsigned int protoff, unsigned int expires = 0; int in_contact = 0, ret; + if (!ct_sip_info) + return NF_DROP; + /* According to RFC 3261, "UAs MUST NOT send a new registration until * they have received a final response from the registrar for the * previous one or the previous REGISTER request has timed out". @@ -1550,6 +1584,9 @@ static int process_sip_request(struct sk_buff *skb, unsigned int protoff, union nf_inet_addr addr; __be16 port; + if (!ct_sip_info) + return NF_DROP; + /* Many Cisco IP phones use a high source port for SIP requests, but * listen for the response on port 5060. If we are the local * router for one of these phones, save the port number from the @@ -1731,6 +1768,7 @@ static int sip_help_udp(struct sk_buff *skb, unsigned int protoff, } static struct nf_conntrack_helper sip[MAX_PORTS * 4] __read_mostly; +static struct nf_conntrack_helper *sip_ptr[MAX_PORTS * 4] __read_mostly; static const struct nf_conntrack_expect_policy sip_exp_policy[SIP_EXPECT_MAX + 1] = { [SIP_EXPECT_SIGNALLING] = { @@ -1757,7 +1795,7 @@ static const struct nf_conntrack_expect_policy sip_exp_policy[SIP_EXPECT_MAX + 1 static void __exit nf_conntrack_sip_fini(void) { - nf_conntrack_helpers_unregister(sip, ports_c * 4); + nf_conntrack_helpers_unregister(sip_ptr, ports_c * 4); } static int __init nf_conntrack_sip_init(void) @@ -1788,7 +1826,7 @@ static int __init nf_conntrack_sip_init(void) NULL, THIS_MODULE); } - ret = nf_conntrack_helpers_register(sip, ports_c * 4); + ret = nf_conntrack_helpers_register(sip, ports_c * 4, sip_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_conntrack_snmp.c b/net/netfilter/nf_conntrack_snmp.c index 7b7eed43c54f1..b6fce5703fce7 100644 --- a/net/netfilter/nf_conntrack_snmp.c +++ b/net/netfilter/nf_conntrack_snmp.c @@ -47,25 +47,24 @@ static struct nf_conntrack_expect_policy exp_policy = { .max_expected = 1, }; -static struct nf_conntrack_helper helper __read_mostly = { - .name = "snmp", - .tuple.src.l3num = NFPROTO_IPV4, - .tuple.src.u.udp.port = cpu_to_be16(SNMP_PORT), - .tuple.dst.protonum = IPPROTO_UDP, - .me = THIS_MODULE, - .help = snmp_conntrack_help, - .expect_policy = &exp_policy, -}; +static struct nf_conntrack_helper helper __read_mostly; +static struct nf_conntrack_helper *helper_ptr __read_mostly; static int __init nf_conntrack_snmp_init(void) { exp_policy.timeout = timeout; - return nf_conntrack_helper_register(&helper); + + nf_ct_helper_init(&helper, AF_INET, IPPROTO_UDP, + "snmp", SNMP_PORT, SNMP_PORT, SNMP_PORT, + &exp_policy, 0, snmp_conntrack_help, NULL, + THIS_MODULE); + + return nf_conntrack_helper_register(&helper, &helper_ptr); } static void __exit nf_conntrack_snmp_fini(void) { - nf_conntrack_helper_unregister(&helper); + nf_conntrack_helper_unregister(helper_ptr); } module_init(nf_conntrack_snmp_init); diff --git a/net/netfilter/nf_conntrack_tftp.c b/net/netfilter/nf_conntrack_tftp.c index a2e6833a0bf76..4393c435aa354 100644 --- a/net/netfilter/nf_conntrack_tftp.c +++ b/net/netfilter/nf_conntrack_tftp.c @@ -96,6 +96,7 @@ static int tftp_help(struct sk_buff *skb, } static struct nf_conntrack_helper tftp[MAX_PORTS * 2] __read_mostly; +static struct nf_conntrack_helper *tftp_ptr[MAX_PORTS * 2] __read_mostly; static const struct nf_conntrack_expect_policy tftp_exp_policy = { .max_expected = 1, @@ -104,7 +105,7 @@ static const struct nf_conntrack_expect_policy tftp_exp_policy = { static void __exit nf_conntrack_tftp_fini(void) { - nf_conntrack_helpers_unregister(tftp, ports_c * 2); + nf_conntrack_helpers_unregister(tftp_ptr, ports_c * 2); } static int __init nf_conntrack_tftp_init(void) @@ -127,7 +128,7 @@ static int __init nf_conntrack_tftp_init(void) THIS_MODULE); } - ret = nf_conntrack_helpers_register(tftp, ports_c * 2); + ret = nf_conntrack_helpers_register(tftp, ports_c * 2, tftp_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_conntrack_timeout.c b/net/netfilter/nf_conntrack_timeout.c index 0cc584d3dbb1d..c81becde2afa9 100644 --- a/net/netfilter/nf_conntrack_timeout.c +++ b/net/netfilter/nf_conntrack_timeout.c @@ -25,17 +25,32 @@ const struct nf_ct_timeout_hooks __rcu *nf_ct_timeout_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_ct_timeout_hook); +/* nf_ct_iterate_cleanup() holds the conntrack lock. */ static int untimeout(struct nf_conn *ct, void *timeout) { struct nf_conn_timeout *timeout_ext = nf_ct_timeout_find(ct); if (timeout_ext) { - const struct nf_ct_timeout *t; + struct nf_ct_timeout *t; - t = rcu_access_pointer(timeout_ext->timeout); + rcu_read_lock(); + t = rcu_dereference(timeout_ext->timeout); + if (!t) { + rcu_read_unlock(); + return 0; + } - if (!timeout || t == timeout) + if (!timeout || t == timeout) { RCU_INIT_POINTER(timeout_ext->timeout, NULL); + + /* No race with nf_conntrack_free() which is called + * only after the conntrack has been removed from + * the hashes. + */ + if (refcount_dec_and_test(&t->refcnt)) + kfree_rcu(t, rcu); + } + rcu_read_unlock(); } /* We are not intended to delete this conntrack. */ @@ -70,6 +85,8 @@ int nf_ct_set_timeout(struct net *net, struct nf_conn *ct, const char *errmsg = NULL; int ret = 0; + WARN_ON_ONCE(!nf_ct_is_template(ct)); + rcu_read_lock(); h = rcu_dereference(nf_ct_timeout_hook); if (!h) { @@ -127,6 +144,8 @@ void nf_ct_destroy_timeout(struct nf_conn *ct) struct nf_conn_timeout *timeout_ext; const struct nf_ct_timeout_hooks *h; + WARN_ON_ONCE(!nf_ct_is_template(ct)); + rcu_read_lock(); h = rcu_dereference(nf_ct_timeout_hook); @@ -139,6 +158,8 @@ void nf_ct_destroy_timeout(struct nf_conn *ct) if (t) h->timeout_put(t); RCU_INIT_POINTER(timeout_ext->timeout, NULL); + if (t && refcount_dec_and_test(&t->refcnt)) + kfree_rcu(t, rcu); } } rcu_read_unlock(); diff --git a/net/netfilter/nf_dup_netdev.c b/net/netfilter/nf_dup_netdev.c index 3d88ef927f31c..c6bd5c29bed66 100644 --- a/net/netfilter/nf_dup_netdev.c +++ b/net/netfilter/nf_dup_netdev.c @@ -16,11 +16,6 @@ static void nf_do_netdev_egress(struct sk_buff *skb, struct net_device *dev, enum nf_dev_hooks hook) { - u8 *nf_dup_skb_recursion = nf_get_nf_dup_skb_recursion(); - - if (*nf_dup_skb_recursion > NF_RECURSION_LIMIT) - goto err; - if (hook == NF_NETDEV_INGRESS && skb_mac_header_was_set(skb)) { if (skb_cow_head(skb, skb->mac_len)) goto err; @@ -30,9 +25,15 @@ static void nf_do_netdev_egress(struct sk_buff *skb, struct net_device *dev, skb->dev = dev; skb_clear_tstamp(skb); - (*nf_dup_skb_recursion)++; + local_bh_disable(); + if (nf_dev_xmit_recursion()) { + local_bh_enable(); + goto err; + } + nf_dev_xmit_recursion_inc(); dev_queue_xmit(skb); - (*nf_dup_skb_recursion)--; + nf_dev_xmit_recursion_dec(); + local_bh_enable(); return; err: kfree_skb(skb); diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index 785d8c244a771..b66e654393411 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -127,12 +127,18 @@ static int flow_offload_fill_route(struct flow_offload *flow, switch (route->tuple[dir].xmit_type) { case FLOW_OFFLOAD_XMIT_DIRECT: + if (route->tuple[!dir].in.num_tuns) { + flow_tuple->dst_cache = dst; + flow_tuple->dst_cookie = + flow_offload_dst_cookie(flow_tuple); + } else { + dst_release(dst); + } memcpy(flow_tuple->out.h_dest, route->tuple[dir].out.h_dest, ETH_ALEN); memcpy(flow_tuple->out.h_source, route->tuple[dir].out.h_source, ETH_ALEN); flow_tuple->out.ifidx = route->tuple[dir].out.ifindex; - dst_release(dst); break; case FLOW_OFFLOAD_XMIT_XFRM: case FLOW_OFFLOAD_XMIT_NEIGH: @@ -152,9 +158,7 @@ static int flow_offload_fill_route(struct flow_offload *flow, static void nft_flow_dst_release(struct flow_offload *flow, enum flow_offload_tuple_dir dir) { - if (flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_NEIGH || - flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM) - dst_release(flow->tuplehash[dir].tuple.dst_cache); + dst_release(flow->tuplehash[dir].tuple.dst_cache); } void flow_offload_route_init(struct flow_offload *flow, @@ -345,10 +349,8 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow) nf_ct_refresh(flow->ct, NF_CT_DAY); - if (nf_flowtable_hw_offload(flow_table)) { - __set_bit(NF_FLOW_HW, &flow->flags); + if (nf_flowtable_hw_offload(flow_table)) nf_flow_offload_add(flow_table, flow); - } return 0; } @@ -369,7 +371,8 @@ void flow_offload_refresh(struct nf_flowtable *flow_table, test_bit(NF_FLOW_CLOSING, &flow->flags)) return; - nf_flow_offload_add(flow_table, flow); + if (test_bit(NF_FLOW_HW, &flow->flags)) + nf_flow_offload_refresh(flow_table, flow); } EXPORT_SYMBOL_GPL(flow_offload_refresh); @@ -505,8 +508,13 @@ static u32 nf_flow_table_tcp_timeout(const struct nf_conn *ct) */ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct) { - static const u32 min_timeout = 5 * 60 * HZ; - u32 expires = nf_ct_expires(ct); + static const s32 min_timeout = 5 * 60 * HZ; + u32 ct_timeout = READ_ONCE(ct->timeout); + s32 expires; + + expires = ct_timeout - nfct_time_stamp; + if (expires <= 0) /* already expired */ + return; /* normal case: large enough timeout, nothing to do. */ if (likely(expires >= min_timeout)) @@ -524,7 +532,7 @@ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct) if (nf_ct_is_confirmed(ct) && test_bit(IPS_OFFLOAD_BIT, &ct->status)) { u8 l4proto = nf_ct_protonum(ct); - u32 new_timeout = true; + u32 new_timeout = 1; switch (l4proto) { case IPPROTO_UDP: @@ -549,7 +557,7 @@ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct) */ if (new_timeout) { new_timeout += nfct_time_stamp; - cmpxchg(&ct->timeout, expires, new_timeout); + cmpxchg(&ct->timeout, ct_timeout, new_timeout); } } diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c index 9c05a50d60138..0b78decce8a9b 100644 --- a/net/netfilter/nf_flow_table_ip.c +++ b/net/netfilter/nf_flow_table_ip.c @@ -299,8 +299,7 @@ static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple) { - if (tuple->xmit_type != FLOW_OFFLOAD_XMIT_NEIGH && - tuple->xmit_type != FLOW_OFFLOAD_XMIT_XFRM) + if (!tuple->dst_cache) return true; return dst_check(tuple->dst_cache, tuple->dst_cookie); @@ -326,8 +325,10 @@ static bool nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx *ctx, return false; iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset); - size = iph->ihl << 2; + if (iph->ihl < 5) + return false; + size = iph->ihl << 2; if (ip_is_fragment(iph) || unlikely(ip_has_options(size))) return false; @@ -335,9 +336,9 @@ static bool nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx *ctx, return false; if (iph->protocol == IPPROTO_IPIP) { - ctx->tun.proto = IPPROTO_IPIP; + ctx->tun.proto = iph->protocol; ctx->tun.hdr_size = size; - ctx->offset += size; + ctx->offset += ctx->tun.hdr_size; } return true; @@ -347,29 +348,23 @@ static bool nf_flow_ip6_tunnel_proto(struct nf_flowtable_ctx *ctx, struct sk_buff *skb) { #if IS_ENABLED(CONFIG_IPV6) - struct ipv6hdr *ip6h, _ip6h; - __be16 frag_off; - u8 nexthdr; - int hdrlen; + struct ipv6hdr *ip6h; - ip6h = skb_header_pointer(skb, ctx->offset, sizeof(*ip6h), &_ip6h); - if (!ip6h) + if (!pskb_may_pull(skb, sizeof(*ip6h) + ctx->offset)) return false; + ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset); if (ip6h->hop_limit <= 1) return false; - nexthdr = ip6h->nexthdr; - hdrlen = ipv6_skip_exthdr(skb, sizeof(*ip6h) + ctx->offset, &nexthdr, - &frag_off); - if (hdrlen < 0) + if (ipv6_ext_hdr(ip6h->nexthdr)) return false; - if (nexthdr == IPPROTO_IPV6) { - ctx->tun.hdr_size = hdrlen; - ctx->tun.proto = IPPROTO_IPV6; + if (ip6h->nexthdr == IPPROTO_IPV6) { + ctx->tun.proto = ip6h->nexthdr; + ctx->tun.hdr_size = sizeof(*ip6h); + ctx->offset += ctx->tun.hdr_size; } - ctx->offset += ctx->tun.hdr_size; return true; #else @@ -594,10 +589,10 @@ static int nf_flow_pppoe_push(struct sk_buff *skb, u16 id, static int nf_flow_tunnel_ipip_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - __be32 *ip_daddr) + struct dst_entry *dst, __be32 *ip_daddr) { struct iphdr *iph = (struct iphdr *)skb_network_header(skb); - struct rtable *rt = dst_rtable(tuple->dst_cache); + struct rtable *rt = dst_rtable(dst); u8 tos = iph->tos, ttl = iph->ttl; __be16 frag_off = iph->frag_off; u32 headroom = sizeof(*iph); @@ -640,33 +635,28 @@ static int nf_flow_tunnel_ipip_push(struct net *net, struct sk_buff *skb, static int nf_flow_tunnel_v4_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - __be32 *ip_daddr) + struct dst_entry *dst, __be32 *ip_daddr) { if (tuple->tun_num) - return nf_flow_tunnel_ipip_push(net, skb, tuple, ip_daddr); + return nf_flow_tunnel_ipip_push(net, skb, tuple, dst, ip_daddr); return 0; } -struct ipv6_tel_txoption { - struct ipv6_txoptions ops; - __u8 dst_opt[8]; -}; - static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - struct in6_addr **ip6_daddr, - int encap_limit) + struct dst_entry *dst, + struct in6_addr **ip6_daddr) { struct ipv6hdr *ip6h = (struct ipv6hdr *)skb_network_header(skb); - u8 hop_limit = ip6h->hop_limit, proto = IPPROTO_IPV6; - struct rtable *rt = dst_rtable(tuple->dst_cache); __u8 dsfield = ipv6_get_dsfield(ip6h); + struct rtable *rt = dst_rtable(dst); struct flowi6 fl6 = { .daddr = tuple->tun.src_v6, .saddr = tuple->tun.dst_v6, - .flowi6_proto = proto, + .flowi6_proto = IPPROTO_IPV6, }; + u8 hop_limit = ip6h->hop_limit; int err, mtu; u32 headroom; @@ -674,41 +664,18 @@ static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, if (err) return err; - skb_set_inner_ipproto(skb, proto); + skb_set_inner_ipproto(skb, IPPROTO_IPV6); headroom = sizeof(*ip6h) + LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len; - if (encap_limit) - headroom += 8; err = skb_cow_head(skb, headroom); if (err) return err; skb_scrub_packet(skb, true); mtu = dst_mtu(&rt->dst) - sizeof(*ip6h); - if (encap_limit) - mtu -= 8; mtu = max(mtu, IPV6_MIN_MTU); skb_dst_update_pmtu_no_confirm(skb, mtu); - if (encap_limit > 0) { - struct ipv6_tel_txoption opt = { - .dst_opt[2] = IPV6_TLV_TNL_ENCAP_LIMIT, - .dst_opt[3] = 1, - .dst_opt[4] = encap_limit, - .dst_opt[5] = IPV6_TLV_PADN, - .dst_opt[6] = 1, - }; - struct ipv6_opt_hdr *hopt; - - opt.ops.dst1opt = (struct ipv6_opt_hdr *)opt.dst_opt; - opt.ops.opt_nflen = 8; - - hopt = skb_push(skb, ipv6_optlen(opt.ops.dst1opt)); - memcpy(hopt, opt.ops.dst1opt, ipv6_optlen(opt.ops.dst1opt)); - hopt->nexthdr = IPPROTO_IPV6; - proto = NEXTHDR_DEST; - } - skb_push(skb, sizeof(*ip6h)); skb_reset_network_header(skb); @@ -716,7 +683,7 @@ static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, ip6_flow_hdr(ip6h, dsfield, ip6_make_flowlabel(net, skb, fl6.flowlabel, true, &fl6)); ip6h->hop_limit = hop_limit; - ip6h->nexthdr = proto; + ip6h->nexthdr = IPPROTO_IPV6; ip6h->daddr = tuple->tun.src_v6; ip6h->saddr = tuple->tun.dst_v6; ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(*ip6h)); @@ -729,12 +696,11 @@ static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, static int nf_flow_tunnel_v6_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - struct in6_addr **ip6_daddr, - int encap_limit) + struct dst_entry *dst, + struct in6_addr **ip6_daddr) { if (tuple->tun_num) - return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, ip6_daddr, - encap_limit); + return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, dst, ip6_daddr); return 0; } @@ -877,7 +843,8 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, other_tuple = &flow->tuplehash[!dir].tuple; ip_daddr = other_tuple->src_v4.s_addr; - if (nf_flow_tunnel_v4_push(state->net, skb, other_tuple, &ip_daddr) < 0) + if (nf_flow_tunnel_v4_push(state->net, skb, other_tuple, + tuplehash->tuple.dst_cache, &ip_daddr) < 0) return NF_DROP; switch (tuplehash->tuple.xmit_type) { @@ -1089,7 +1056,7 @@ static int nf_flow_tuple_ipv6(struct nf_flowtable_ctx *ctx, struct sk_buff *skb, static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx, struct nf_flowtable *flow_table, struct flow_offload_tuple_rhash *tuplehash, - struct sk_buff *skb, int encap_limit) + struct sk_buff *skb) { enum flow_offload_tuple_dir dir; struct flow_offload *flow; @@ -1100,11 +1067,8 @@ static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx, flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset; - if (flow->tuplehash[!dir].tuple.tun_num) { + if (flow->tuplehash[!dir].tuple.tun_num) mtu -= sizeof(*ip6h); - if (encap_limit > 0) - mtu -= 8; /* encap limit option */ - } if (unlikely(nf_flow_exceeds_mtu(skb, mtu))) return 0; @@ -1158,7 +1122,6 @@ unsigned int nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state) { - int encap_limit = IPV6_DEFAULT_TNL_ENCAP_LIMIT; struct flow_offload_tuple_rhash *tuplehash; struct nf_flowtable *flow_table = priv; struct flow_offload_tuple *other_tuple; @@ -1177,8 +1140,7 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, if (tuplehash == NULL) return NF_ACCEPT; - ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, skb, - encap_limit); + ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, skb); if (ret < 0) return NF_DROP; else if (ret == 0) @@ -1198,7 +1160,8 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, ip6_daddr = &other_tuple->src_v6; if (nf_flow_tunnel_v6_push(state->net, skb, other_tuple, - &ip6_daddr, encap_limit) < 0) + tuplehash->tuple.dst_cache, + &ip6_daddr) < 0) return NF_DROP; switch (tuplehash->tuple.xmit_type) { diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 002ec15d988bd..801a3dd9ceea3 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -1101,9 +1101,17 @@ nf_flow_offload_work_alloc(struct nf_flowtable *flowtable, return offload; } +static bool nf_flow_offload_unsupported(struct flow_offload *flow) +{ + if (flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.tun_num || + flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.tun_num) + return true; -void nf_flow_offload_add(struct nf_flowtable *flowtable, - struct flow_offload *flow) + return false; +} + +void nf_flow_offload_refresh(struct nf_flowtable *flowtable, + struct flow_offload *flow) { struct flow_offload_work *offload; @@ -1114,6 +1122,16 @@ void nf_flow_offload_add(struct nf_flowtable *flowtable, flow_offload_queue_work(offload); } +void nf_flow_offload_add(struct nf_flowtable *flowtable, + struct flow_offload *flow) +{ + if (nf_flow_offload_unsupported(flow)) + return; + + set_bit(NF_FLOW_HW, &flow->flags); + nf_flow_offload_refresh(flowtable, flow); +} + void nf_flow_offload_del(struct nf_flowtable *flowtable, struct flow_offload *flow) { diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c index 9e88ea6a2eef7..14e8a3e4a6f87 100644 --- a/net/netfilter/nf_flow_table_path.c +++ b/net/netfilter/nf_flow_table_path.c @@ -53,8 +53,10 @@ static int nft_dev_fill_forward_path(const struct nf_flow_route *route, struct neighbour *n; u8 nud_state; - if (!nft_is_valid_ether_device(dev)) + if (!nft_is_valid_ether_device(dev)) { + eth_zero_addr(ha); goto out; + } n = dst_neigh_lookup(dst_cache, daddr); if (!n) @@ -163,10 +165,11 @@ static void nft_dev_path_info(const struct net_device_path_stack *stack, info->num_encaps++; break; case DEV_PATH_BR_VLAN_UNTAG: - if (WARN_ON_ONCE(info->num_encaps-- == 0)) { + if (info->num_encaps == 0) { info->indev = NULL; break; } + info->num_encaps--; break; case DEV_PATH_BR_VLAN_KEEP: break; diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 2bbf5163c0e27..63ff6b4d5d214 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -1181,6 +1181,16 @@ int nf_nat_register_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops, struct nf_hook_ops *nat_ops; int i, ret; +#ifndef MODULE + /* If nf_nat_core is built-in and nf_nat_init() fails, dependent + * modules like nft_chain_nat.ko may still call this function. + * However, nat_net would be invalid, likely pointing to some other + * per-net structure. + */ + if (WARN_ON_ONCE(!nf_nat_hook)) + return -EOPNOTSUPP; +#endif + if (WARN_ON_ONCE(pf >= ARRAY_SIZE(nat_net->nat_proto_net))) return -EINVAL; diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index 00838c0cc5bb2..aea02f6aff092 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -106,6 +106,9 @@ static int map_addr(struct sk_buff *skb, unsigned int protoff, union nf_inet_addr newaddr; __be16 newport; + if (!ct_sip_info) + return 0; + if (nf_inet_addr_cmp(&ct->tuplehash[dir].tuple.src.u3, addr) && ct->tuplehash[dir].tuple.src.u.udp.port == port) { newaddr = ct->tuplehash[!dir].tuple.dst.u3; @@ -158,6 +161,9 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, __be16 port; int request, in_header; + if (!ct_sip_info) + return NF_DROP; + /* Basic rules: requests and responses. */ if (strncasecmp(*dptr, "SIP/2.0", strlen("SIP/2.0")) != 0) { if (ct_sip_parse_request(ct, *dptr, *datalen, @@ -283,13 +289,24 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, /* Mangle destination port for Cisco phones, then fix up checksums */ if (dir == IP_CT_DIR_REPLY && ct_sip_info->forced_dport) { + int doff = *dptr - (const char *)skb->data; struct udphdr *uh; + if (doff <= 0) { + DEBUG_NET_WARN_ON_ONCE(1); + return NF_DROP; + } + + /* ct_sip_info->forced_dport only expected with UDP */ + if (nf_ct_protonum(ct) != IPPROTO_UDP) + return NF_DROP; + if (skb_ensure_writable(skb, skb->len)) { nf_ct_helper_log(skb, ct, "cannot mangle packet"); return NF_DROP; } + *dptr = skb->data + doff; uh = (void *)skb->data + protoff; uh->dest = ct_sip_info->forced_dport; @@ -326,6 +343,9 @@ static void nf_nat_sip_expected(struct nf_conn *ct, int range_set_for_snat = 0; struct nf_nat_range2 range; + if (!help) + return; + /* This must be a fresh one. */ BUG_ON(ct->status & IPS_NAT_DONE_MASK); @@ -390,6 +410,9 @@ static unsigned int nf_nat_sip_expect(struct sk_buff *skb, unsigned int protoff, char buffer[INET6_ADDRSTRLEN + sizeof("[]:nnnnn")]; unsigned int buflen; + if (!ct_sip_info) + return NF_DROP; + /* Connection will come from reply */ if (nf_inet_addr_cmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3)) diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c index 57b450024a99e..73363ceedebe3 100644 --- a/net/netfilter/nf_queue.c +++ b/net/netfilter/nf_queue.c @@ -68,6 +68,7 @@ static void nf_queue_entry_release_refs(struct nf_queue_entry *entry) nf_queue_sock_put(state->sk); #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) + dev_put(entry->bridge_dev); dev_put(entry->physin); dev_put(entry->physout); #endif @@ -84,6 +85,8 @@ static void __nf_queue_entry_init_physdevs(struct nf_queue_entry *entry) { #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) const struct sk_buff *skb = entry->skb; + struct dst_entry *dst = skb_dst(skb); + struct net_device *dev = NULL; if (nf_bridge_info_exists(skb)) { entry->physin = nf_bridge_get_physindev(skb, entry->state.net); @@ -92,6 +95,16 @@ static void __nf_queue_entry_init_physdevs(struct nf_queue_entry *entry) entry->physin = NULL; entry->physout = NULL; } + + if (entry->state.pf == NFPROTO_BRIDGE && + dst && (dst->flags & DST_FAKE_RTABLE)) + dev = dst_dev_rcu(dst); + + /* Must hold a reference on the bridge device: dst_hold() protects + * the dst itself, but the fake rtable is embedded in bridge-private + * storage that netdevice teardown can free independently. + */ + entry->bridge_dev = dev; #endif } @@ -108,6 +121,7 @@ bool nf_queue_entry_get_refs(struct nf_queue_entry *entry) dev_hold(state->out); #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) + dev_hold(entry->bridge_dev); dev_hold(entry->physin); dev_hold(entry->physout); #endif diff --git a/net/netfilter/nf_synproxy_core.c b/net/netfilter/nf_synproxy_core.c index ed00114f65f39..acd360515972f 100644 --- a/net/netfilter/nf_synproxy_core.c +++ b/net/netfilter/nf_synproxy_core.c @@ -184,23 +184,23 @@ synproxy_check_timestamp_cookie(struct synproxy_options *opts) opts->options |= opts->tsecr & (1 << 5) ? NF_SYNPROXY_OPT_ECN : 0; } -static unsigned int +static bool synproxy_tstamp_adjust(struct sk_buff *skb, unsigned int protoff, struct tcphdr *th, struct nf_conn *ct, enum ip_conntrack_info ctinfo, const struct nf_conn_synproxy *synproxy) { unsigned int optoff, optend; - __be32 *ptr, old; + u32 new, old; if (synproxy->tsoff == 0) - return 1; + return true; optoff = protoff + sizeof(struct tcphdr); optend = protoff + th->doff * 4; if (skb_ensure_writable(skb, optend)) - return 0; + return false; th = (struct tcphdr *)(skb->data + protoff); @@ -209,7 +209,7 @@ synproxy_tstamp_adjust(struct sk_buff *skb, unsigned int protoff, switch (op[0]) { case TCPOPT_EOL: - return 1; + return true; case TCPOPT_NOP: optoff++; continue; @@ -217,28 +217,26 @@ synproxy_tstamp_adjust(struct sk_buff *skb, unsigned int protoff, if (optoff + 1 == optend || optoff + op[1] > optend || op[1] < 2) - return 0; + return true; if (op[0] == TCPOPT_TIMESTAMP && op[1] == TCPOLEN_TIMESTAMP) { if (CTINFO2DIR(ctinfo) == IP_CT_DIR_REPLY) { - ptr = (__be32 *)&op[2]; - old = *ptr; - *ptr = htonl(ntohl(*ptr) - - synproxy->tsoff); + old = get_unaligned_be32(&op[2]); + new = old - synproxy->tsoff; + put_unaligned_be32(new, &op[2]); } else { - ptr = (__be32 *)&op[6]; - old = *ptr; - *ptr = htonl(ntohl(*ptr) + - synproxy->tsoff); + old = get_unaligned_be32(&op[6]); + new = old + synproxy->tsoff; + put_unaligned_be32(new, &op[6]); } inet_proto_csum_replace4(&th->check, skb, - old, *ptr, false); - return 1; + cpu_to_be32(old), + cpu_to_be32(new), false); } optoff += op[1]; } } - return 1; + return true; } #ifdef CONFIG_PROC_FS @@ -749,7 +747,9 @@ ipv4_synproxy_hook(void *priv, struct sk_buff *skb, break; } - synproxy_tstamp_adjust(skb, thoff, th, ct, ctinfo, synproxy); + if (!synproxy_tstamp_adjust(skb, thoff, th, ct, ctinfo, synproxy)) + return NF_DROP_REASON(skb, SKB_DROP_REASON_NETFILTER_DROP, ENOMEM); + return NF_ACCEPT; } EXPORT_SYMBOL_GPL(ipv4_synproxy_hook); @@ -1177,7 +1177,9 @@ ipv6_synproxy_hook(void *priv, struct sk_buff *skb, break; } - synproxy_tstamp_adjust(skb, thoff, th, ct, ctinfo, synproxy); + if (!synproxy_tstamp_adjust(skb, thoff, th, ct, ctinfo, synproxy)) + return NF_DROP_REASON(skb, SKB_DROP_REASON_NETFILTER_DROP, ENOMEM); + return NF_ACCEPT; } EXPORT_SYMBOL_GPL(ipv6_synproxy_hook); diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 87387adbca655..e1db3e678656d 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -6559,6 +6559,9 @@ static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set, if (err < 0) return err; + if (!elem.priv) + return 0; + err = -ENOMEM; skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC); if (skb == NULL) diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c index 0d16ad82d70c0..8710fec30df9c 100644 --- a/net/netfilter/nfnetlink_cthelper.c +++ b/net/netfilter/nfnetlink_cthelper.c @@ -32,7 +32,7 @@ MODULE_DESCRIPTION("nfnl_cthelper: User-space connection tracking helpers"); struct nfnl_cthelper { struct list_head list; - struct nf_conntrack_helper helper; + struct nf_conntrack_helper *helper; }; static LIST_HEAD(nfnl_cthelper_list); @@ -41,8 +41,9 @@ static int nfnl_userspace_cthelper(struct sk_buff *skb, unsigned int protoff, struct nf_conn *ct, enum ip_conntrack_info ctinfo) { - const struct nf_conn_help *help; struct nf_conntrack_helper *helper; + const struct nf_conn_help *help; + unsigned int helper_flags; help = nfct_help(ct); if (help == NULL) @@ -53,8 +54,10 @@ nfnl_userspace_cthelper(struct sk_buff *skb, unsigned int protoff, if (helper == NULL) return NF_DROP; + helper_flags = READ_ONCE(helper->flags); + /* This is a user-space helper not yet configured, skip. */ - if ((helper->flags & + if ((helper_flags & (NF_CT_HELPER_F_USERSPACE | NF_CT_HELPER_F_CONFIGURED)) == NF_CT_HELPER_F_USERSPACE) return NF_ACCEPT; @@ -98,6 +101,9 @@ nfnl_cthelper_from_nlattr(struct nlattr *attr, struct nf_conn *ct) struct nf_conn_help *help = nfct_help(ct); const struct nf_conntrack_helper *helper; + if (!help) + return -EINVAL; + if (attr == NULL) return -EINVAL; @@ -115,6 +121,9 @@ nfnl_cthelper_to_nlattr(struct sk_buff *skb, const struct nf_conn *ct) const struct nf_conn_help *help = nfct_help(ct); const struct nf_conntrack_helper *helper; + if (!help) + return 0; + helper = rcu_dereference(help->helper); if (helper && helper->data_len && nla_put(skb, CTA_HELP_INFO, helper->data_len, &help->data)) @@ -154,6 +163,8 @@ nfnl_cthelper_expect_policy(struct nf_conntrack_expect_policy *expect_policy, tb[NFCTH_POLICY_NAME], NF_CT_HELPER_NAME_LEN); expect_policy->max_expected = ntohl(nla_get_be32(tb[NFCTH_POLICY_EXPECT_MAX])); + if (!expect_policy->max_expected) + expect_policy->max_expected = NF_CT_EXPECT_MAX_CNT; if (expect_policy->max_expected > NF_CT_EXPECT_MAX_CNT) return -EINVAL; @@ -173,7 +184,6 @@ nfnl_cthelper_parse_expect_policy(struct nf_conntrack_helper *helper, const struct nlattr *attr) { int i, ret; - struct nf_conntrack_expect_policy *expect_policy; struct nlattr *tb[NFCTH_POLICY_SET_MAX+1]; unsigned int class_max; @@ -192,26 +202,19 @@ nfnl_cthelper_parse_expect_policy(struct nf_conntrack_helper *helper, if (class_max > NF_CT_MAX_EXPECT_CLASSES) return -EOVERFLOW; - expect_policy = kzalloc_objs(struct nf_conntrack_expect_policy, - class_max); - if (expect_policy == NULL) - return -ENOMEM; - for (i = 0; i < class_max; i++) { if (!tb[NFCTH_POLICY_SET+i]) goto err; - ret = nfnl_cthelper_expect_policy(&expect_policy[i], + ret = nfnl_cthelper_expect_policy(&helper->expect_policy[i], tb[NFCTH_POLICY_SET+i]); if (ret < 0) goto err; } helper->expect_class_max = class_max - 1; - helper->expect_policy = expect_policy; return 0; err: - kfree(expect_policy); return -EINVAL; } @@ -227,21 +230,28 @@ nfnl_cthelper_create(const struct nlattr * const tb[], if (!tb[NFCTH_TUPLE] || !tb[NFCTH_POLICY] || !tb[NFCTH_PRIV_DATA_LEN]) return -EINVAL; - nfcth = kzalloc_obj(*nfcth); + nfcth = kzalloc_obj(*nfcth, GFP_KERNEL_ACCOUNT); if (nfcth == NULL) return -ENOMEM; - helper = &nfcth->helper; + + helper = kzalloc_obj(*helper, GFP_KERNEL_ACCOUNT); + if (!helper) { + ret = -ENOMEM; + goto err_cth; + } + + nfcth->helper = helper; ret = nfnl_cthelper_parse_expect_policy(helper, tb[NFCTH_POLICY]); if (ret < 0) - goto err1; + goto err_helper; nla_strscpy(helper->name, tb[NFCTH_NAME], NF_CT_HELPER_NAME_LEN); size = ntohl(nla_get_be32(tb[NFCTH_PRIV_DATA_LEN])); if (size > sizeof_field(struct nf_conn_help, data)) { ret = -ENOMEM; - goto err2; + goto err_helper; } helper->data_len = size; @@ -270,15 +280,15 @@ nfnl_cthelper_create(const struct nlattr * const tb[], } } - ret = nf_conntrack_helper_register(helper); + ret = __nf_conntrack_helper_register(helper); if (ret < 0) - goto err2; + goto err_helper; list_add_tail(&nfcth->list, &nfnl_cthelper_list); return 0; -err2: - kfree(helper->expect_policy); -err1: +err_helper: + kfree(helper); +err_cth: kfree(nfcth); return ret; } @@ -306,6 +316,8 @@ nfnl_cthelper_update_policy_one(const struct nf_conntrack_expect_policy *policy, new_policy->max_expected = ntohl(nla_get_be32(tb[NFCTH_POLICY_EXPECT_MAX])); + if (!new_policy->max_expected) + new_policy->max_expected = NF_CT_EXPECT_MAX_CNT; if (new_policy->max_expected > NF_CT_EXPECT_MAX_CNT) return -EINVAL; @@ -346,8 +358,8 @@ static int nfnl_cthelper_update_policy_all(struct nlattr *tb[], for (i = 0; i < helper->expect_class_max + 1; i++) { policy = (struct nf_conntrack_expect_policy *) &helper->expect_policy[i]; - policy->max_expected = new_policy->max_expected; - policy->timeout = new_policy->timeout; + policy->max_expected = new_policy[i].max_expected; + policy->timeout = new_policy[i].timeout; } err: @@ -404,10 +416,10 @@ nfnl_cthelper_update(const struct nlattr * const tb[], switch(status) { case NFCT_HELPER_STATUS_ENABLED: - helper->flags |= NF_CT_HELPER_F_CONFIGURED; + WRITE_ONCE(helper->flags, helper->flags | NF_CT_HELPER_F_CONFIGURED); break; case NFCT_HELPER_STATUS_DISABLED: - helper->flags &= ~NF_CT_HELPER_F_CONFIGURED; + WRITE_ONCE(helper->flags, helper->flags & ~NF_CT_HELPER_F_CONFIGURED); break; } } @@ -436,7 +448,7 @@ static int nfnl_cthelper_new(struct sk_buff *skb, const struct nfnl_info *info, return ret; list_for_each_entry(nlcth, &nfnl_cthelper_list, list) { - cur = &nlcth->helper; + cur = nlcth->helper; if (strncmp(cur->name, helper_name, NF_CT_HELPER_NAME_LEN)) continue; @@ -529,8 +541,8 @@ static int nfnl_cthelper_fill_info(struct sk_buff *skb, u32 portid, u32 seq, u32 type, int event, struct nf_conntrack_helper *helper) { - struct nlmsghdr *nlh; unsigned int flags = portid ? NLM_F_MULTI : 0; + struct nlmsghdr *nlh; int status; event = nfnl_msg_type(NFNL_SUBSYS_CTHELPER, event); @@ -554,7 +566,7 @@ nfnl_cthelper_fill_info(struct sk_buff *skb, u32 portid, u32 seq, u32 type, if (nla_put_be32(skb, NFCTH_PRIV_DATA_LEN, htonl(helper->data_len))) goto nla_put_failure; - if (helper->flags & NF_CT_HELPER_F_CONFIGURED) + if (READ_ONCE(helper->flags) & NF_CT_HELPER_F_CONFIGURED) status = NFCT_HELPER_STATUS_ENABLED; else status = NFCT_HELPER_STATUS_DISABLED; @@ -575,6 +587,7 @@ static int nfnl_cthelper_dump_table(struct sk_buff *skb, struct netlink_callback *cb) { struct nf_conntrack_helper *cur, *last; + unsigned int helper_flags; rcu_read_lock(); last = (struct nf_conntrack_helper *)cb->args[1]; @@ -583,8 +596,10 @@ nfnl_cthelper_dump_table(struct sk_buff *skb, struct netlink_callback *cb) hlist_for_each_entry_rcu(cur, &nf_ct_helper_hash[cb->args[0]], hnode) { + helper_flags = READ_ONCE(cur->flags); + /* skip non-userspace conntrack helpers. */ - if (!(cur->flags & NF_CT_HELPER_F_USERSPACE)) + if (!(helper_flags & NF_CT_HELPER_F_USERSPACE)) continue; if (cb->args[1]) { @@ -644,7 +659,7 @@ static int nfnl_cthelper_get(struct sk_buff *skb, const struct nfnl_info *info, } list_for_each_entry(nlcth, &nfnl_cthelper_list, list) { - cur = &nlcth->helper; + cur = nlcth->helper; if (helper_name && strncmp(cur->name, helper_name, NF_CT_HELPER_NAME_LEN)) continue; @@ -702,7 +717,7 @@ static int nfnl_cthelper_del(struct sk_buff *skb, const struct nfnl_info *info, ret = -ENOENT; list_for_each_entry_safe(nlcth, n, &nfnl_cthelper_list, list) { - cur = &nlcth->helper; + cur = nlcth->helper; j++; if (helper_name && @@ -717,7 +732,6 @@ static int nfnl_cthelper_del(struct sk_buff *skb, const struct nfnl_info *info, if (refcount_dec_if_one(&cur->refcnt)) { found = true; nf_conntrack_helper_unregister(cur); - kfree(cur->expect_policy); list_del(&nlcth->list); kfree(nlcth); @@ -790,10 +804,9 @@ static void __exit nfnl_cthelper_exit(void) nfnetlink_subsys_unregister(&nfnl_cthelper_subsys); list_for_each_entry_safe(nlcth, n, &nfnl_cthelper_list, list) { - cur = &nlcth->helper; + cur = nlcth->helper; nf_conntrack_helper_unregister(cur); - kfree(cur->expect_policy); kfree(nlcth); } } diff --git a/net/netfilter/nfnetlink_cttimeout.c b/net/netfilter/nfnetlink_cttimeout.c index dca6826af7de3..170d3db860c56 100644 --- a/net/netfilter/nfnetlink_cttimeout.c +++ b/net/netfilter/nfnetlink_cttimeout.c @@ -37,11 +37,8 @@ struct ctnl_timeout { struct list_head head; struct list_head free_head; struct rcu_head rcu_head; - refcount_t refcnt; char name[CTNL_TIMEOUT_NAME_MAX]; - - /* must be at the end */ - struct nf_ct_timeout timeout; + struct nf_ct_timeout *timeout; }; struct nfct_timeout_pernet { @@ -132,12 +129,12 @@ static int cttimeout_new_timeout(struct sk_buff *skb, /* You cannot replace one timeout policy by another of * different kind, sorry. */ - if (matching->timeout.l3num != l3num || - matching->timeout.l4proto->l4proto != l4num) + if (matching->timeout->l3num != l3num || + matching->timeout->l4proto->l4proto != l4num) return -EINVAL; - return ctnl_timeout_parse_policy(&matching->timeout.data, - matching->timeout.l4proto, + return ctnl_timeout_parse_policy(&matching->timeout->data, + matching->timeout->l4proto, info->net, cda[CTA_TIMEOUT_DATA]); } @@ -153,26 +150,35 @@ static int cttimeout_new_timeout(struct sk_buff *skb, goto err_proto_put; } - timeout = kzalloc(sizeof(struct ctnl_timeout) + - l4proto->ctnl_timeout.obj_size, GFP_KERNEL); + timeout = kzalloc(sizeof(*timeout), GFP_KERNEL); if (timeout == NULL) { ret = -ENOMEM; goto err_proto_put; } - ret = ctnl_timeout_parse_policy(&timeout->timeout.data, l4proto, + timeout->timeout = kzalloc(sizeof(*timeout->timeout) + + l4proto->ctnl_timeout.obj_size, GFP_KERNEL); + if (!timeout->timeout) { + ret = -ENOMEM; + goto err; + } + + ret = ctnl_timeout_parse_policy(&timeout->timeout->data, l4proto, info->net, cda[CTA_TIMEOUT_DATA]); if (ret < 0) - goto err; + goto err_free_timeout_policy; strcpy(timeout->name, nla_data(cda[CTA_TIMEOUT_NAME])); - timeout->timeout.l3num = l3num; - timeout->timeout.l4proto = l4proto; - refcount_set(&timeout->refcnt, 1); + timeout->timeout->l3num = l3num; + timeout->timeout->l4proto = l4proto; + refcount_set(&timeout->timeout->refcnt, 1); __module_get(THIS_MODULE); list_add_tail_rcu(&timeout->head, &pernet->nfct_timeout_list); return 0; + +err_free_timeout_policy: + kfree(timeout->timeout); err: kfree(timeout); err_proto_put: @@ -185,7 +191,7 @@ ctnl_timeout_fill_info(struct sk_buff *skb, u32 portid, u32 seq, u32 type, { struct nlmsghdr *nlh; unsigned int flags = portid ? NLM_F_MULTI : 0; - const struct nf_conntrack_l4proto *l4proto = timeout->timeout.l4proto; + const struct nf_conntrack_l4proto *l4proto = timeout->timeout->l4proto; struct nlattr *nest_parms; int ret; @@ -197,17 +203,17 @@ ctnl_timeout_fill_info(struct sk_buff *skb, u32 portid, u32 seq, u32 type, if (nla_put_string(skb, CTA_TIMEOUT_NAME, timeout->name) || nla_put_be16(skb, CTA_TIMEOUT_L3PROTO, - htons(timeout->timeout.l3num)) || + htons(timeout->timeout->l3num)) || nla_put_u8(skb, CTA_TIMEOUT_L4PROTO, l4proto->l4proto) || nla_put_be32(skb, CTA_TIMEOUT_USE, - htonl(refcount_read(&timeout->refcnt)))) + htonl(refcount_read(&timeout->timeout->refcnt)))) goto nla_put_failure; nest_parms = nla_nest_start(skb, CTA_TIMEOUT_DATA); if (!nest_parms) goto nla_put_failure; - ret = l4proto->ctnl_timeout.obj_to_nlattr(skb, &timeout->timeout.data); + ret = l4proto->ctnl_timeout.obj_to_nlattr(skb, &timeout->timeout->data); if (ret < 0) goto nla_put_failure; @@ -307,23 +313,17 @@ static int cttimeout_get_timeout(struct sk_buff *skb, return ret; } -/* try to delete object, fail if it is still in use. */ -static int ctnl_timeout_try_del(struct net *net, struct ctnl_timeout *timeout) +static void ctnl_timeout_del(struct net *net, struct ctnl_timeout *timeout) { - int ret = 0; + /* We are protected by nfnl mutex. */ + list_del_rcu(&timeout->head); + nf_ct_untimeout(net, timeout->timeout); - /* We want to avoid races with ctnl_timeout_put. So only when the - * current refcnt is 1, we decrease it to 0. - */ - if (refcount_dec_if_one(&timeout->refcnt)) { - /* We are protected by nfnl mutex. */ - list_del_rcu(&timeout->head); - nf_ct_untimeout(net, &timeout->timeout); - kfree_rcu(timeout, rcu_head); - } else { - ret = -EBUSY; - } - return ret; + if (refcount_dec_and_test(&timeout->timeout->refcnt)) + kfree_rcu(timeout->timeout, rcu); + + kfree_rcu(timeout, rcu_head); + module_put(THIS_MODULE); } static int cttimeout_del_timeout(struct sk_buff *skb, @@ -338,7 +338,7 @@ static int cttimeout_del_timeout(struct sk_buff *skb, if (!cda[CTA_TIMEOUT_NAME]) { list_for_each_entry_safe(cur, tmp, &pernet->nfct_timeout_list, head) - ctnl_timeout_try_del(info->net, cur); + ctnl_timeout_del(info->net, cur); return 0; } @@ -348,10 +348,8 @@ static int cttimeout_del_timeout(struct sk_buff *skb, if (strncmp(cur->name, name, CTNL_TIMEOUT_NAME_MAX) != 0) continue; - ret = ctnl_timeout_try_del(info->net, cur); - if (ret < 0) - return ret; - + ctnl_timeout_del(info->net, cur); + ret = 0; break; } return ret; @@ -511,24 +509,22 @@ static struct nf_ct_timeout *ctnl_timeout_find_get(struct net *net, if (strncmp(timeout->name, name, CTNL_TIMEOUT_NAME_MAX) != 0) continue; - if (!refcount_inc_not_zero(&timeout->refcnt)) + if (!refcount_inc_not_zero(&timeout->timeout->refcnt)) goto err; matching = timeout; + __module_get(THIS_MODULE); break; } err: - return matching ? &matching->timeout : NULL; + return matching ? matching->timeout : NULL; } -static void ctnl_timeout_put(struct nf_ct_timeout *t) +static void ctnl_timeout_put(struct nf_ct_timeout *timeout) { - struct ctnl_timeout *timeout = - container_of(t, struct ctnl_timeout, timeout); + if (refcount_dec_and_test(&timeout->refcnt)) + kfree_rcu(timeout, rcu); - if (refcount_dec_and_test(&timeout->refcnt)) { - kfree_rcu(timeout, rcu_head); - module_put(THIS_MODULE); - } + module_put(THIS_MODULE); } static const struct nfnl_callback cttimeout_cb[IPCTNL_MSG_TIMEOUT_MAX] = { @@ -609,8 +605,11 @@ static void __net_exit cttimeout_net_exit(struct net *net) list_for_each_entry_safe(cur, tmp, &pernet->nfct_timeout_freelist, free_head) { list_del(&cur->free_head); - if (refcount_dec_and_test(&cur->refcnt)) - kfree_rcu(cur, rcu_head); + if (refcount_dec_and_test(&cur->timeout->refcnt)) + kfree_rcu(cur->timeout, rcu); + + kfree_rcu(cur, rcu_head); + module_put(THIS_MODULE); } } @@ -649,24 +648,13 @@ static int __init cttimeout_init(void) return ret; } -static int untimeout(struct nf_conn *ct, void *timeout) -{ - struct nf_conn_timeout *timeout_ext = nf_ct_timeout_find(ct); - - if (timeout_ext) - RCU_INIT_POINTER(timeout_ext->timeout, NULL); - - return 0; -} - static void __exit cttimeout_exit(void) { nfnetlink_subsys_unregister(&cttimeout_subsys); unregister_pernet_subsys(&cttimeout_ops); RCU_INIT_POINTER(nf_ct_timeout_hook, NULL); - - nf_ct_iterate_destroy(untimeout, NULL); + synchronize_net(); } module_init(cttimeout_init); diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index fa36575998616..5fee61b3813cb 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -676,7 +676,7 @@ __build_packet_message(struct nfnl_log_net *log, goto nla_put_failure; if (skb_copy_bits(skb, 0, nla_data(nla), data_len)) - BUG(); + goto nla_put_failure; } nlh->nlmsg_len = inst->skb->tail - old_tail; @@ -698,6 +698,21 @@ static const struct nf_loginfo default_loginfo = { }, }; +static unsigned int nfulnl_get_copy_len(const struct nf_loginfo *li, + const struct sk_buff *skb, + unsigned int copy_len) +{ + unsigned int len = skb->len; + + if ((li->u.ulog.flags & NF_LOG_F_COPY_LEN) && + li->u.ulog.copy_len < copy_len) + copy_len = li->u.ulog.copy_len; + if (!skb_frags_readable(skb)) + len = skb_headlen(skb); + + return min(len, copy_len); +} + /* log handler for internal netfilter logging api */ static void nfulnl_log_packet(struct net *net, @@ -790,14 +805,7 @@ nfulnl_log_packet(struct net *net, break; case NFULNL_COPY_PACKET: - data_len = inst->copy_range; - if ((li->u.ulog.flags & NF_LOG_F_COPY_LEN) && - (li->u.ulog.copy_len < data_len)) - data_len = li->u.ulog.copy_len; - - if (data_len > skb->len) - data_len = skb->len; - + data_len = nfulnl_get_copy_len(li, skb, inst->copy_range); size += nla_total_size(data_len); break; diff --git a/net/netfilter/nfnetlink_osf.c b/net/netfilter/nfnetlink_osf.c index acb753ec5697a..92002079f8eab 100644 --- a/net/netfilter/nfnetlink_osf.c +++ b/net/netfilter/nfnetlink_osf.c @@ -95,11 +95,7 @@ static bool nf_osf_match_one(const struct sk_buff *skb, switch (*optp) { case OSFOPT_MSS: - mss = optp[3]; - mss <<= 8; - mss |= optp[2]; - - mss = ntohs((__force __be16)mss); + mss = get_unaligned_be16(&optp[2]); break; case OSFOPT_TS: break; diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index c5e29fec419bb..d4f5003280d70 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -690,6 +690,17 @@ static int nfqnl_put_master_ifindex(struct sk_buff *nlskb, int attr, } #endif +static unsigned int nfqnl_get_data_len(const struct sk_buff *entskb, + unsigned int copy_range) +{ + unsigned int data_len = entskb->len; + + if (!skb_frags_readable(entskb)) + data_len = skb_headlen(entskb); + + return min(data_len, copy_range); +} + static struct sk_buff * nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, struct nf_queue_entry *entry, @@ -755,10 +766,7 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, nf_queue_checksum_help(entskb)) return NULL; - data_len = READ_ONCE(queue->copy_range); - if (data_len > entskb->len) - data_len = entskb->len; - + data_len = nfqnl_get_data_len(entskb, READ_ONCE(queue->copy_range)); hlen = skb_zerocopy_headlen(entskb); hlen = min_t(unsigned int, hlen, data_len); size += sizeof(struct nlattr) + hlen; @@ -1262,6 +1270,9 @@ dev_cmp(struct nf_queue_entry *entry, unsigned long ifindex) if (physinif == ifindex || physoutif == ifindex) return 1; + + if (entry->bridge_dev && entry->bridge_dev->ifindex == ifindex) + return 1; #endif if (entry->skb_dev && entry->skb_dev->ifindex == ifindex) return 1; diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index 0caa9304d2d03..63864b9282590 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -397,6 +397,22 @@ static int nft_target_validate(const struct nft_ctx *ctx, return 0; } +static int nft_target_bridge_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct xt_target *target = expr->ops->data; + + /* Do not allow UNSPEC to stand-in for NFPROTO_BRIDGE + * targets: they are incompatible. ebtables targets return + * EBT_ACCEPT, DROP and so on which are not compatible with + * NF_ACCEPT, NF_DROP and so on. + */ + if (target->family != NFPROTO_BRIDGE) + return -ENOENT; + + return nft_target_validate(ctx, expr); +} + static void __nft_match_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt, @@ -932,13 +948,15 @@ nft_target_select_ops(const struct nft_ctx *ctx, ops->init = nft_target_init; ops->destroy = nft_target_destroy; ops->dump = nft_target_dump; - ops->validate = nft_target_validate; ops->data = target; - if (family == NFPROTO_BRIDGE) + if (family == NFPROTO_BRIDGE) { ops->eval = nft_target_eval_bridge; - else + ops->validate = nft_target_bridge_validate; + } else { ops->eval = nft_target_eval_xt; + ops->validate = nft_target_validate; + } return ops; err: diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 357513c6dcea0..8a79e9c1e1aed 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -897,8 +897,6 @@ static void nft_ct_timeout_obj_eval(struct nft_object *obj, } } - rcu_assign_pointer(timeout->timeout, priv->timeout); - /* adjust the timeout as per 'new' state. ct is unconfirmed, * so the current timestamp must not be added. */ @@ -949,6 +947,7 @@ static int nft_ct_timeout_obj_init(const struct nft_ctx *ctx, timeout->l3num = l3num; timeout->l4proto = l4proto; + refcount_set(&timeout->refcnt, 1); ret = nf_ct_netns_get(ctx->net, ctx->family); if (ret < 0) @@ -969,10 +968,10 @@ static void nft_ct_timeout_obj_destroy(const struct nft_ctx *ctx, struct nft_ct_timeout_obj *priv = nft_obj_data(obj); struct nf_ct_timeout *timeout = priv->timeout; - nf_queue_nf_hook_drop(ctx->net); nf_ct_untimeout(ctx->net, timeout); nf_ct_netns_put(ctx->net, ctx->family); - kfree_rcu(priv->timeout, rcu); + if (refcount_dec_and_test(&timeout->refcnt)) + kfree_rcu(priv->timeout, rcu); } static int nft_ct_timeout_obj_dump(struct sk_buff *skb, @@ -1147,7 +1146,6 @@ static void nft_ct_helper_obj_eval(struct nft_object *obj, help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); if (help) { rcu_assign_pointer(help->helper, to_assign); - set_bit(IPS_HELPER_BIT, &ct->status); if ((ct->status & IPS_NAT_MASK) && !nfct_seqadj(ct)) if (!nfct_seqadj_ext_add(ct)) @@ -1218,11 +1216,23 @@ struct nft_ct_expect_obj { u32 timeout; }; +static int nft_ct_expect_timeout_get(const struct nlattr *attr, u32 *val) +{ + unsigned long jiffies_val = msecs_to_jiffies(nla_get_u32(attr)); + + if (jiffies_val > UINT_MAX) + return -ERANGE; + + *val = jiffies_val; + return 0; +} + static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, const struct nlattr * const tb[], struct nft_object *obj) { struct nft_ct_expect_obj *priv = nft_obj_data(obj); + int err; if (!tb[NFTA_CT_EXPECT_L4PROTO] || !tb[NFTA_CT_EXPECT_DPORT] || @@ -1257,8 +1267,11 @@ static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, return -EOPNOTSUPP; } + err = nft_ct_expect_timeout_get(tb[NFTA_CT_EXPECT_TIMEOUT], &priv->timeout); + if (err) + return err; + priv->dport = nla_get_be16(tb[NFTA_CT_EXPECT_DPORT]); - priv->timeout = nla_get_u32(tb[NFTA_CT_EXPECT_TIMEOUT]); priv->size = nla_get_u8(tb[NFTA_CT_EXPECT_SIZE]); return nf_ct_netns_get(ctx->net, ctx->family); @@ -1278,7 +1291,7 @@ static int nft_ct_expect_obj_dump(struct sk_buff *skb, if (nla_put_be16(skb, NFTA_CT_EXPECT_L3PROTO, htons(priv->l3num)) || nla_put_u8(skb, NFTA_CT_EXPECT_L4PROTO, priv->l4proto) || nla_put_be16(skb, NFTA_CT_EXPECT_DPORT, priv->dport) || - nla_put_u32(skb, NFTA_CT_EXPECT_TIMEOUT, priv->timeout) || + nla_put_u32(skb, NFTA_CT_EXPECT_TIMEOUT, jiffies_to_msecs(priv->timeout)) || nla_put_u8(skb, NFTA_CT_EXPECT_SIZE, priv->size)) return -1; @@ -1328,7 +1341,7 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj, &ct->tuplehash[!dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, priv->l4proto, NULL, &priv->dport); - exp->timeout.expires = jiffies + priv->timeout * HZ; + exp->timeout += priv->timeout; if (nf_ct_expect_related(exp, 0) != 0) regs->verdict.code = NF_DROP; diff --git a/net/netfilter/nft_fib.c b/net/netfilter/nft_fib.c index a1632e308f182..7529f58a6d36f 100644 --- a/net/netfilter/nft_fib.c +++ b/net/netfilter/nft_fib.c @@ -31,6 +31,15 @@ int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) const struct nft_fib *priv = nft_expr_priv(expr); unsigned int hooks; + switch (ctx->family) { + case NFPROTO_IPV4: + case NFPROTO_IPV6: + case NFPROTO_INET: + break; + default: + return -EOPNOTSUPP; + } + switch (priv->result) { case NFT_FIB_RESULT_OIF: case NFT_FIB_RESULT_OIFNAME: diff --git a/net/netfilter/nft_fib_netdev.c b/net/netfilter/nft_fib_netdev.c index 3f3478abd8454..5774a75440278 100644 --- a/net/netfilter/nft_fib_netdev.c +++ b/net/netfilter/nft_fib_netdev.c @@ -50,6 +50,33 @@ static void nft_fib_netdev_eval(const struct nft_expr *expr, regs->verdict.code = NFT_BREAK; } +static int nft_fib_netdev_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + const struct nft_fib *priv = nft_expr_priv(expr); + unsigned int hooks; + + switch (priv->result) { + case NFT_FIB_RESULT_OIF: + case NFT_FIB_RESULT_OIFNAME: + hooks = (1 << NF_NETDEV_INGRESS); + break; + case NFT_FIB_RESULT_ADDRTYPE: + if (priv->flags & NFTA_FIB_F_IIF) + hooks = (1 << NF_NETDEV_INGRESS); + else if (priv->flags & NFTA_FIB_F_OIF) + hooks = (1 << NF_NETDEV_EGRESS); + else + hooks = (1 << NF_NETDEV_INGRESS) | + (1 << NF_NETDEV_EGRESS); + break; + default: + return -EINVAL; + } + + return nft_chain_validate_hooks(ctx->chain, hooks); +} + static struct nft_expr_type nft_fib_netdev_type; static const struct nft_expr_ops nft_fib_netdev_ops = { .type = &nft_fib_netdev_type, @@ -57,7 +84,7 @@ static const struct nft_expr_ops nft_fib_netdev_ops = { .eval = nft_fib_netdev_eval, .init = nft_fib_init, .dump = nft_fib_dump, - .validate = nft_fib_validate, + .validate = nft_fib_netdev_validate, }; static struct nft_expr_type nft_fib_netdev_type __read_mostly = { diff --git a/net/netfilter/nft_fwd_netdev.c b/net/netfilter/nft_fwd_netdev.c index b9e88d7cf3081..a48c2f765bbaa 100644 --- a/net/netfilter/nft_fwd_netdev.c +++ b/net/netfilter/nft_fwd_netdev.c @@ -95,7 +95,6 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt) { - u8 *nf_dup_skb_recursion = nf_get_nf_dup_skb_recursion(); struct nft_fwd_neigh *priv = nft_expr_priv(expr); void *addr = ®s->data[priv->sreg_addr]; int oif = regs->data[priv->sreg_dev]; @@ -154,13 +153,15 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr, goto out; } - if (*nf_dup_skb_recursion > NF_RECURSION_LIMIT) { + dev = dev_get_by_index_rcu(nft_net(pkt), oif); + if (!dev) { verdict = NF_DROP; goto out; } - dev = dev_get_by_index_rcu(nft_net(pkt), oif); - if (dev == NULL) { + local_bh_disable(); + if (nf_dev_xmit_recursion()) { + local_bh_enable(); verdict = NF_DROP; goto out; } @@ -169,16 +170,18 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr, if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { skb = skb_expand_head(skb, hh_len); if (!skb) { - verdict = NF_STOLEN; + local_bh_enable(); goto out; } } skb->dev = dev; skb_clear_tstamp(skb); - (*nf_dup_skb_recursion)++; + + nf_dev_xmit_recursion_inc(); neigh_xmit(neigh_table, dev, addr, skb); - (*nf_dup_skb_recursion)--; + nf_dev_xmit_recursion_dec(); + local_bh_enable(); out: regs->verdict.code = verdict; } diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index 9fafe5afc490e..8e2c3601133f0 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -103,13 +103,13 @@ void nft_lookup_eval(const struct nft_expr *expr, bool found; ext = nft_set_do_lookup(net, set, ®s->data[priv->sreg]); + if (!ext) + ext = nft_set_catchall_lookup(net, set); + found = !!ext ^ priv->invert; if (!found) { - ext = nft_set_catchall_lookup(net, set); - if (!ext) { - regs->verdict.code = NFT_BREAK; - return; - } + regs->verdict.code = NFT_BREAK; + return; } if (ext) { diff --git a/net/netfilter/nft_meta.c b/net/netfilter/nft_meta.c index 5b25851381e52..db937d66210d9 100644 --- a/net/netfilter/nft_meta.c +++ b/net/netfilter/nft_meta.c @@ -635,8 +635,8 @@ static int nft_meta_get_validate_xfrm(const struct nft_ctx *ctx) #endif } -static int nft_meta_get_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr) +int nft_meta_get_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) { const struct nft_meta *priv = nft_expr_priv(expr); @@ -652,6 +652,7 @@ static int nft_meta_get_validate(const struct nft_ctx *ctx, return 0; } +EXPORT_SYMBOL_GPL(nft_meta_get_validate); int nft_meta_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index 484a5490832e4..4438d05f8018c 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -224,11 +224,17 @@ static int nft_payload_init(const struct nft_ctx *ctx, const struct nlattr * const tb[]) { struct nft_payload *priv = nft_expr_priv(expr); + u32 offset; + int err; priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); - priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); + err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); + if (err < 0) + return err; + priv->offset = offset; + return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], &priv->dreg, NULL, NFT_DATA_VALUE, priv->len); @@ -621,7 +627,8 @@ static int nft_payload_inner_init(const struct nft_ctx *ctx, const struct nlattr * const tb[]) { struct nft_payload *priv = nft_expr_priv(expr); - u32 base; + u32 base, offset; + int err; if (!tb[NFTA_PAYLOAD_BASE] || !tb[NFTA_PAYLOAD_OFFSET] || !tb[NFTA_PAYLOAD_LEN] || !tb[NFTA_PAYLOAD_DREG]) @@ -639,8 +646,11 @@ static int nft_payload_inner_init(const struct nft_ctx *ctx, } priv->base = base; - priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); + err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); + if (err < 0) + return err; + priv->offset = offset; return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], &priv->dreg, NULL, NFT_DATA_VALUE, diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 50d4a4f04309a..61b8601ee3770 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -342,6 +342,8 @@ #include "nft_set_pipapo_avx2.h" #include "nft_set_pipapo.h" +static void nft_pipapo_abort(const struct nft_set *set); + /** * pipapo_refill() - For each set bit, set bits from selected mapping table item * @map: Bitmap to be scanned for set bits @@ -1296,7 +1298,7 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, const u8 *start_p, *end_p; int i, bsize_max, err = 0; - if (!m) + if (!m || m->state == NFT_PIPAPO_CLONE_ERR) return -ENOMEM; if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END)) @@ -1367,8 +1369,10 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, else ret = pipapo_expand(f, start, end, f->groups * f->bb); - if (ret < 0) - return ret; + if (ret < 0) { + err = ret; + goto abort; + } if (f->bsize > bsize_max) bsize_max = f->bsize; @@ -1384,7 +1388,7 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, err = pipapo_realloc_scratch(m, bsize_max); if (err) - return err; + goto abort; m->bsize_max = bsize_max; } else { @@ -1396,7 +1400,26 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, pipapo_map(m, rulemap, e); + m->state = NFT_PIPAPO_CLONE_MOD; return 0; +abort: + DEBUG_NET_WARN_ON_ONCE(m->state == NFT_PIPAPO_CLONE_ERR); + + /* Two rollback cases: + * 1) no previous changes. nft_pipapo_abort is not + * guaranteed to be invoked (there might be no further + * add/delete requests coming after this). + * + * 2) we had previous changes: there are transaction + * records pointing to this set. Leave the rollback to + * the transaction handling. + */ + if (m->state == NFT_PIPAPO_CLONE_NEW) + nft_pipapo_abort(set); /* releases m */ + else + m->state = NFT_PIPAPO_CLONE_ERR; + + return err; } /** @@ -1473,6 +1496,7 @@ static struct nft_pipapo_match *pipapo_clone(struct nft_pipapo_match *old) dst++; } + new->state = NFT_PIPAPO_CLONE_NEW; return new; out_mt: @@ -1896,7 +1920,7 @@ nft_pipapo_deactivate(const struct net *net, const struct nft_set *set, /* removal must occur on priv->clone, if we are low on memory * we have no choice and must fail the removal request. */ - if (!m) + if (!m || m->state == NFT_PIPAPO_CLONE_ERR) return NULL; e = pipapo_get(m, (const u8 *)elem->key.val.data, diff --git a/net/netfilter/nft_set_pipapo.h b/net/netfilter/nft_set_pipapo.h index b82abb03576ed..a19e980d06ef4 100644 --- a/net/netfilter/nft_set_pipapo.h +++ b/net/netfilter/nft_set_pipapo.h @@ -131,9 +131,16 @@ struct nft_pipapo_scratch { unsigned long __map[]; }; +enum nft_pipapo_clone_state { + NFT_PIPAPO_CLONE_NEW, + NFT_PIPAPO_CLONE_MOD, + NFT_PIPAPO_CLONE_ERR, +}; + /** * struct nft_pipapo_match - Data used for lookup and matching * @field_count: Amount of fields in set + * @state: add/delete state; used from control plane * @bsize_max: Maximum lookup table bucket size of all fields, in longs * @scratch: Preallocated per-CPU maps for partial matching results * @rcu: Matching data is swapped on commits @@ -141,6 +148,7 @@ struct nft_pipapo_scratch { */ struct nft_pipapo_match { u8 field_count; + enum nft_pipapo_clone_state state:8; unsigned int bsize_max; struct nft_pipapo_scratch * __percpu *scratch; struct rcu_head rcu; diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index 560fbe6e3f750..b4b67f48ad64e 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -184,10 +184,14 @@ nft_rbtree_get(const struct net *net, const struct nft_set *set, if (!interval || nft_set_elem_expired(interval->from)) return ERR_PTR(-ENOENT); - if (flags & NFT_SET_ELEM_INTERVAL_END) + if (flags & NFT_SET_ELEM_INTERVAL_END) { + if (!interval->to) + return NULL; + rbe = container_of(interval->to, struct nft_rbtree_elem, ext); - else + } else { rbe = container_of(interval->from, struct nft_rbtree_elem, ext); + } return &rbe->priv; } diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index 7641f249614c9..9ed288c9d1688 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -24,14 +24,13 @@ static const struct nla_policy nft_synproxy_policy[NFTA_SYNPROXY_MAX + 1] = { static void nft_synproxy_tcp_options(struct synproxy_options *opts, const struct tcphdr *tcp, struct synproxy_net *snet, - struct nf_synproxy_info *info, - const struct nft_synproxy *priv) + struct nf_synproxy_info *info) { this_cpu_inc(snet->stats->syn_received); if (tcp->ece && tcp->cwr) opts->options |= NF_SYNPROXY_OPT_ECN; - opts->options &= priv->info.options; + opts->options &= info->options; opts->mss_encode = opts->mss_option; opts->mss_option = info->mss; if (opts->options & NF_SYNPROXY_OPT_TIMESTAMP) @@ -56,7 +55,7 @@ static void nft_synproxy_eval_v4(const struct nft_synproxy *priv, if (tcp->syn) { /* Initial SYN from client */ - nft_synproxy_tcp_options(opts, tcp, snet, &info, priv); + nft_synproxy_tcp_options(opts, tcp, snet, &info); synproxy_send_client_synack(net, skb, tcp, opts); consume_skb(skb); regs->verdict.code = NF_STOLEN; @@ -87,7 +86,7 @@ static void nft_synproxy_eval_v6(const struct nft_synproxy *priv, if (tcp->syn) { /* Initial SYN from client */ - nft_synproxy_tcp_options(opts, tcp, snet, &info, priv); + nft_synproxy_tcp_options(opts, tcp, snet, &info); synproxy_send_client_synack_ipv6(net, skb, tcp, opts); consume_skb(skb); regs->verdict.code = NF_STOLEN; diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c index d2aeacf94230f..b94f004d5f5c2 100644 --- a/net/netfilter/xt_CT.c +++ b/net/netfilter/xt_CT.c @@ -284,7 +284,7 @@ static void xt_ct_tg_destroy(const struct xt_tgdtor_param *par, struct nf_conn_help *help; if (ct) { - if (info->helper[0] || info->timeout[0]) + if (info->helper[0]) nf_queue_nf_hook_drop(par->net); help = nfct_help(ct); diff --git a/net/netfilter/xt_cluster.c b/net/netfilter/xt_cluster.c index 908fd5f2c3c84..eaf2511d63f0e 100644 --- a/net/netfilter/xt_cluster.c +++ b/net/netfilter/xt_cluster.c @@ -107,7 +107,7 @@ xt_cluster_mt(const struct sk_buff *skb, struct xt_action_param *par) } ct = nf_ct_get(skb, &ctinfo); - if (ct == NULL) + if (!ct || nf_ct_is_template(ct)) return false; if (ct->master) diff --git a/net/netfilter/xt_connmark.c b/net/netfilter/xt_connmark.c index 4277084de2e70..2cf27f7d59b95 100644 --- a/net/netfilter/xt_connmark.c +++ b/net/netfilter/xt_connmark.c @@ -112,6 +112,16 @@ static int connmark_tg_check(const struct xt_tgchk_param *par) return ret; } +static int connmark_tg_check_v2(const struct xt_tgchk_param *par) +{ + const struct xt_connmark_tginfo2 *info = par->targinfo; + + if (info->shift_dir > D_SHIFT_RIGHT || info->shift_bits >= 32) + return -EINVAL; + + return connmark_tg_check(par); +} + static void connmark_tg_destroy(const struct xt_tgdtor_param *par) { nf_ct_netns_put(par->net, par->family); @@ -162,7 +172,7 @@ static struct xt_target connmark_tg_reg[] __read_mostly = { .name = "CONNMARK", .revision = 2, .family = NFPROTO_IPV4, - .checkentry = connmark_tg_check, + .checkentry = connmark_tg_check_v2, .target = connmark_tg_v2, .targetsize = sizeof(struct xt_connmark_tginfo2), .destroy = connmark_tg_destroy, @@ -183,7 +193,7 @@ static struct xt_target connmark_tg_reg[] __read_mostly = { .name = "CONNMARK", .revision = 2, .family = NFPROTO_IPV6, - .checkentry = connmark_tg_check, + .checkentry = connmark_tg_check_v2, .target = connmark_tg_v2, .targetsize = sizeof(struct xt_connmark_tginfo2), .destroy = connmark_tg_destroy, diff --git a/net/netfilter/xt_nat.c b/net/netfilter/xt_nat.c index b4f7bbc3f3caf..51c7f7ce88d9a 100644 --- a/net/netfilter/xt_nat.c +++ b/net/netfilter/xt_nat.c @@ -26,6 +26,15 @@ static int xt_nat_checkentry_v0(const struct xt_tgchk_param *par) static int xt_nat_checkentry(const struct xt_tgchk_param *par) { + switch (par->family) { + case NFPROTO_IPV4: + case NFPROTO_IPV6: + case NFPROTO_INET: + break; + default: + return -EINVAL; + } + return nf_ct_netns_get(par->net, par->family); } diff --git a/net/netfilter/xt_physdev.c b/net/netfilter/xt_physdev.c index dd98f758176c2..a388881c68d42 100644 --- a/net/netfilter/xt_physdev.c +++ b/net/netfilter/xt_physdev.c @@ -130,11 +130,6 @@ static int physdev_mt_check(const struct xt_mtchk_param *par) if (X(physoutdev)) return -ENAMETOOLONG; } - - if (X(in_mask)) - return -ENAMETOOLONG; - if (X(out_mask)) - return -ENAMETOOLONG; #undef X if (!brnf_probed) { diff --git a/net/netfilter/xt_rateest.c b/net/netfilter/xt_rateest.c index b1d736c15fcbe..7c05b63425784 100644 --- a/net/netfilter/xt_rateest.c +++ b/net/netfilter/xt_rateest.c @@ -16,7 +16,7 @@ xt_rateest_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_rateest_match_info *info = par->matchinfo; struct gnet_stats_rate_est64 sample = {0}; - u_int32_t bps1, bps2, pps1, pps2; + u64 bps1, bps2, pps1, pps2; bool ret = true; gen_estimator_read(&info->est1->rate_est, &sample); diff --git a/net/netfilter/xt_u32.c b/net/netfilter/xt_u32.c index 117d4615d6684..dabbaa742874c 100644 --- a/net/netfilter/xt_u32.c +++ b/net/netfilter/xt_u32.c @@ -14,8 +14,8 @@ #include #include -static bool u32_match_it(const struct xt_u32 *data, - const struct sk_buff *skb) +static int u32_match_it(const struct xt_u32 *data, + const struct sk_buff *skb) { const struct xt_u32_test *ct; unsigned int testind; @@ -40,7 +40,8 @@ static bool u32_match_it(const struct xt_u32 *data, return false; if (skb_copy_bits(skb, pos, &n, sizeof(n)) < 0) - BUG(); + return -1; + val = ntohl(n); nnums = ct->nnums; @@ -68,7 +69,7 @@ static bool u32_match_it(const struct xt_u32 *data, if (skb_copy_bits(skb, at + pos, &n, sizeof(n)) < 0) - BUG(); + return -1; val = ntohl(n); break; } @@ -90,9 +91,14 @@ static bool u32_match_it(const struct xt_u32 *data, static bool u32_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_u32 *data = par->matchinfo; - bool ret; + int ret; ret = u32_match_it(data, skb); + if (ret < 0) { + par->hotdrop = true; + return false; + } + return ret ^ data->invert; } @@ -100,7 +106,7 @@ static int u32_mt_checkentry(const struct xt_mtchk_param *par) { const struct xt_u32 *data = par->matchinfo; const struct xt_u32_test *ct; - unsigned int i; + unsigned int i, j; if (data->ntests > ARRAY_SIZE(data->tests)) return -EINVAL; @@ -111,6 +117,16 @@ static int u32_mt_checkentry(const struct xt_mtchk_param *par) if (ct->nnums > ARRAY_SIZE(ct->location) || ct->nvalues > ARRAY_SIZE(ct->value)) return -EINVAL; + + for (j = 1; j < ct->nnums; ++j) { + switch (ct->location[j].nextop) { + case XT_U32_LEFTSH: + case XT_U32_RIGHTSH: + if (ct->location[j].number >= 32) + return -EINVAL; + break; + } + } } return 0; diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index 7c9256572284f..c6fd9c424e8fe 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -1797,10 +1797,10 @@ static int ovs_ct_limit_get_zone_limit(struct net *net, } else { rcu_read_lock(); limit = ct_limit_get(info, zone); - rcu_read_unlock(); err = __ovs_ct_limit_get_zone_limit( net, info->data, zone, limit, reply); + rcu_read_unlock(); if (err) return err; } diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c index 13052408a132f..d8079dee700ed 100644 --- a/net/openvswitch/flow_netlink.c +++ b/net/openvswitch/flow_netlink.c @@ -2496,13 +2496,56 @@ static inline int add_nested_action_start(struct sw_flow_actions **sfa, return used; } -static inline void add_nested_action_end(struct sw_flow_actions *sfa, - int st_offset) +static inline int add_nested_action_end(struct sw_flow_actions *sfa, + int st_offset) { - struct nlattr *a = (struct nlattr *) ((unsigned char *)sfa->actions + - st_offset); + struct nlattr *a; + u32 attr_len; + + if (WARN_ON_ONCE(st_offset < 0 || + (u32)st_offset > sfa->actions_len)) + return -EINVAL; + + attr_len = sfa->actions_len - (u32)st_offset; + if (WARN_ON_ONCE(attr_len < NLA_HDRLEN)) + return -EINVAL; - a->nla_len = sfa->actions_len - st_offset; + if (attr_len > U16_MAX) + return -EMSGSIZE; + + a = (struct nlattr *)((u8 *)sfa->actions + st_offset); + a->nla_len = attr_len; + return 0; +} + +/* Free the generated action-list tail at @start and truncate it. + * If @nested, @start points to its containing nlattr header. + */ +static void ovs_nla_trim(struct sw_flow_actions *sfa, int start, bool nested) +{ + const struct nlattr *actions; + u32 len; + + if (start < 0) + return; + + if (WARN_ON_ONCE((u32)start > sfa->actions_len)) + return; + + actions = (const struct nlattr *)((u8 *)sfa->actions + start); + len = sfa->actions_len - (u32)start; + + if (nested) { + if (len < NLA_HDRLEN) + goto out; + + actions = (const struct nlattr *)((u8 *)actions + NLA_HDRLEN); + len -= NLA_HDRLEN; + } + + ovs_nla_free_nested_actions(actions, len); +out: + sfa->actions_len = start; } static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr, @@ -2522,6 +2565,7 @@ static int validate_and_copy_sample(struct net *net, const struct nlattr *attr, const struct nlattr *attrs[OVS_SAMPLE_ATTR_MAX + 1]; const struct nlattr *probability, *actions; const struct nlattr *a; + int actions_start; int rem, start, err; struct sample_arg arg; @@ -2565,18 +2609,27 @@ static int validate_and_copy_sample(struct net *net, const struct nlattr *attr, err = ovs_nla_add_action(sfa, OVS_SAMPLE_ATTR_ARG, &arg, sizeof(arg), log); if (err) - return err; + goto err; + actions_start = (*sfa)->actions_len; err = __ovs_nla_copy_actions(net, actions, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, actions_start, false); +err: + (*sfa)->actions_len = start; + return err; } static int validate_and_copy_dec_ttl(struct net *net, @@ -2624,18 +2677,31 @@ static int validate_and_copy_dec_ttl(struct net *net, return start; action_start = add_nested_action_start(sfa, OVS_DEC_TTL_ATTR_ACTION, log); - if (action_start < 0) - return action_start; + if (action_start < 0) { + err = action_start; + goto err; + } err = __ovs_nla_copy_actions(net, actions, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; + + err = add_nested_action_end(*sfa, action_start); + if (err) + goto err_free; - add_nested_action_end(*sfa, action_start); - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, action_start, true); +err: + (*sfa)->actions_len = start; + return err; } static int validate_and_copy_clone(struct net *net, @@ -2646,6 +2712,7 @@ static int validate_and_copy_clone(struct net *net, u32 mpls_label_count, bool log, bool last, u32 depth) { + int actions_start; int start, err; u32 exec; @@ -2661,17 +2728,26 @@ static int validate_and_copy_clone(struct net *net, err = ovs_nla_add_action(sfa, OVS_CLONE_ATTR_EXEC, &exec, sizeof(exec), log); if (err) - return err; + goto err; + actions_start = (*sfa)->actions_len; err = __ovs_nla_copy_actions(net, attr, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, actions_start, false); +err: + (*sfa)->actions_len = start; + return err; } void ovs_match_init(struct sw_flow_match *match, @@ -2763,20 +2839,20 @@ static int validate_and_copy_set_tun(const struct nlattr *attr, tun_dst = metadata_dst_alloc(key.tun_opts_len, METADATA_IP_TUNNEL, GFP_KERNEL); - if (!tun_dst) - return -ENOMEM; + if (!tun_dst) { + err = -ENOMEM; + goto err; + } err = dst_cache_init(&tun_dst->u.tun_info.dst_cache, GFP_KERNEL); - if (err) { - dst_release((struct dst_entry *)tun_dst); - return err; - } + if (err) + goto err_free_tun_dst; a = __add_action(sfa, OVS_KEY_ATTR_TUNNEL_INFO, NULL, sizeof(*ovs_tun), log); if (IS_ERR(a)) { - dst_release((struct dst_entry *)tun_dst); - return PTR_ERR(a); + err = PTR_ERR(a); + goto err_free_tun_dst; } ovs_tun = nla_data(a); @@ -2797,8 +2873,16 @@ static int validate_and_copy_set_tun(const struct nlattr *attr, ip_tunnel_info_opts_set(tun_info, TUN_METADATA_OPTS(&key, key.tun_opts_len), key.tun_opts_len, dst_opt_type); - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (WARN_ON_ONCE(err)) + goto err_free_tun_dst; + + return 0; +err_free_tun_dst: + dst_release((struct dst_entry *)tun_dst); +err: + (*sfa)->actions_len = start; return err; } @@ -2971,7 +3055,7 @@ static int validate_set(const struct nlattr *a, /* Convert non-masked non-tunnel set actions to masked set actions. */ if (!masked && key_type != OVS_KEY_ATTR_TUNNEL) { - int start, len = key_len * 2; + int err, start, len = key_len * 2; struct nlattr *at; *skip_copy = true; @@ -2983,8 +3067,11 @@ static int validate_set(const struct nlattr *a, return start; at = __add_action(sfa, key_type, NULL, len, log); - if (IS_ERR(at)) - return PTR_ERR(at); + if (IS_ERR(at)) { + err = PTR_ERR(at); + (*sfa)->actions_len = start; + return err; + } memcpy(nla_data(at), nla_data(ovs_key), key_len); /* Key. */ memset(nla_data(at) + key_len, 0xff, key_len); /* Mask. */ @@ -2994,7 +3081,11 @@ static int validate_set(const struct nlattr *a, mask->ipv6_label &= htonl(0x000FFFFF); } - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (WARN_ON_ONCE(err)) { + (*sfa)->actions_len = start; + return err; + } } return 0; @@ -3040,7 +3131,8 @@ static int validate_and_copy_check_pkt_len(struct net *net, const struct nlattr *acts_if_greater, *acts_if_lesser_eq; struct nlattr *a[OVS_CHECK_PKT_LEN_ATTR_MAX + 1]; struct check_pkt_len_arg arg; - int nested_acts_start; + int greater_acts_start = -1; + int lesser_acts_start = -1; int start, err; err = nla_parse_deprecated_strict(a, OVS_CHECK_PKT_LEN_ATTR_MAX, @@ -3075,37 +3167,58 @@ static int validate_and_copy_check_pkt_len(struct net *net, err = ovs_nla_add_action(sfa, OVS_CHECK_PKT_LEN_ATTR_ARG, &arg, sizeof(arg), log); if (err) - return err; + goto err_free; - nested_acts_start = add_nested_action_start(sfa, - OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL, log); - if (nested_acts_start < 0) - return nested_acts_start; + lesser_acts_start = + add_nested_action_start(sfa, + OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL, + log); + if (lesser_acts_start < 0) { + err = lesser_acts_start; + goto err_free; + } err = __ovs_nla_copy_actions(net, acts_if_lesser_eq, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; - add_nested_action_end(*sfa, nested_acts_start); + err = add_nested_action_end(*sfa, lesser_acts_start); + if (err) + goto err_free; - nested_acts_start = add_nested_action_start(sfa, - OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER, log); - if (nested_acts_start < 0) - return nested_acts_start; + greater_acts_start = + add_nested_action_start(sfa, + OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER, + log); + if (greater_acts_start < 0) { + err = greater_acts_start; + goto err_free; + } err = __ovs_nla_copy_actions(net, acts_if_greater, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; + + err = add_nested_action_end(*sfa, greater_acts_start); + if (err) + goto err_free; - add_nested_action_end(*sfa, nested_acts_start); - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, greater_acts_start, true); + ovs_nla_trim(*sfa, lesser_acts_start, true); + ovs_nla_trim(*sfa, start, false); + return err; } static int validate_psample(const struct nlattr *attr) diff --git a/net/psample/psample.c b/net/psample/psample.c index 7763662036fb7..c112e1f0ccacb 100644 --- a/net/psample/psample.c +++ b/net/psample/psample.c @@ -476,15 +476,17 @@ void psample_sample_packet(struct psample_group *group, goto error; if (data_len) { - int nla_len = nla_total_size(data_len); + int nla_len = nla_attr_size(data_len); struct nlattr *nla; nla = skb_put(nl_skb, nla_len); nla->nla_type = PSAMPLE_ATTR_DATA; - nla->nla_len = nla_attr_size(data_len); + nla->nla_len = nla_len; if (skb_copy_bits(skb, 0, nla_data(nla), data_len)) goto error; + + skb_put_zero(nl_skb, nla_padlen(data_len)); } #ifdef CONFIG_INET diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c index db823177e6361..2288159f5b1bb 100644 --- a/net/qrtr/af_qrtr.c +++ b/net/qrtr/af_qrtr.c @@ -496,7 +496,7 @@ int qrtr_endpoint_post(struct qrtr_endpoint *ep, const void *data, size_t len) if (cb->dst_port == QRTR_PORT_CTRL_LEGACY) cb->dst_port = QRTR_PORT_CTRL; - if (!size || len != ALIGN(size, 4) + hdrlen) + if (!size || size > len || len != ALIGN(size, 4) + hdrlen) goto err; if ((cb->type == QRTR_TYPE_NEW_SERVER || diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index 98f2165159d72..d2b31d15851b9 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -650,7 +650,9 @@ enum rxrpc_call_event { enum rxrpc_call_state { RXRPC_CALL_UNINITIALISED, RXRPC_CALL_CLIENT_AWAIT_CONN, /* - client waiting for connection to become available */ + RXRPC_CALL_CLIENT_PRE_SEND, /* - client is connected, but hasn't sent anything yet */ RXRPC_CALL_CLIENT_SEND_REQUEST, /* - client sending request phase */ + RXRPC_CALL_CLIENT_AWAIT_ACK, /* - client awaiting ACKs of request */ RXRPC_CALL_CLIENT_AWAIT_REPLY, /* - client awaiting reply */ RXRPC_CALL_CLIENT_RECV_REPLY, /* - client receiving reply phase */ RXRPC_CALL_SERVER_PREALLOC, /* - service preallocation */ @@ -1355,9 +1357,9 @@ static inline struct rxrpc_net *rxrpc_net(struct net *net) } /* - * out_of_band.c + * oob.c */ -void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb); +bool rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb); void rxrpc_add_pending_oob(struct rxrpc_sock *rx, struct sk_buff *skb); int rxrpc_sendmsg_oob(struct rxrpc_sock *rx, struct msghdr *msg, size_t len); diff --git a/net/rxrpc/call_accept.c b/net/rxrpc/call_accept.c index ee2d1319e69a8..47824120f1da1 100644 --- a/net/rxrpc/call_accept.c +++ b/net/rxrpc/call_accept.c @@ -471,13 +471,26 @@ int rxrpc_kernel_charge_accept(struct socket *sock, rxrpc_notify_rx_t notify_rx, unsigned long user_call_ID, gfp_t gfp, unsigned int debug_id) { - struct rxrpc_sock *rx = rxrpc_sk(sock->sk); - struct rxrpc_backlog *b = rx->backlog; + struct rxrpc_backlog *b; + struct rxrpc_sock *rx; + struct sock *sk; + int ret; - if (sock->sk->sk_state == RXRPC_CLOSE) - return -ESHUTDOWN; + sk = sock->sk; + rx = rxrpc_sk(sk); + + lock_sock(sk); + if (sk->sk_state != RXRPC_SERVER_LISTENING || !rx->backlog) { + ret = -ESHUTDOWN; + goto out; + } + + b = rx->backlog; + ret = rxrpc_service_prealloc_one(rx, b, notify_rx, user_call_ID, + gfp, debug_id); - return rxrpc_service_prealloc_one(rx, b, notify_rx, user_call_ID, - gfp, debug_id); +out: + release_sock(sk); + return ret; } EXPORT_SYMBOL(rxrpc_kernel_charge_accept); diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c index fec59d9338b9f..21be9c86d7a74 100644 --- a/net/rxrpc/call_event.c +++ b/net/rxrpc/call_event.c @@ -178,7 +178,7 @@ static void rxrpc_close_tx_phase(struct rxrpc_call *call) switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: - rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_REPLY); + rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_ACK); break; case RXRPC_CALL_SERVER_SEND_REPLY: rxrpc_set_call_state(call, RXRPC_CALL_SERVER_AWAIT_ACK); @@ -244,6 +244,8 @@ static void rxrpc_transmit_fresh_data(struct rxrpc_call *call, unsigned int limi break; } while (req.n < limit && before(seq, send_top)); + if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_PRE_SEND) + rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_SEND_REQUEST); if (txb->flags & RXRPC_LAST_PACKET) { rxrpc_close_tx_phase(call); tq = NULL; @@ -267,6 +269,7 @@ void rxrpc_transmit_some_data(struct rxrpc_call *call, unsigned int limit, fallthrough; case RXRPC_CALL_SERVER_SEND_REPLY: + case RXRPC_CALL_CLIENT_PRE_SEND: case RXRPC_CALL_CLIENT_SEND_REQUEST: if (!rxrpc_tx_window_space(call)) return; diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index fcb9d38bb5214..817ed9acb91e6 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -18,7 +18,9 @@ const char *const rxrpc_call_states[NR__RXRPC_CALL_STATES] = { [RXRPC_CALL_UNINITIALISED] = "Uninit ", [RXRPC_CALL_CLIENT_AWAIT_CONN] = "ClWtConn", + [RXRPC_CALL_CLIENT_PRE_SEND] = "ClPreSnd", [RXRPC_CALL_CLIENT_SEND_REQUEST] = "ClSndReq", + [RXRPC_CALL_CLIENT_AWAIT_ACK] = "ClAwtAck", [RXRPC_CALL_CLIENT_AWAIT_REPLY] = "ClAwtRpl", [RXRPC_CALL_CLIENT_RECV_REPLY] = "ClRcvRpl", [RXRPC_CALL_SERVER_PREALLOC] = "SvPrealc", diff --git a/net/rxrpc/conn_client.c b/net/rxrpc/conn_client.c index 9b757798deddb..48519f0de1853 100644 --- a/net/rxrpc/conn_client.c +++ b/net/rxrpc/conn_client.c @@ -449,7 +449,7 @@ static void rxrpc_activate_one_channel(struct rxrpc_connection *conn, trace_rxrpc_connect_call(call); call->tx_last_sent = ktime_get_real(); rxrpc_start_call_timer(call); - rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_SEND_REQUEST); + rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_PRE_SEND); wake_up(&call->waitq); } diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index c96ca615b787c..611c790bc6d0c 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -436,7 +436,7 @@ static bool rxrpc_post_challenge(struct rxrpc_connection *conn, struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rxrpc_call *call = NULL; struct rxrpc_sock *rx; - bool respond = false; + bool respond = false, queued = false; sp->chall.conn = rxrpc_get_connection(conn, rxrpc_conn_get_challenge_input); @@ -472,8 +472,13 @@ static bool rxrpc_post_challenge(struct rxrpc_connection *conn, } if (call) - rxrpc_notify_socket_oob(call, skb); + queued = rxrpc_notify_socket_oob(call, skb); rcu_read_unlock(); + if (call && !queued) { + rxrpc_put_connection(conn, rxrpc_conn_put_challenge_input); + sp->chall.conn = NULL; + return false; + } if (!call) rxrpc_post_packet_to_conn(conn, skb); diff --git a/net/rxrpc/input.c b/net/rxrpc/input.c index ce761466b02d6..73cafe6bfa9f5 100644 --- a/net/rxrpc/input.c +++ b/net/rxrpc/input.c @@ -181,7 +181,8 @@ void rxrpc_congestion_degrade(struct rxrpc_call *call) if (call->cong_ca_state != RXRPC_CA_SLOW_START && call->cong_ca_state != RXRPC_CA_CONGEST_AVOIDANCE) return; - if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_REPLY) + if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_ACK || + __rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_REPLY) return; rtt = ns_to_ktime(call->srtt_us * (NSEC_PER_USEC / 8)); @@ -236,6 +237,9 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, call->acks_lowest_nak = to; } + if (after(seq, to)) + return false; + /* We may have a left over fully-consumed buffer at the front that we * couldn't drop before (rotate_and_keep below). */ @@ -247,7 +251,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, tq = call->tx_queue; } - do { + while (before_eq(seq, to)) { unsigned int ix = seq - call->tx_qbase; _debug("tq=%x seq=%x i=%d f=%x", tq->qbase, seq, ix, tq->bufs[ix]->flags); @@ -317,8 +321,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, break; } } - - } while (before_eq(seq, to)); + } if (trace) trace_rxrpc_rack_update(call, summary); @@ -356,6 +359,7 @@ static void rxrpc_end_tx_phase(struct rxrpc_call *call, bool reply_begun, switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: + case RXRPC_CALL_CLIENT_AWAIT_ACK: case RXRPC_CALL_CLIENT_AWAIT_REPLY: if (reply_begun) { rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_RECV_REPLY); @@ -392,6 +396,14 @@ static bool rxrpc_receiving_reply(struct rxrpc_call *call) trace_rxrpc_timer_can(call, rxrpc_timer_trace_delayed_ack); } + /* Deal with an apparent reply coming in before we've got the request + * queued or transmitted. + */ + if (!test_bit(RXRPC_CALL_EXPOSED, &call->flags)) { + rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply); + return false; + } + if (!test_bit(RXRPC_CALL_TX_LAST, &call->flags)) { if (!rxrpc_rotate_tx_window(call, top, &summary)) { rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply); @@ -694,6 +706,7 @@ static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb) switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: + case RXRPC_CALL_CLIENT_AWAIT_ACK: case RXRPC_CALL_CLIENT_AWAIT_REPLY: /* Received data implicitly ACKs all of the request * packets we sent when we're acting as a client. @@ -1154,10 +1167,12 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) if (hard_ack + 1 == 0) return rxrpc_proto_abort(call, 0, rxrpc_eproto_ackr_zero); - /* Ignore ACKs unless we are or have just been transmitting. */ + /* Ignore ACKs unless we are transmitting or are waiting for + * acknowledgement of the packets we've just been transmitting. + */ switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: - case RXRPC_CALL_CLIENT_AWAIT_REPLY: + case RXRPC_CALL_CLIENT_AWAIT_ACK: case RXRPC_CALL_SERVER_SEND_REPLY: case RXRPC_CALL_SERVER_AWAIT_ACK: break; @@ -1215,7 +1230,17 @@ static void rxrpc_input_ackall(struct rxrpc_call *call, struct sk_buff *skb) { struct rxrpc_ack_summary summary = { 0 }; - if (rxrpc_rotate_tx_window(call, call->tx_top, &summary)) + switch (__rxrpc_call_state(call)) { + case RXRPC_CALL_CLIENT_SEND_REQUEST: + case RXRPC_CALL_CLIENT_AWAIT_ACK: + case RXRPC_CALL_SERVER_SEND_REPLY: + case RXRPC_CALL_SERVER_AWAIT_ACK: + break; + default: + return; + } + + if (rxrpc_rotate_tx_window(call, call->tx_transmitted, &summary)) rxrpc_end_tx_phase(call, false, rxrpc_eproto_unexpected_ackall); } diff --git a/net/rxrpc/local_object.c b/net/rxrpc/local_object.c index 111f574fe667d..169f9dfdaa77f 100644 --- a/net/rxrpc/local_object.c +++ b/net/rxrpc/local_object.c @@ -194,7 +194,7 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net) tuncfg.encap_rcv = rxrpc_encap_rcv; tuncfg.encap_err_rcv = rxrpc_encap_err_rcv; tuncfg.sk_user_data = local; - setup_udp_tunnel_sock(net, local->socket, &tuncfg); + setup_udp_tunnel_sock(net, local->socket->sk, &tuncfg); /* set the socket up */ usk = local->socket->sk; diff --git a/net/rxrpc/oob.c b/net/rxrpc/oob.c index 05ca9c1faa577..c80ee2487d094 100644 --- a/net/rxrpc/oob.c +++ b/net/rxrpc/oob.c @@ -32,11 +32,12 @@ struct rxrpc_oob_params { * Post an out-of-band message for attention by the socket or kernel service * associated with a reference call. */ -void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) +bool rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rxrpc_sock *rx; struct sock *sk; + bool queued = false; rcu_read_lock(); @@ -49,6 +50,7 @@ void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) skb->skb_mstamp_ns = rx->oob_id_counter++; rxrpc_get_skb(skb, rxrpc_skb_get_post_oob); skb_queue_tail(&rx->recvmsg_oobq, skb); + queued = true; trace_rxrpc_notify_socket(call->debug_id, sp->hdr.serial); if (rx->app_ops) @@ -56,11 +58,12 @@ void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) } spin_unlock_irq(&rx->recvmsg_lock); - if (!rx->app_ops && !sock_flag(sk, SOCK_DEAD)) + if (queued && !rx->app_ops && !sock_flag(sk, SOCK_DEAD)) sk->sk_data_ready(sk); } rcu_read_unlock(); + return queued; } /* @@ -210,6 +213,11 @@ static int rxrpc_respond_to_oob(struct rxrpc_sock *rx, break; } + switch (skb->mark) { + case RXRPC_OOB_CHALLENGE: + rxrpc_put_connection(sp->chall.conn, rxrpc_conn_put_oob); + break; + } rxrpc_free_skb(skb, rxrpc_skb_put_oob); return ret; } diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c index c940600117a48..efcba4b2e74f0 100644 --- a/net/rxrpc/recvmsg.c +++ b/net/rxrpc/recvmsg.c @@ -27,8 +27,6 @@ void rxrpc_notify_socket(struct rxrpc_call *call) _enter("%d", call->debug_id); - if (!list_empty(&call->recvmsg_link)) - return; if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) { rxrpc_see_call(call, rxrpc_call_see_notify_released); return; @@ -161,7 +159,7 @@ static int rxrpc_verify_data(struct rxrpc_call *call, struct sk_buff *skb) struct rxrpc_skb_priv *sp = rxrpc_skb(skb); int ret; - if (sp->len > call->rx_dec_bsize) { + if (sp->len > call->rx_dec_bsize || !call->rx_dec_buffer) { /* Make sure we can hold a 1412-byte jumbo subpacket and make * sure that the buffer size is aligned to a crypto blocksize. */ @@ -262,12 +260,13 @@ static int rxrpc_recvmsg_oob(struct socket *sock, struct msghdr *msg, break; } - if (!(flags & MSG_PEEK)) + if (!(flags & MSG_PEEK)) { skb_unlink(skb, &rx->recvmsg_oobq); - if (need_response) - rxrpc_add_pending_oob(rx, skb); - else - rxrpc_free_skb(skb, rxrpc_skb_put_oob); + if (need_response) + rxrpc_add_pending_oob(rx, skb); + else + rxrpc_free_skb(skb, rxrpc_skb_put_oob); + } return ret; } @@ -437,7 +436,8 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, return -EAGAIN; } - if (list_empty(&rx->recvmsg_q)) { + if (list_empty(&rx->recvmsg_q) && + skb_queue_empty_lockless(&rx->recvmsg_oobq)) { ret = -EWOULDBLOCK; if (timeo == 0) { call = NULL; @@ -470,7 +470,7 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, release_sock(&rx->sk); if (ret == -EAGAIN) goto try_again; - goto error_no_call; + goto error_trace; } /* Find the next call and dequeue it if we're not just peeking. If we @@ -529,8 +529,7 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) { rxrpc_see_call(call, rxrpc_call_see_already_released); mutex_unlock(&call->user_mutex); - if (!(flags & MSG_PEEK)) - rxrpc_put_call(call, rxrpc_call_put_recvmsg); + rxrpc_put_call(call, rxrpc_call_put_recvmsg); goto try_again; } diff --git a/net/rxrpc/rxgk.c b/net/rxrpc/rxgk.c index a1ee102abae18..77a67ace1d248 100644 --- a/net/rxrpc/rxgk.c +++ b/net/rxrpc/rxgk.c @@ -687,16 +687,17 @@ static int rxgk_issue_challenge(struct rxrpc_connection *conn) ret = do_udp_sendmsg(conn->local->socket, &msg, len); if (ret > 0) rxrpc_peer_mark_tx(conn->peer); - __free_page(page); if (ret < 0) { trace_rxrpc_tx_fail(conn->debug_id, serial, ret, rxrpc_tx_point_rxgk_challenge); + __free_page(page); return -EAGAIN; } trace_rxrpc_tx_packet(conn->debug_id, whdr, rxrpc_tx_point_rxgk_challenge); + __free_page(page); _leave(" = 0"); return 0; } diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index c35de4fd75e31..ed2c9a51005ad 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -366,7 +366,8 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, if (state >= RXRPC_CALL_COMPLETE) goto maybe_error; ret = -EPROTO; - if (state != RXRPC_CALL_CLIENT_SEND_REQUEST && + if (state != RXRPC_CALL_CLIENT_PRE_SEND && + state != RXRPC_CALL_CLIENT_SEND_REQUEST && state != RXRPC_CALL_SERVER_ACK_REQUEST && state != RXRPC_CALL_SERVER_SEND_REPLY) { /* Request phase complete for this client call */ diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 04ea11c90e033..b4a7e1c63cb87 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -148,10 +148,15 @@ static void offload_action_hw_count_dec(struct tc_action *act, static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) { - if (is_tcf_pedit(act)) - return tcf_pedit_nkeys(act); - else - return 1; + unsigned int count; + + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + count = tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + return count; + } + return 1; } static bool tc_act_skip_hw(u32 flags) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index c2b5bc19e0911..284800b7d3c55 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -44,7 +44,7 @@ TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb, tcf_lastuse_update(&prog->tcf_tm); bstats_update(this_cpu_ptr(prog->common.cpu_bstats), skb); - filter = rcu_dereference(prog->filter); + filter = rcu_dereference_bh(prog->filter); if (at_ingress) { __skb_push(skb, skb->mac_len); filter_res = bpf_prog_run_data_pointers(filter, skb); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 6158e13c98d35..be535a261fa04 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -844,11 +844,11 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, u8 family, u16 zone, bool *defrag) { enum ip_conntrack_info ctinfo; + struct tc_skb_cb cb; struct nf_conn *ct; int err = 0; bool frag; u8 proto; - u16 mru; /* Previously seen (loopback)? Ignore. */ ct = nf_ct_get(skb, &ctinfo); @@ -862,12 +862,13 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, if (err || !frag) return err; - err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &mru); + cb = *tc_skb_cb(skb); + err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &cb.mru); if (err) return err; *defrag = true; - tc_skb_cb(skb)->mru = mru; + *tc_skb_cb(skb) = cb; return 0; } @@ -1295,7 +1296,8 @@ static int tcf_ct_fill_params(struct net *net, if (tb[TCA_CT_ZONE]) { if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) { NL_SET_ERR_MSG_MOD(extack, "Conntrack zones isn't enabled."); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto err; } tcf_ct_set_key_val(tb, @@ -1308,7 +1310,8 @@ static int tcf_ct_fill_params(struct net *net, tmpl = nf_ct_tmpl_alloc(net, &zone, GFP_KERNEL); if (!tmpl) { NL_SET_ERR_MSG_MOD(extack, "Failed to allocate conntrack template"); - return -ENOMEM; + err = -ENOMEM; + goto err; } p->tmpl = tmpl; if (tb[TCA_CT_HELPER_NAME]) { diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index bd3b1da3cd63b..437a0563786b9 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -563,9 +563,18 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, { if (bind) { struct flow_action_entry *entry = entry_data; + int nkeys = tcf_pedit_nkeys_locked(act); int k; - for (k = 0; k < tcf_pedit_nkeys(act); k++) { + /* If the required keys exceed the remaining capacity return + * -ENOSPC to abort the offload and fallback to software. + */ + if (nkeys > *index_inc) { + NL_SET_ERR_MSG_MOD(extack, "Not enough space to offload all pedit keys"); + return -ENOSPC; + } + + for (k = 0; k < nkeys; k++) { switch (tcf_pedit_cmd(act, k)) { case TCA_PEDIT_KEY_EX_CMD_SET: entry->id = FLOW_ACTION_MANGLE; @@ -602,7 +611,7 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return -EOPNOTSUPP; } - for (k = 1; k < tcf_pedit_nkeys(act); k++) { + for (k = 1; k < tcf_pedit_nkeys_locked(act); k++) { if (cmd != tcf_pedit_cmd(act, k)) { NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); return -EOPNOTSUPP; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 20f7f9ee0b353..ffeea6db83374 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3886,12 +3886,21 @@ int tc_setup_action(struct flow_action *flow_action, entry = &flow_action->entries[j]; spin_lock_bh(&act->tcfa_lock); + + /* Abort the offload if we have exhausted the allocated capacity */ + if (j >= flow_action->num_entries) { + NL_SET_ERR_MSG_MOD(extack, "Flow action buffer overflow"); + err = -ENOSPC; + goto err_out_locked; + } + err = tcf_act_get_user_cookie(entry, act); if (err) goto err_out_locked; - index = 0; - err = tc_setup_offload_act(act, entry, &index, extack); + index = flow_action->num_entries - j; + err = tc_setup_offload_act(act, entry, &index, + extack); if (err) goto err_out_locked; @@ -3945,10 +3954,13 @@ unsigned int tcf_exts_num_actions(struct tcf_exts *exts) int i; tcf_exts_for_each_action(i, act, exts) { - if (is_tcf_pedit(act)) - num_acts += tcf_pedit_nkeys(act); - else + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + num_acts += tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + } else { num_acts++; + } } return num_acts; } @@ -4049,6 +4061,9 @@ struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, stru skb_do_redirect(skb); *ret = __NET_XMIT_STOLEN; return NULL; + case TC_ACT_CONSUMED: + *ret = __NET_XMIT_STOLEN; + return NULL; } return skb; diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 9a346b6221b35..001d8c4ebfedc 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -142,7 +142,8 @@ static bool cls_bpf_is_ebpf(const struct cls_bpf_prog *prog) static int cls_bpf_offload_cmd(struct tcf_proto *tp, struct cls_bpf_prog *prog, struct cls_bpf_prog *oldprog, - struct netlink_ext_ack *extack) + struct netlink_ext_ack *extack, + bool is_rollback) { struct tcf_block *block = tp->chain->block; struct tc_cls_bpf_offload cls_bpf = {}; @@ -177,7 +178,8 @@ static int cls_bpf_offload_cmd(struct tcf_proto *tp, struct cls_bpf_prog *prog, &oldprog->in_hw_count, true); if (prog && err) { - cls_bpf_offload_cmd(tp, oldprog, prog, extack); + if (!is_rollback) + cls_bpf_offload_cmd(tp, oldprog, prog, extack, true); return err; } @@ -208,7 +210,7 @@ static int cls_bpf_offload(struct tcf_proto *tp, struct cls_bpf_prog *prog, if (!prog && !oldprog) return 0; - return cls_bpf_offload_cmd(tp, prog, oldprog, extack); + return cls_bpf_offload_cmd(tp, prog, oldprog, extack, false); } static void cls_bpf_stop_offload(struct tcf_proto *tp, @@ -217,7 +219,7 @@ static void cls_bpf_stop_offload(struct tcf_proto *tp, { int err; - err = cls_bpf_offload_cmd(tp, NULL, prog, extack); + err = cls_bpf_offload_cmd(tp, NULL, prog, extack, false); if (err) pr_err("Stopping hardware offload failed: %d\n", err); } diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index ab364e4e46862..356c68ebc3895 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -57,11 +58,15 @@ struct flow_filter { struct rcu_work rwork; }; +static siphash_aligned_key_t flow_keys_secret __read_mostly; + static inline u32 addr_fold(void *addr) { - unsigned long a = (unsigned long)addr; - - return (a & 0xFFFFFFFF) ^ (BITS_PER_LONG > 32 ? a >> 32 : 0); +#ifdef CONFIG_64BIT + return (u32)siphash_1u64((u64)addr, &flow_keys_secret); +#else + return (u32)siphash_1u32((u32)addr, &flow_keys_secret); +#endif } static u32 flow_get_src(const struct sk_buff *skb, const struct flow_keys *flow) @@ -596,6 +601,7 @@ static int flow_init(struct tcf_proto *tp) return -ENOBUFS; INIT_LIST_HEAD(&head->filters); rcu_assign_pointer(tp->root, head); + net_get_random_once(&flow_keys_secret, sizeof(flow_keys_secret)); return 0; } diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index ed869a5ffc737..0dd3efd863938 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -805,7 +805,7 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) cl = cops->find(sch, parentid); cops->qlen_notify(sch, cl); } - sch->q.qlen -= n; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - n); sch->qstats.backlog -= len; __qdisc_qstats_drop(sch, drops); } diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index 5862933be8d74..07ea175657bb8 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1389,10 +1389,7 @@ static u32 cake_calc_overhead(struct cake_sched_data *qd, u32 len, u32 off) if (qd->min_netlen > len) WRITE_ONCE(qd->min_netlen, len); - len += q->rate_overhead; - - if (len < q->rate_mpu) - len = q->rate_mpu; + len = max((s32)len + q->rate_overhead, (s32)q->rate_mpu); if (q->atm_mode == CAKE_ATM_ATM) { len += 47; @@ -1612,7 +1609,7 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free) cake_advance_shaper(q, b, skb, now, true); qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT); - sch->q.qlen--; + qdisc_qlen_dec(sch); cake_heapify(q, 0); @@ -1822,7 +1819,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, segs); flow_queue_add(flow, segs); - sch->q.qlen++; + qdisc_qlen_inc(sch); numsegs++; slen += segs->len; q->buffer_used += segs->truesize; @@ -1861,7 +1858,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, qdisc_tree_reduce_backlog(sch, 1, ack_pkt_len); consume_skb(ack); } else { - sch->q.qlen++; + qdisc_qlen_inc(sch); q->buffer_used += skb->truesize; } @@ -1987,7 +1984,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch) WRITE_ONCE(b->tin_backlog, b->tin_backlog - len); sch->qstats.backlog -= len; q->buffer_used -= skb->truesize; - sch->q.qlen--; + qdisc_qlen_dec(sch); if (q->overflow_timeout) cake_heapify(q, b->overflow_idx[q->cur_flow]); diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c index 0f953bd46b581..0994da70816ec 100644 --- a/net/sched/sch_cbs.c +++ b/net/sched/sch_cbs.c @@ -97,7 +97,7 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch, return err; sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -168,7 +168,7 @@ static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child) qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c index 2875bcdb18a41..73d3e673dc7b1 100644 --- a/net/sched/sch_choke.c +++ b/net/sched/sch_choke.c @@ -123,7 +123,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx, if (idx == q->tail) choke_zap_tail_holes(q); - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb)); qdisc_drop(skb, sch, to_free); @@ -271,7 +271,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (sch->q.qlen < q->limit) { q->tab[q->tail] = skb; q->tail = (q->tail + 1) & q->tab_mask; - ++sch->q.qlen; + qdisc_qlen_inc(sch); qdisc_qstats_backlog_inc(sch, skb); return NET_XMIT_SUCCESS; } @@ -298,7 +298,7 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch) skb = q->tab[q->head]; q->tab[q->head] = NULL; choke_zap_head_holes(q); - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); @@ -396,7 +396,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt, } dropped += qdisc_pkt_len(skb); qdisc_qstats_backlog_dec(sch, skb); - --sch->q.qlen; + qdisc_qlen_dec(sch); rtnl_qdisc_drop(skb, sch); } qdisc_tree_reduce_backlog(sch, oqlen - sch->q.qlen, dropped); diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index 317aae0ec7bd6..7d5076196aff6 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -56,7 +56,7 @@ static void drop_func(struct sk_buff *skb, void *ctx) qdisc_qstats_drop(sch); } -static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch) +static struct sk_buff *__codel_qdisc_dequeue(struct Qdisc *sch) { struct codel_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; @@ -65,13 +65,51 @@ static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch) &q->stats, qdisc_pkt_len, codel_get_enqueue_time, drop_func, dequeue_func); + if (skb) + qdisc_bstats_update(sch, skb); + return skb; +} + +static void codel_dequeue_drop(struct Qdisc *sch) +{ + struct codel_sched_data *q = qdisc_priv(sch); + if (q->stats.drop_count) { - qdisc_tree_reduce_backlog(sch, q->stats.drop_count, q->stats.drop_len); + qdisc_tree_reduce_backlog(sch, q->stats.drop_count, + q->stats.drop_len); q->stats.drop_count = 0; q->stats.drop_len = 0; } - if (skb) - qdisc_bstats_update(sch, skb); +} + +static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch) +{ + struct sk_buff *skb; + + skb = __codel_qdisc_dequeue(sch); + + codel_dequeue_drop(sch); + + return skb; +} + +static struct sk_buff *codel_peek(struct Qdisc *sch) +{ + struct sk_buff *skb = skb_peek(&sch->gso_skb); + + if (!skb) { + skb = __codel_qdisc_dequeue(sch); + + if (skb) { + __skb_queue_head(&sch->gso_skb, skb); + /* it's still part of the queue */ + qdisc_qstats_backlog_inc(sch, skb); + sch->q.qlen++; + } + + codel_dequeue_drop(sch); + } + return skb; } @@ -257,7 +295,7 @@ static struct Qdisc_ops codel_qdisc_ops __read_mostly = { .enqueue = codel_qdisc_enqueue, .dequeue = codel_qdisc_dequeue, - .peek = qdisc_peek_dequeued, + .peek = codel_peek, .init = codel_init, .reset = codel_reset, .change = codel_change, diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 01335a49e0914..2eedd3a4322ad 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -270,7 +270,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg, memset(&xstats, 0, sizeof(xstats)); if (qlen) - xstats.deficit = cl->deficit; + xstats.deficit = READ_ONCE(cl->deficit); if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 || gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 || @@ -362,11 +362,11 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (!cl_is_active(cl)) { list_add_tail(&cl->alist, &q->active); - cl->deficit = cl->quantum; + WRITE_ONCE(cl->deficit, cl->quantum); } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return err; } @@ -389,7 +389,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch) len = qdisc_pkt_len(skb); if (len <= cl->deficit) { - cl->deficit -= len; + WRITE_ONCE(cl->deficit, cl->deficit - len); skb = qdisc_dequeue_peeked(cl->qdisc); if (unlikely(skb == NULL)) goto out; @@ -399,11 +399,11 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch) bstats_update(&cl->bstats, skb); qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } - cl->deficit += cl->quantum; + WRITE_ONCE(cl->deficit, cl->deficit + cl->quantum); list_move_tail(&cl->alist, &q->active); } out: diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index a22489c14458e..27088760eff44 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -190,7 +190,7 @@ static bool skb_apply_step(struct sk_buff *skb, struct dualpi2_sched_data *q) static bool dualpi2_mark(struct dualpi2_sched_data *q, struct sk_buff *skb) { if (INET_ECN_set_ce(skb)) { - q->ecn_mark++; + WRITE_ONCE(q->ecn_mark, q->ecn_mark + 1); return true; } return false; @@ -198,7 +198,7 @@ static bool dualpi2_mark(struct dualpi2_sched_data *q, struct sk_buff *skb) static void dualpi2_reset_c_protection(struct dualpi2_sched_data *q) { - q->c_protection_credit = q->c_protection_init; + WRITE_ONCE(q->c_protection_credit, q->c_protection_init); } /* This computes the initial credit value and WRR weight for the L queue (wl) @@ -346,6 +346,8 @@ static int dualpi2_skb_classify(struct dualpi2_sched_data *q, struct tcf_proto *fl; int result; + cb->classified = DUALPI2_C_CLASSIC; + dualpi2_read_ect(skb); if (cb->ect & q->ecn_mask) { cb->classified = DUALPI2_C_L4S; @@ -359,10 +361,8 @@ static int dualpi2_skb_classify(struct dualpi2_sched_data *q, } fl = rcu_dereference_bh(q->tcf_filters); - if (!fl) { - cb->classified = DUALPI2_C_CLASSIC; + if (!fl) return NET_XMIT_SUCCESS; - } result = tcf_classify(skb, NULL, fl, &res, false); if (result >= 0) { @@ -403,28 +403,28 @@ static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch, cb = dualpi2_skb_cb(skb); cb->ts = ktime_get_ns(); - q->memory_used += skb->truesize; + WRITE_ONCE(q->memory_used, q->memory_used + skb->truesize); if (q->memory_used > q->max_memory_used) - q->max_memory_used = q->memory_used; + WRITE_ONCE(q->max_memory_used, q->memory_used); if (qdisc_qlen(sch) > q->maxq) - q->maxq = qdisc_qlen(sch); + WRITE_ONCE(q->maxq, qdisc_qlen(sch)); if (skb_in_l_queue(skb)) { /* Apply step thresh if skb is L4S && L-queue len >= min_qlen */ dualpi2_skb_cb(skb)->apply_step = skb_apply_step(skb, q); /* Keep the overall qdisc stats consistent */ - ++sch->q.qlen; + qdisc_qlen_inc(sch); qdisc_qstats_backlog_inc(sch, skb); - ++q->packets_in_l; + WRITE_ONCE(q->packets_in_l, q->packets_in_l + 1); if (!q->l_head_ts) - q->l_head_ts = cb->ts; + WRITE_ONCE(q->l_head_ts, cb->ts); return qdisc_enqueue_tail(skb, q->l_queue); } - ++q->packets_in_c; + WRITE_ONCE(q->packets_in_c, q->packets_in_c + 1); if (!q->c_head_ts) - q->c_head_ts = cb->ts; + WRITE_ONCE(q->c_head_ts, cb->ts); return qdisc_enqueue_tail(skb, sch); } @@ -461,7 +461,7 @@ static int dualpi2_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (IS_ERR_OR_NULL(nskb)) return qdisc_drop(skb, sch, to_free); - cnt = 1; + cnt = 0; byte_len = 0; orig_len = qdisc_pkt_len(skb); skb_list_walk_safe(nskb, nskb, next) { @@ -488,16 +488,15 @@ static int dualpi2_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, byte_len += nskb->len; } } - if (cnt > 1) { + if (cnt > 0) { /* The caller will add the original skb stats to its * backlog, compensate this if any nskb is enqueued. */ - --cnt; - byte_len -= orig_len; + qdisc_tree_reduce_backlog(sch, 1 - cnt, + orig_len - byte_len); } - qdisc_tree_reduce_backlog(sch, -cnt, -byte_len); consume_skb(skb); - return err; + return cnt > 0 ? NET_XMIT_SUCCESS : err; } return dualpi2_enqueue_skb(skb, sch, to_free); } @@ -530,18 +529,17 @@ static struct sk_buff *dequeue_packet(struct Qdisc *sch, qdisc_qstats_backlog_dec(q->l_queue, skb); /* Keep the global queue size consistent */ - --sch->q.qlen; - q->memory_used -= skb->truesize; + qdisc_qlen_dec(sch); } else if (c_len) { skb = __qdisc_dequeue_head(&sch->q); WRITE_ONCE(q->c_head_ts, head_enqueue_time(sch)); if (qdisc_qlen(q->l_queue)) *credit_change = ~((s32)q->c_protection_wl) + 1; - q->memory_used -= skb->truesize; } else { dualpi2_reset_c_protection(q); return NULL; } + WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize); *credit_change *= qdisc_pkt_len(skb); qdisc_qstats_backlog_dec(sch, skb); return skb; @@ -564,7 +562,7 @@ static int do_step_aqm(struct dualpi2_sched_data *q, struct sk_buff *skb, } if (dualpi2_mark(q, skb)) - ++q->step_marks; + WRITE_ONCE(q->step_marks, q->step_marks + 1); } qdisc_bstats_update(q->l_queue, skb); return 0; @@ -579,7 +577,7 @@ static void drop_and_retry(struct dualpi2_sched_data *q, struct sk_buff *skb, qdisc_qstats_drop(sch); } -static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch) +static struct sk_buff *__dualpi2_qdisc_dequeue(struct Qdisc *sch) { struct dualpi2_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; @@ -600,17 +598,55 @@ static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch) continue; } - q->c_protection_credit += credit_change; + WRITE_ONCE(q->c_protection_credit, + q->c_protection_credit + credit_change); qdisc_bstats_update(sch, skb); break; } + return skb; +} + +static void dualpi2_dequeue_drop(struct Qdisc *sch) +{ + struct dualpi2_sched_data *q = qdisc_priv(sch); + if (q->deferred_drops_cnt) { qdisc_tree_reduce_backlog(sch, q->deferred_drops_cnt, q->deferred_drops_len); q->deferred_drops_cnt = 0; q->deferred_drops_len = 0; } +} + +static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch) +{ + struct sk_buff *skb; + + skb = __dualpi2_qdisc_dequeue(sch); + + dualpi2_dequeue_drop(sch); + + return skb; +} + +static struct sk_buff *dualpi2_peek(struct Qdisc *sch) +{ + struct sk_buff *skb = skb_peek(&sch->gso_skb); + + if (!skb) { + skb = __dualpi2_qdisc_dequeue(sch); + + if (skb) { + __skb_queue_head(&sch->gso_skb, skb); + /* it's still part of the queue */ + qdisc_qstats_backlog_inc(sch, skb); + sch->q.qlen++; + } + + dualpi2_dequeue_drop(sch); + } + return skb; } @@ -876,7 +912,7 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt, WARN_ON_ONCE(1); break; } - q->memory_used -= skb->truesize; + WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize); rtnl_qdisc_drop(skb, sch); } else if (qdisc_qlen(q->l_queue)) { skb = qdisc_dequeue_internal(q->l_queue, true); @@ -888,9 +924,9 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt, * l_queue on enqueue; qdisc_dequeue_internal() * handled l_queue, so we further account for sch. */ - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); - q->memory_used -= skb->truesize; + WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize); rtnl_qdisc_drop(skb, q->l_queue); qdisc_qstats_drop(sch); } else { @@ -1046,15 +1082,15 @@ static int dualpi2_dump_stats(struct Qdisc *sch, struct gnet_dump *d) struct dualpi2_sched_data *q = qdisc_priv(sch); struct tc_dualpi2_xstats st = { .prob = READ_ONCE(q->pi2_prob), - .packets_in_c = q->packets_in_c, - .packets_in_l = q->packets_in_l, - .maxq = q->maxq, - .ecn_mark = q->ecn_mark, - .credit = q->c_protection_credit, - .step_marks = q->step_marks, - .memory_used = q->memory_used, - .max_memory_used = q->max_memory_used, - .memory_limit = q->memory_limit, + .packets_in_c = READ_ONCE(q->packets_in_c), + .packets_in_l = READ_ONCE(q->packets_in_l), + .maxq = READ_ONCE(q->maxq), + .ecn_mark = READ_ONCE(q->ecn_mark), + .credit = READ_ONCE(q->c_protection_credit), + .step_marks = READ_ONCE(q->step_marks), + .memory_used = READ_ONCE(q->memory_used), + .max_memory_used = READ_ONCE(q->max_memory_used), + .memory_limit = READ_ONCE(q->memory_limit), }; u64 qc, ql; @@ -1074,16 +1110,16 @@ static void dualpi2_reset(struct Qdisc *sch) qdisc_reset_queue(sch); qdisc_reset_queue(q->l_queue); - q->c_head_ts = 0; - q->l_head_ts = 0; - q->pi2_prob = 0; - q->packets_in_c = 0; - q->packets_in_l = 0; - q->maxq = 0; - q->ecn_mark = 0; - q->step_marks = 0; - q->memory_used = 0; - q->max_memory_used = 0; + WRITE_ONCE(q->c_head_ts, 0); + WRITE_ONCE(q->l_head_ts, 0); + WRITE_ONCE(q->pi2_prob, 0); + WRITE_ONCE(q->packets_in_c, 0); + WRITE_ONCE(q->packets_in_l, 0); + WRITE_ONCE(q->maxq, 0); + WRITE_ONCE(q->ecn_mark, 0); + WRITE_ONCE(q->step_marks, 0); + WRITE_ONCE(q->memory_used, 0); + WRITE_ONCE(q->max_memory_used, 0); dualpi2_reset_c_protection(q); } @@ -1165,7 +1201,7 @@ static struct Qdisc_ops dualpi2_qdisc_ops __read_mostly = { .priv_size = sizeof(struct dualpi2_sched_data), .enqueue = dualpi2_qdisc_enqueue, .dequeue = dualpi2_qdisc_dequeue, - .peek = qdisc_peek_dequeued, + .peek = dualpi2_peek, .init = dualpi2_init, .destroy = dualpi2_destroy, .reset = dualpi2_reset, @@ -1174,6 +1210,7 @@ static struct Qdisc_ops dualpi2_qdisc_ops __read_mostly = { .dump_stats = dualpi2_dump_stats, .owner = THIS_MODULE, }; +MODULE_ALIAS_NET_SCH("dualpi2"); static int __init dualpi2_module_init(void) { diff --git a/net/sched/sch_etf.c b/net/sched/sch_etf.c index c74d778c32a1e..ada87a81da6ac 100644 --- a/net/sched/sch_etf.c +++ b/net/sched/sch_etf.c @@ -189,7 +189,7 @@ static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch, rb_insert_color_cached(&nskb->rbnode, &q->head, leftmost); qdisc_qstats_backlog_inc(sch, nskb); - sch->q.qlen++; + qdisc_qlen_inc(sch); /* Now we may need to re-arm the qdisc watchdog for the next packet. */ reset_watchdog(sch); @@ -222,7 +222,7 @@ static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb, qdisc_qstats_backlog_dec(sch, skb); qdisc_drop(skb, sch, &to_free); qdisc_qstats_overlimit(sch); - sch->q.qlen--; + qdisc_qlen_dec(sch); } kfree_skb_list(to_free); @@ -247,7 +247,7 @@ static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb) q->last = skb->tstamp; - sch->q.qlen--; + qdisc_qlen_dec(sch); } static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch) @@ -426,7 +426,7 @@ static void timesortedlist_clear(struct Qdisc *sch) rb_erase_cached(&skb->rbnode, &q->head); rtnl_kfree_skbs(skb, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index a4b07b661b775..c817e0a6c1465 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -449,7 +449,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return err; } @@ -458,7 +458,7 @@ ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index f2edcf872981f..1e34ac136b15c 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -497,7 +497,7 @@ static void fq_dequeue_skb(struct Qdisc *sch, struct fq_flow *flow, fq_erase_head(sch, flow, skb); skb_mark_not_on_list(skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); } @@ -597,7 +597,7 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch, flow_queue_add(f, skb); qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -801,7 +801,7 @@ static void fq_reset(struct Qdisc *sch) struct fq_flow *f; unsigned int idx; - sch->q.qlen = 0; + WRITE_ONCE(sch->q.qlen, 0); sch->qstats.backlog = 0; fq_flow_purge(&q->internal); diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 24db54684e8a5..09ab84580160a 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -178,7 +178,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets, q->memory_usage -= mem; sch->qstats.drops += i; sch->qstats.backlog -= len; - sch->q.qlen -= i; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - i); return idx; } @@ -215,7 +215,8 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch, get_codel_cb(skb)->mem_usage = skb->truesize; q->memory_usage += get_codel_cb(skb)->mem_usage; memory_limited = q->memory_usage > q->memory_limit; - if (++sch->q.qlen <= sch->limit && !memory_limited) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= sch->limit && !memory_limited) return NET_XMIT_SUCCESS; prev_backlog = sch->qstats.backlog; @@ -266,7 +267,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx) WRITE_ONCE(q->backlogs[flow - q->flows], q->backlogs[flow - q->flows] - qdisc_pkt_len(skb)); q->memory_usage -= get_codel_cb(skb)->mem_usage; - sch->q.qlen--; + qdisc_qlen_dec(sch); sch->qstats.backlog -= qdisc_pkt_len(skb); } return skb; @@ -280,7 +281,7 @@ static void drop_func(struct sk_buff *skb, void *ctx) qdisc_qstats_drop(sch); } -static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch) +static struct sk_buff *__fq_codel_dequeue(struct Qdisc *sch) { struct fq_codel_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; @@ -317,12 +318,49 @@ static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch) qdisc_bstats_update(sch, skb); WRITE_ONCE(flow->deficit, flow->deficit - qdisc_pkt_len(skb)); + return skb; +} + +static void fq_codel_dequeue_drop(struct Qdisc *sch) +{ + struct fq_codel_sched_data *q = qdisc_priv(sch); + if (q->cstats.drop_count) { qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len); q->cstats.drop_count = 0; q->cstats.drop_len = 0; } +} + +static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch) +{ + struct sk_buff *skb; + + skb = __fq_codel_dequeue(sch); + + fq_codel_dequeue_drop(sch); + + return skb; +} + +static struct sk_buff *fq_codel_peek(struct Qdisc *sch) +{ + struct sk_buff *skb = skb_peek(&sch->gso_skb); + + if (!skb) { + skb = __fq_codel_dequeue(sch); + + if (skb) { + __skb_queue_head(&sch->gso_skb, skb); + /* it's still part of the queue */ + qdisc_qstats_backlog_inc(sch, skb); + sch->q.qlen++; + } + + fq_codel_dequeue_drop(sch); + } + return skb; } @@ -725,7 +763,7 @@ static struct Qdisc_ops fq_codel_qdisc_ops __read_mostly = { .priv_size = sizeof(struct fq_codel_sched_data), .enqueue = fq_codel_enqueue, .dequeue = fq_codel_dequeue, - .peek = qdisc_peek_dequeued, + .peek = fq_codel_peek, .init = fq_codel_init, .reset = fq_codel_reset, .destroy = fq_codel_destroy, diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 7becbf5362b31..0a4eca4ab086e 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -185,7 +185,7 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, q->stats.packets_in++; q->memory_usage += skb->truesize; sch->qstats.backlog += pkt_len; - sch->q.qlen++; + qdisc_qlen_inc(sch); flow_queue_add(sel_flow, skb); if (list_empty(&sel_flow->flowchain)) { list_add_tail(&sel_flow->flowchain, &q->new_flows); @@ -263,7 +263,7 @@ static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch) skb = dequeue_head(flow); pkt_len = qdisc_pkt_len(skb); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index a93321db8fd75..fe73ac40d409e 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -118,7 +118,7 @@ static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q) qdisc_qstats_cpu_qlen_dec(q); } else { qdisc_qstats_backlog_dec(q, skb); - q->q.qlen--; + qdisc_qlen_dec(q); } } else { skb = SKB_XOFF_MAGIC; @@ -159,7 +159,7 @@ static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q, qdisc_qstats_cpu_qlen_inc(q); } else { qdisc_qstats_backlog_inc(q, skb); - q->q.qlen++; + qdisc_qlen_inc(q); } if (lock) @@ -188,7 +188,7 @@ static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) } else { q->qstats.requeues++; qdisc_qstats_backlog_inc(q, skb); - q->q.qlen++; + qdisc_qlen_inc(q); } skb = next; @@ -294,7 +294,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, qdisc_qstats_cpu_qlen_dec(q); } else { qdisc_qstats_backlog_dec(q, skb); - q->q.qlen--; + qdisc_qlen_dec(q); } } else { skb = NULL; @@ -568,16 +568,24 @@ static void dev_watchdog(struct timer_list *t) dev->netdev_ops->ndo_tx_timeout(dev, i); netif_unfreeze_queues(dev); } - if (!mod_timer(&dev->watchdog_timer, - round_jiffies(oldest_start + - dev->watchdog_timeo))) - release = false; + spin_lock(&dev->watchdog_lock); + mod_timer(&dev->watchdog_timer, + round_jiffies(oldest_start + + dev->watchdog_timeo)); + release = false; + spin_unlock(&dev->watchdog_lock); } } spin_unlock(&dev->tx_global_lock); - if (release) + spin_lock(&dev->watchdog_lock); + if (timer_pending(&dev->watchdog_timer)) + release = false; + if (release && dev->watchdog_ref_held) { netdev_put(dev, &dev->watchdog_dev_tracker); + dev->watchdog_ref_held = false; + } + spin_unlock(&dev->watchdog_lock); } void netdev_watchdog_up(struct net_device *dev) @@ -586,18 +594,31 @@ void netdev_watchdog_up(struct net_device *dev) return; if (dev->watchdog_timeo <= 0) dev->watchdog_timeo = 5*HZ; + + spin_lock_bh(&dev->watchdog_lock); if (!mod_timer(&dev->watchdog_timer, - round_jiffies(jiffies + dev->watchdog_timeo))) - netdev_hold(dev, &dev->watchdog_dev_tracker, - GFP_ATOMIC); + round_jiffies(jiffies + dev->watchdog_timeo))) { + if (!dev->watchdog_ref_held) { + netdev_hold(dev, &dev->watchdog_dev_tracker, + GFP_ATOMIC); + dev->watchdog_ref_held = true; + } + } + spin_unlock_bh(&dev->watchdog_lock); } EXPORT_SYMBOL_GPL(netdev_watchdog_up); static void netdev_watchdog_down(struct net_device *dev) { netif_tx_lock_bh(dev); - if (timer_delete(&dev->watchdog_timer)) + + spin_lock(&dev->watchdog_lock); + if (timer_delete(&dev->watchdog_timer)) { netdev_put(dev, &dev->watchdog_dev_tracker); + dev->watchdog_ref_held = false; + } + spin_unlock(&dev->watchdog_lock); + netif_tx_unlock_bh(dev); } @@ -614,8 +635,6 @@ void netif_carrier_on(struct net_device *dev) return; atomic_inc(&dev->carrier_up_count); linkwatch_fire_event(dev); - if (netif_running(dev)) - netdev_watchdog_up(dev); } } EXPORT_SYMBOL(netif_carrier_on); @@ -1059,7 +1078,7 @@ void qdisc_reset(struct Qdisc *qdisc) __skb_queue_purge(&qdisc->gso_skb); __skb_queue_purge(&qdisc->skb_bad_txq); - qdisc->q.qlen = 0; + WRITE_ONCE(qdisc->q.qlen, 0); qdisc->qstats.backlog = 0; } EXPORT_SYMBOL(qdisc_reset); diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 83b2ca2e37fc8..50112c8a71e03 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -715,7 +715,7 @@ init_vf(struct hfsc_class *cl, unsigned int len) rtsc_min(&cl->cl_virtual, &cl->cl_fsc, cl->cl_vt, cl->cl_total); cl->cl_vtadj = 0; - cl->cl_vtperiod++; /* increment vt period */ + WRITE_ONCE(cl->cl_vtperiod, cl->cl_vtperiod + 1); /* increment vt period */ cl->cl_parentperiod = cl->cl_parent->cl_vtperiod; if (cl->cl_parent->cl_nactive == 0) cl->cl_parentperiod++; @@ -753,11 +753,11 @@ update_vf(struct hfsc_class *cl, unsigned int len, u64 cur_time) u64 f; /* , myf_bound, delta; */ int go_passive = 0; - if (cl->qdisc->q.qlen == 0 && cl->cl_flags & HFSC_FSC) + if (cl->qdisc->q.qlen == 0 && cl->cl_flags & HFSC_FSC && cl->cl_nactive) go_passive = 1; for (; cl->cl_parent != NULL; cl = cl->cl_parent) { - cl->cl_total += len; + WRITE_ONCE(cl->cl_total, cl->cl_total + len); if (!(cl->cl_flags & HFSC_FSC) || cl->cl_nactive == 0) continue; @@ -847,7 +847,7 @@ hfsc_adjust_levels(struct hfsc_class *cl) if (p->level >= level) level = p->level + 1; } - cl->level = level; + WRITE_ONCE(cl->level, level); } while ((cl = cl->cl_parent) != NULL); } @@ -1338,10 +1338,10 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg, __u32 qlen; qdisc_qstats_qlen_backlog(cl->qdisc, &qlen, &cl->qstats.backlog); - xstats.level = cl->level; - xstats.period = cl->cl_vtperiod; - xstats.work = cl->cl_total; - xstats.rtwork = cl->cl_cumul; + xstats.level = READ_ONCE(cl->level); + xstats.period = READ_ONCE(cl->cl_vtperiod); + xstats.work = READ_ONCE(cl->cl_total); + xstats.rtwork = READ_ONCE(cl->cl_cumul); if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 || gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 || @@ -1452,15 +1452,15 @@ hfsc_change_qdisc(struct Qdisc *sch, struct nlattr *opt, static void hfsc_reset_class(struct hfsc_class *cl) { - cl->cl_total = 0; - cl->cl_cumul = 0; + WRITE_ONCE(cl->cl_total, 0); + WRITE_ONCE(cl->cl_cumul, 0); cl->cl_d = 0; cl->cl_e = 0; cl->cl_vt = 0; cl->cl_vtadj = 0; cl->cl_cvtmin = 0; cl->cl_cvtoff = 0; - cl->cl_vtperiod = 0; + WRITE_ONCE(cl->cl_vtperiod, 0); cl->cl_parentperiod = 0; cl->cl_f = 0; cl->cl_myf = 0; @@ -1561,7 +1561,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); if (first && !cl_in_el_or_vttree(cl)) { if (cl->cl_flags & HFSC_RSC) @@ -1626,7 +1626,7 @@ hfsc_dequeue(struct Qdisc *sch) bstats_update(&cl->bstats, skb); update_vf(cl, qdisc_pkt_len(skb), cur_time); if (realtime) - cl->cl_cumul += qdisc_pkt_len(skb); + WRITE_ONCE(cl->cl_cumul, cl->cl_cumul + qdisc_pkt_len(skb)); if (cl->cl_flags & HFSC_RSC) { if (cl->qdisc->q.qlen != 0) { @@ -1650,7 +1650,7 @@ hfsc_dequeue(struct Qdisc *sch) qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index 96021f52d835b..d85cb0263b675 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -360,7 +360,7 @@ static unsigned int hhf_drop(struct Qdisc *sch, struct sk_buff **to_free) if (bucket->head) { struct sk_buff *skb = dequeue_head(bucket); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_drop(skb, sch, to_free); } @@ -400,7 +400,8 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } bucket->deficit = weight * q->quantum; } - if (++sch->q.qlen <= sch->limit) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= sch->limit) return NET_XMIT_SUCCESS; prev_backlog = sch->qstats.backlog; @@ -443,7 +444,7 @@ static struct sk_buff *hhf_dequeue(struct Qdisc *sch) if (bucket->head) { skb = dequeue_head(bucket); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); } @@ -461,12 +462,39 @@ static struct sk_buff *hhf_dequeue(struct Qdisc *sch) return skb; } +static void hhf_reset_classifier(struct hhf_sched_data *q) +{ + int i; + + if (!q->hh_flows) + return; + + for (i = 0; i < HH_FLOWS_CNT; i++) { + struct hh_flow_state *flow, *next; + struct list_head *head = &q->hh_flows[i]; + + list_for_each_entry_safe(flow, next, head, flowchain) { + list_del(&flow->flowchain); + kfree(flow); + } + } + WRITE_ONCE(q->hh_flows_current_cnt, 0); + + for (i = 0; i < HHF_ARRAYS_CNT; i++) { + if (q->hhf_valid_bits[i]) + bitmap_zero(q->hhf_valid_bits[i], HHF_ARRAYS_LEN); + } + q->hhf_arrays_reset_timestamp = hhf_time_stamp(); +} + static void hhf_reset(struct Qdisc *sch) { + struct hhf_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; while ((skb = hhf_dequeue(sch)) != NULL) rtnl_kfree_skbs(skb, skb); + hhf_reset_classifier(q); } static void hhf_destroy(struct Qdisc *sch) diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index eb12381795ce1..ccabafed3fe44 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -568,7 +568,7 @@ htb_change_class_mode(struct htb_sched *q, struct htb_class *cl, s64 *diff) if (new_mode == HTB_CANT_SEND) { cl->overlimits++; - q->overlimits++; + WRITE_ONCE(q->overlimits, q->overlimits + 1); } if (cl->prio_activity) { /* not necessary: speed optimization */ @@ -628,7 +628,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, /* enqueue to helper queue */ if (q->direct_queue.qlen < q->direct_qlen) { __qdisc_enqueue_tail(skb, &q->direct_queue); - q->direct_pkts++; + WRITE_ONCE(q->direct_pkts, q->direct_pkts + 1); } else { return qdisc_drop(skb, sch, to_free); } @@ -651,7 +651,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -951,7 +951,7 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch) ok: qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } @@ -1147,6 +1147,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, * parts (especially calling ndo_setup_tc) on errors. */ q->offload = true; + sch->flags |= TCQ_F_OFFLOADED; return 0; } @@ -1207,17 +1208,12 @@ static int htb_dump(struct Qdisc *sch, struct sk_buff *skb) struct nlattr *nest; struct tc_htb_glob gopt; - if (q->offload) - sch->flags |= TCQ_F_OFFLOADED; - else - sch->flags &= ~TCQ_F_OFFLOADED; - - sch->qstats.overlimits = q->overlimits; + sch->qstats.overlimits = READ_ONCE(q->overlimits); /* Its safe to not acquire qdisc lock. As we hold RTNL, * no change can happen on the qdisc parameters. */ - gopt.direct_pkts = q->direct_pkts; + gopt.direct_pkts = READ_ONCE(q->direct_pkts); gopt.version = HTB_VER; gopt.rate2quantum = q->rate2quantum; gopt.defcls = q->defcls; diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c index a0133a7b9d3b0..ec8c91d3fde04 100644 --- a/net/sched/sch_mq.c +++ b/net/sched/sch_mq.c @@ -143,10 +143,10 @@ EXPORT_SYMBOL_NS_GPL(mq_attach, "NET_SCHED_INTERNAL"); void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) { struct net_device *dev = qdisc_dev(sch); + unsigned int qlen = 0; struct Qdisc *qdisc; unsigned int ntx; - sch->q.qlen = 0; gnet_stats_basic_sync_init(&sch->bstats); memset(&sch->qstats, 0, sizeof(sch->qstats)); @@ -163,10 +163,11 @@ void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false); gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } + WRITE_ONCE(sch->q.qlen, qlen); } EXPORT_SYMBOL_NS_GPL(mq_dump_common, "NET_SCHED_INTERNAL"); diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c index 002add5ce9e0a..91a92992cd24a 100644 --- a/net/sched/sch_mqprio.c +++ b/net/sched/sch_mqprio.c @@ -555,10 +555,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) struct mqprio_sched *priv = qdisc_priv(sch); struct nlattr *nla = (struct nlattr *)skb_tail_pointer(skb); struct tc_mqprio_qopt opt = { 0 }; + unsigned int qlen = 0; struct Qdisc *qdisc; unsigned int ntx; - sch->q.qlen = 0; + qlen = 0; gnet_stats_basic_sync_init(&sch->bstats); memset(&sch->qstats, 0, sizeof(sch->qstats)); @@ -575,10 +576,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false); gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } + WRITE_ONCE(sch->q.qlen, qlen); mqprio_qopt_reconstruct(dev, &opt); opt.hw = priv->hw_offload; @@ -663,12 +665,12 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, __acquires(d->lock) { if (cl >= TC_H_MIN_PRIORITY) { - int i; - __u32 qlen; - struct gnet_stats_queue qstats = {0}; - struct gnet_stats_basic_sync bstats; struct net_device *dev = qdisc_dev(sch); struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK]; + struct gnet_stats_queue qstats = {0}; + struct gnet_stats_basic_sync bstats; + u32 qlen = 0; + int i; gnet_stats_basic_sync_init(&bstats); /* Drop lock here it will be reclaimed before touching @@ -689,11 +691,11 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, &qdisc->bstats, false); gnet_stats_add_queue(&qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } - qlen = qdisc_qlen(sch) + qstats.qlen; + qlen = qlen + qstats.qlen; /* Reclaim root sleeping lock before completing stats */ if (d->lock) diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c index 9f822fee113df..a467dd1223697 100644 --- a/net/sched/sch_multiq.c +++ b/net/sched/sch_multiq.c @@ -76,7 +76,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret = qdisc_enqueue(skb, qdisc, to_free); if (ret == NET_XMIT_SUCCESS) { - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } if (net_xmit_drop_count(ret)) @@ -103,10 +103,10 @@ static struct sk_buff *multiq_dequeue(struct Qdisc *sch) if (!netif_xmit_stopped( netdev_get_tx_queue(qdisc_dev(sch), q->curband))) { qdisc = q->queues[q->curband]; - skb = qdisc->dequeue(qdisc); + skb = qdisc_dequeue_peeked(qdisc); if (skb) { qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 17a79fe2f0911..db44cdb47dcef 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -416,7 +416,7 @@ static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch) rb_insert_color(&nskb->rbnode, &q->t_root); } q->t_len++; - sch->q.qlen++; + qdisc_qlen_inc(sch); } /* netem can't properly corrupt a megapacket (like we get from GSO), so instead @@ -750,19 +750,19 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch) if (net_xmit_drop_count(err)) qdisc_qstats_drop(sch); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_tree_reduce_backlog(sch, 1, pkt_len); } goto tfifo_dequeue; } - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } if (q->qdisc) { skb = q->qdisc->ops->dequeue(q->qdisc); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } } @@ -775,7 +775,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch) if (q->qdisc) { skb = q->qdisc->ops->dequeue(q->qdisc); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } } diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 9e2b9a490db23..fe42ae3d6b696 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -86,7 +86,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) ret = qdisc_enqueue(skb, qdisc, to_free); if (ret == NET_XMIT_SUCCESS) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } if (net_xmit_drop_count(ret)) @@ -119,7 +119,7 @@ static struct sk_buff *prio_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } } diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index 699e45873f861..195c434aae5f7 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -1152,12 +1152,12 @@ static struct sk_buff *qfq_dequeue(struct Qdisc *sch) if (!skb) return NULL; - sch->q.qlen--; + qdisc_qlen_dec(sch); skb = agg_dequeue(in_serv_agg, cl, len); if (!skb) { - sch->q.qlen++; + qdisc_qlen_inc(sch); return NULL; } @@ -1265,7 +1265,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, _bstats_update(&cl->bstats, len, gso_segs); sch->qstats.backlog += len; - ++sch->q.qlen; + qdisc_qlen_inc(sch); agg = cl->agg; /* if the class is active, then done here */ diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 4d0e44a2e7c66..0719590dfd73b 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -139,7 +139,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); } else if (net_xmit_drop_count(ret)) { WRITE_ONCE(q->stats.pdrop, q->stats.pdrop + 1); @@ -166,7 +166,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } else { if (!red_is_idling(&q->vars)) red_start_of_idle_period(&q->vars); diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c index d3ee8e5479b35..efd9251c3add3 100644 --- a/net/sched/sch_sfb.c +++ b/net/sched/sch_sfb.c @@ -416,7 +416,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); increment_qlen(&cb, q); } else if (net_xmit_drop_count(ret)) { WRITE_ONCE(q->stats.childdrop, @@ -446,7 +446,7 @@ static struct sk_buff *sfb_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); decrement_qlen(skb, q); } diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index f39822babf88b..f9807ee2cf6c7 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -302,7 +302,7 @@ static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free) len = qdisc_pkt_len(skb); WRITE_ONCE(slot->backlog, slot->backlog - len); sfq_dec(q, x); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT); return len; @@ -456,7 +456,8 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) /* We could use a bigger initial quantum for new flows */ WRITE_ONCE(slot->allot, q->quantum); } - if (++sch->q.qlen <= q->limit) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= q->limit) return NET_XMIT_SUCCESS; qlen = slot->qlen; @@ -497,7 +498,7 @@ sfq_dequeue(struct Qdisc *sch) skb = slot_dequeue_head(slot); sfq_dec(q, a); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); WRITE_ONCE(slot->backlog, slot->backlog - qdisc_pkt_len(skb)); /* Is the slot empty? */ @@ -596,7 +597,7 @@ static void sfq_rehash(struct Qdisc *sch) WRITE_ONCE(slot->allot, q->quantum); } } - sch->q.qlen -= dropped; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - dropped); qdisc_tree_reduce_backlog(sch, dropped, drop_len); } diff --git a/net/sched/sch_skbprio.c b/net/sched/sch_skbprio.c index f485f62ab721a..52abfb4015a36 100644 --- a/net/sched/sch_skbprio.c +++ b/net/sched/sch_skbprio.c @@ -93,7 +93,7 @@ static int skbprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (prio < q->lowest_prio) q->lowest_prio = prio; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -145,7 +145,7 @@ static struct sk_buff *skbprio_dequeue(struct Qdisc *sch) if (unlikely(!skb)) return NULL; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 45245157e00a6..d2d11a127f039 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -574,7 +574,7 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, } qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); return qdisc_enqueue(skb, child, to_free); } @@ -749,13 +749,13 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq, return NULL; skip_peek_checks: - skb = child->ops->dequeue(child); + skb = qdisc_dequeue_peeked(child); if (unlikely(!skb)) return NULL; qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c index f2340164f579a..25edf11a7d671 100644 --- a/net/sched/sch_tbf.c +++ b/net/sched/sch_tbf.c @@ -231,7 +231,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch, len += seg_len; } } - sch->q.qlen += nb; + WRITE_ONCE(sch->q.qlen, sch->q.qlen + nb); sch->qstats.backlog += len; if (nb > 0) { qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len); @@ -264,7 +264,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -309,7 +309,7 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch) q->tokens = toks; q->ptokens = ptoks; qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); return skb; } diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index ec4039a201a2c..5c42a29a981cb 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -52,7 +52,8 @@ struct teql_master { struct Qdisc_ops qops; struct net_device *dev; - struct Qdisc *slaves; + struct Qdisc __rcu *slaves; + spinlock_t slaves_lock; /* serializes writes to ->slaves */ struct list_head master_list; unsigned long tx_bytes; unsigned long tx_packets; @@ -61,7 +62,7 @@ struct teql_master { }; struct teql_sched_data { - struct Qdisc *next; + struct Qdisc __rcu *next; struct teql_master *m; struct sk_buff_head q; }; @@ -101,13 +102,15 @@ teql_dequeue(struct Qdisc *sch) if (skb == NULL) { struct net_device *m = qdisc_dev(q); if (m) { - dat->m->slaves = sch; + spin_lock_bh(&dat->m->slaves_lock); + rcu_assign_pointer(dat->m->slaves, sch); + spin_unlock_bh(&dat->m->slaves_lock); netif_wake_queue(m); } } else { qdisc_bstats_update(sch, skb); } - sch->q.qlen = dat->q.qlen + q->q.qlen; + WRITE_ONCE(sch->q.qlen, dat->q.qlen + READ_ONCE(q->q.qlen)); return skb; } @@ -132,34 +135,49 @@ teql_destroy(struct Qdisc *sch) struct Qdisc *q, *prev; struct teql_sched_data *dat = qdisc_priv(sch); struct teql_master *master = dat->m; + struct netdev_queue *txq = NULL; + bool reset_master_queue = false; if (!master) return; - prev = master->slaves; + spin_lock_bh(&master->slaves_lock); + prev = rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)); if (prev) { do { - q = NEXT_SLAVE(prev); - if (q == sch) { - NEXT_SLAVE(prev) = NEXT_SLAVE(q); - if (q == master->slaves) { - master->slaves = NEXT_SLAVE(q); - if (q == master->slaves) { - struct netdev_queue *txq; - - txq = netdev_get_tx_queue(master->dev, 0); - master->slaves = NULL; - - dev_reset_queue(master->dev, - txq, NULL); - } - } - skb_queue_purge(&dat->q); - break; + struct Qdisc *head, *next; + + q = rcu_dereference_protected(NEXT_SLAVE(prev), + lockdep_is_held(&master->slaves_lock)); + if (q != sch) { + prev = q; + continue; } - } while ((prev = q) != master->slaves); + next = rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock)); + rcu_assign_pointer(NEXT_SLAVE(prev), next); + + head = rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)); + if (q == head) { + rcu_assign_pointer(master->slaves, next); + if (q == next) { + txq = netdev_get_tx_queue(master->dev, 0); + rcu_assign_pointer(master->slaves, NULL); + reset_master_queue = true; + } + } + skb_queue_purge(&dat->q); + break; + } while (prev != rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock))); } + spin_unlock_bh(&master->slaves_lock); + + if (reset_master_queue) + dev_reset_queue(master->dev, txq, NULL); } static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, @@ -168,6 +186,7 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, struct net_device *dev = qdisc_dev(sch); struct teql_master *m = (struct teql_master *)sch->ops; struct teql_sched_data *q = qdisc_priv(sch); + struct Qdisc *first; if (dev->hard_header_len > m->dev->hard_header_len) return -EINVAL; @@ -184,7 +203,9 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, skb_queue_head_init(&q->q); - if (m->slaves) { + spin_lock_bh(&m->slaves_lock); + first = rcu_dereference_protected(m->slaves, lockdep_is_held(&m->slaves_lock)); + if (first) { if (m->dev->flags & IFF_UP) { if ((m->dev->flags & IFF_POINTOPOINT && !(dev->flags & IFF_POINTOPOINT)) || @@ -192,8 +213,10 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, !(dev->flags & IFF_BROADCAST)) || (m->dev->flags & IFF_MULTICAST && !(dev->flags & IFF_MULTICAST)) || - dev->mtu < m->dev->mtu) + dev->mtu < m->dev->mtu) { + spin_unlock_bh(&m->slaves_lock); return -EINVAL; + } } else { if (!(dev->flags&IFF_POINTOPOINT)) m->dev->flags &= ~IFF_POINTOPOINT; @@ -204,14 +227,17 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, if (dev->mtu < m->dev->mtu) m->dev->mtu = dev->mtu; } - q->next = NEXT_SLAVE(m->slaves); - NEXT_SLAVE(m->slaves) = sch; + rcu_assign_pointer(q->next, + rcu_dereference_protected(NEXT_SLAVE(first), + lockdep_is_held(&m->slaves_lock))); + rcu_assign_pointer(NEXT_SLAVE(first), sch); } else { - q->next = sch; - m->slaves = sch; + rcu_assign_pointer(q->next, sch); + rcu_assign_pointer(m->slaves, sch); m->dev->mtu = dev->mtu; m->dev->flags = (m->dev->flags&~FMASK)|(dev->flags&FMASK); } + spin_unlock_bh(&m->slaves_lock); return 0; } @@ -285,12 +311,14 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) int subq = skb_get_queue_mapping(skb); struct sk_buff *skb_res = NULL; - start = master->slaves; - restart: nores = 0; busy = 0; + rcu_read_lock(); + + start = rcu_dereference(master->slaves); + q = start; if (!q) goto drop; @@ -317,10 +345,17 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) netdev_start_xmit(skb, slave, slave_txq, false) == NETDEV_TX_OK) { __netif_tx_unlock(slave_txq); - master->slaves = NEXT_SLAVE(q); + spin_lock(&master->slaves_lock); + if (rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)) == q) + rcu_assign_pointer(master->slaves, + rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock))); + spin_unlock(&master->slaves_lock); netif_wake_queue(dev); master->tx_packets++; master->tx_bytes += length; + rcu_read_unlock(); return NETDEV_TX_OK; } __netif_tx_unlock(slave_txq); @@ -329,45 +364,56 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) busy = 1; break; case 1: - master->slaves = NEXT_SLAVE(q); + spin_lock(&master->slaves_lock); + if (rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)) == q) + rcu_assign_pointer(master->slaves, + rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock))); + spin_unlock(&master->slaves_lock); + rcu_read_unlock(); return NETDEV_TX_OK; default: nores = 1; break; } __skb_pull(skb, skb_network_offset(skb)); - } while ((q = NEXT_SLAVE(q)) != start); + } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); if (nores && skb_res == NULL) { skb_res = skb; + rcu_read_unlock(); goto restart; } if (busy) { netif_stop_queue(dev); + rcu_read_unlock(); return NETDEV_TX_BUSY; } master->tx_errors++; drop: master->tx_dropped++; + rcu_read_unlock(); dev_kfree_skb(skb); return NETDEV_TX_OK; } static int teql_master_open(struct net_device *dev) { - struct Qdisc *q; + struct Qdisc *q, *first; struct teql_master *m = netdev_priv(dev); int mtu = 0xFFFE; unsigned int flags = IFF_NOARP | IFF_MULTICAST; - if (m->slaves == NULL) + first = rtnl_dereference(m->slaves); + if (!first) return -EUNATCH; flags = FMASK; - q = m->slaves; + q = first; do { struct net_device *slave = qdisc_dev(q); @@ -389,7 +435,7 @@ static int teql_master_open(struct net_device *dev) flags &= ~IFF_BROADCAST; if (!(slave->flags&IFF_MULTICAST)) flags &= ~IFF_MULTICAST; - } while ((q = NEXT_SLAVE(q)) != m->slaves); + } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); m->dev->mtu = mtu; m->dev->flags = (m->dev->flags&~FMASK) | flags; @@ -417,14 +463,15 @@ static void teql_master_stats64(struct net_device *dev, static int teql_master_mtu(struct net_device *dev, int new_mtu) { struct teql_master *m = netdev_priv(dev); - struct Qdisc *q; + struct Qdisc *q, *first; - q = m->slaves; + first = rtnl_dereference(m->slaves); + q = first; if (q) { do { if (new_mtu > qdisc_dev(q)->mtu) return -EINVAL; - } while ((q = NEXT_SLAVE(q)) != m->slaves); + } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); } WRITE_ONCE(dev->mtu, new_mtu); @@ -444,6 +491,7 @@ static __init void teql_master_setup(struct net_device *dev) struct teql_master *master = netdev_priv(dev); struct Qdisc_ops *ops = &master->qops; + spin_lock_init(&master->slaves_lock); master->dev = dev; ops->priv_size = sizeof(struct teql_sched_data); diff --git a/net/sctp/diag.c b/net/sctp/diag.c index d758f5c3e06e5..c2a0de2adf6fd 100644 --- a/net/sctp/diag.c +++ b/net/sctp/diag.c @@ -92,6 +92,7 @@ static int inet_diag_msg_sctpladdrs_fill(struct sk_buff *skb, if (!--addrcnt) break; } + WARN_ON_ONCE(addrcnt); rcu_read_unlock(); return 0; @@ -373,42 +374,39 @@ static int sctp_ep_dump(struct sctp_endpoint *ep, void *p) struct sk_buff *skb = commp->skb; struct netlink_callback *cb = commp->cb; const struct inet_diag_req_v2 *r = commp->r; - struct net *net = sock_net(skb->sk); struct inet_sock *inet = inet_sk(sk); int err = 0; - if (!net_eq(sock_net(sk), net)) + lock_sock(sk); + if (ep->base.dead) goto out; - if (cb->args[4] < cb->args[1]) - goto next; - - if (!(r->idiag_states & TCPF_LISTEN) && !list_empty(&ep->asocs)) - goto next; + /* Skip eps with assocs if non-LISTEN states were requested, since + * they'll be dumped by sctp_sock_dump() during assoc traversal. + */ + if ((r->idiag_states & ~(TCPF_LISTEN | TCPF_CLOSE)) && + !list_empty(&ep->asocs)) + goto out; if (r->sdiag_family != AF_UNSPEC && sk->sk_family != r->sdiag_family) - goto next; + goto out; if (r->id.idiag_sport != inet->inet_sport && r->id.idiag_sport) - goto next; + goto out; if (r->id.idiag_dport != inet->inet_dport && r->id.idiag_dport) - goto next; - - if (inet_sctp_diag_fill(sk, NULL, skb, r, - sk_user_ns(NETLINK_CB(cb->skb).sk), - NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, NLM_F_MULTI, - cb->nlh, commp->net_admin) < 0) { - err = 2; goto out; - } -next: - cb->args[4]++; + + err = inet_sctp_diag_fill(sk, NULL, skb, r, + sk_user_ns(NETLINK_CB(cb->skb).sk), + NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, NLM_F_MULTI, + cb->nlh, commp->net_admin); out: + release_sock(sk); return err; } @@ -479,41 +477,40 @@ static void sctp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb, .r = r, .net_admin = netlink_net_capable(cb->skb, CAP_NET_ADMIN), }; - int pos = cb->args[2]; + int pos; /* eps hashtable dumps * args: * 0 : if it will traversal listen sock * 1 : to record the sock pos of this time's traversal - * 4 : to work as a temporary variable to traversal list */ if (cb->args[0] == 0) { - if (!(idiag_states & TCPF_LISTEN)) - goto skip; - if (sctp_for_each_endpoint(sctp_ep_dump, &commp)) - goto done; -skip: + if (idiag_states & TCPF_LISTEN) { + pos = cb->args[1]; + if (sctp_for_each_endpoint(sctp_ep_dump, net, &pos, + &commp)) { + cb->args[1] = pos; + return; + } + } cb->args[0] = 1; cb->args[1] = 0; - cb->args[4] = 0; } + if (!(idiag_states & ~(TCPF_LISTEN | TCPF_CLOSE))) + return; + /* asocs by transport hashtable dump * args: * 1 : to record the assoc pos of this time's traversal * 2 : to record the transport pos of this time's traversal * 3 : to mark if we have dumped the ep info of the current asoc - * 4 : to work as a temporary variable to traversal list - * 5 : to save the sk we get from travelsing the tsp list. + * 4 : to track position within ep->asocs list in sctp_sock_dump() */ - if (!(idiag_states & ~(TCPF_LISTEN | TCPF_CLOSE))) - goto done; - + pos = cb->args[2]; sctp_transport_traverse_process(sctp_sock_filter, sctp_sock_dump, net, &pos, &commp); cb->args[2] = pos; - -done: cb->args[1] = cb->args[4]; cb->args[4] = 0; } diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index 5800e7ee7ea02..e95061b672ef4 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -663,8 +663,9 @@ static void sctp_free_addr_wq(struct net *net) struct sctp_sockaddr_entry *addrw; struct sctp_sockaddr_entry *temp; + timer_shutdown_sync(&net->sctp.addr_wq_timer); + spin_lock_bh(&net->sctp.addr_wq_lock); - timer_delete(&net->sctp.addr_wq_timer); list_for_each_entry_safe(addrw, temp, &net->sctp.addr_waitq, list) { list_del(&addrw->list); kfree(addrw); @@ -864,7 +865,7 @@ int sctp_udp_sock_start(struct net *net) tuncfg.encap_type = 1; tuncfg.encap_rcv = sctp_udp_rcv; tuncfg.encap_err_lookup = sctp_udp_v4_err; - setup_udp_tunnel_sock(net, sock, &tuncfg); + setup_udp_tunnel_sock(net, sock->sk, &tuncfg); net->sctp.udp4_sock = sock->sk; #if IS_ENABLED(CONFIG_IPV6) @@ -878,7 +879,7 @@ int sctp_udp_sock_start(struct net *net) err = udp_sock_create(net, &udp_conf, &sock); if (err) { pr_err("Failed to create the SCTP UDP tunneling v6 sock\n"); - udp_tunnel_sock_release(net->sctp.udp4_sock->sk_socket); + udp_tunnel_sock_release(net->sctp.udp4_sock); net->sctp.udp4_sock = NULL; return err; } @@ -886,7 +887,7 @@ int sctp_udp_sock_start(struct net *net) tuncfg.encap_type = 1; tuncfg.encap_rcv = sctp_udp_rcv; tuncfg.encap_err_lookup = sctp_udp_v6_err; - setup_udp_tunnel_sock(net, sock, &tuncfg); + setup_udp_tunnel_sock(net, sock->sk, &tuncfg); net->sctp.udp6_sock = sock->sk; #endif @@ -896,11 +897,11 @@ int sctp_udp_sock_start(struct net *net) void sctp_udp_sock_stop(struct net *net) { if (net->sctp.udp4_sock) { - udp_tunnel_sock_release(net->sctp.udp4_sock->sk_socket); + udp_tunnel_sock_release(net->sctp.udp4_sock); net->sctp.udp4_sock = NULL; } if (net->sctp.udp6_sock) { - udp_tunnel_sock_release(net->sctp.udp6_sock->sk_socket); + udp_tunnel_sock_release(net->sctp.udp6_sock); net->sctp.udp6_sock = NULL; } } diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index 1741a9f33d8c7..8adac9e0cd66b 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -1761,6 +1761,8 @@ struct sctp_association *sctp_unpack_cookie( bear_cookie = &cookie->c; ch = (struct sctp_chunkhdr *)(bear_cookie + 1); + if (ch->type != SCTP_CID_INIT) + goto malformed; chlen = ntohs(ch->length); if (chlen < sizeof(struct sctp_init_chunk)) goto malformed; @@ -2298,7 +2300,8 @@ int sctp_verify_init(struct net *net, const struct sctp_endpoint *ep, * VIOLATION error. We build the ERROR chunk here and let the normal * error handling code build and send the packet. */ - if (param.v != (void *)chunk->chunk_end) + if (param.v != (void *)peer_init + + SCTP_PAD4(ntohs(peer_init->chunk_hdr.length))) return sctp_process_inv_paramlength(asoc, param.p, chunk, errp); /* The only missing mandatory param possible today is @@ -2641,6 +2644,9 @@ static int sctp_process_param(struct sctp_association *asoc, goto fall_through; addr_param = param.v + sizeof(struct sctp_addip_param); + if (ntohs(addr_param->p.length) > + ntohs(param.p->length) - sizeof(struct sctp_addip_param)) + break; af = sctp_get_af_specific(param_type2af(addr_param->p.type)); if (!af) @@ -3039,13 +3045,16 @@ static __be16 sctp_process_asconf_param(struct sctp_association *asoc, union sctp_addr addr; struct sctp_af *af; - addr_param = (void *)asconf_param + sizeof(*asconf_param); - if (asconf_param->param_hdr.type != SCTP_PARAM_ADD_IP && asconf_param->param_hdr.type != SCTP_PARAM_DEL_IP && asconf_param->param_hdr.type != SCTP_PARAM_SET_PRIMARY) return SCTP_ERROR_UNKNOWN_PARAM; + addr_param = (void *)asconf_param + sizeof(*asconf_param); + if (ntohs(addr_param->p.length) > + ntohs(asconf_param->param_hdr.length) - sizeof(*asconf_param)) + return SCTP_ERROR_PROTO_VIOLATION; + switch (addr_param->p.type) { case SCTP_PARAM_IPV6_ADDRESS: if (!asoc->peer.ipv6_address) diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 9b23c11cbb9ea..3893b44448b38 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -74,7 +74,8 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( const struct sctp_association *asoc, const union sctp_subtype type, void *arg, - struct sctp_cmd_seq *commands); + struct sctp_cmd_seq *commands, + struct sctp_errhdr *err); static enum sctp_disposition sctp_sf_shut_8_4_5( struct net *net, const struct sctp_endpoint *ep, @@ -415,6 +416,8 @@ enum sctp_disposition sctp_sf_do_5_1B_init(struct net *net, /* Update socket peer label if first association. */ if (security_sctp_assoc_request(new_asoc, chunk->skb)) { sctp_association_free(new_asoc); + if (err_chunk) + sctp_chunk_free(err_chunk); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); } @@ -705,11 +708,12 @@ enum sctp_disposition sctp_sf_do_5_1D_ce(struct net *net, struct sctp_cmd_seq *commands) { struct sctp_ulpevent *ev, *ai_ev = NULL, *auth_ev = NULL; + struct sctp_chunk *err_chk_p = NULL; struct sctp_association *new_asoc; struct sctp_init_chunk *peer_init; struct sctp_chunk *chunk = arg; - struct sctp_chunk *err_chk_p; struct sctp_chunk *repl; + enum sctp_cid cid; struct sock *sk; int error = 0; @@ -783,6 +787,19 @@ enum sctp_disposition sctp_sf_do_5_1D_ce(struct net *net, } } + peer_init = (struct sctp_init_chunk *)(chunk->subh.cookie_hdr + 1); + cid = peer_init->chunk_hdr.type; + if (!sctp_sk(sk)->cookie_auth_enable && + !sctp_verify_init(net, ep, asoc, cid, peer_init, chunk, + &err_chk_p)) { + sctp_association_free(new_asoc); + if (err_chk_p) + sctp_chunk_free(err_chk_p); + return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); + } + if (err_chk_p) + sctp_chunk_free(err_chk_p); + if (security_sctp_assoc_request(new_asoc, chunk->head_skb ?: chunk->skb)) { sctp_association_free(new_asoc); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); @@ -796,7 +813,6 @@ enum sctp_disposition sctp_sf_do_5_1D_ce(struct net *net, /* This is a brand-new association, so these are not yet side * effects--it is safe to run them here. */ - peer_init = (struct sctp_init_chunk *)(chunk->subh.cookie_hdr + 1); if (!sctp_process_init(new_asoc, chunk, &chunk->subh.cookie_hdr->c.peer_addr, peer_init, GFP_ATOMIC)) @@ -1606,6 +1622,8 @@ static enum sctp_disposition sctp_sf_do_unexpected_init( /* Update socket peer label if first association. */ if (security_sctp_assoc_request(new_asoc, chunk->skb)) { sctp_association_free(new_asoc); + if (err_chunk) + sctp_chunk_free(err_chunk); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); } @@ -1671,6 +1689,7 @@ static enum sctp_disposition sctp_sf_do_unexpected_init( * parameter type. */ sctp_addto_chunk(repl, len, unk_param); + sctp_chunk_free(err_chunk); } sctp_add_cmd_sf(commands, SCTP_CMD_NEW_ASOC, SCTP_ASOC(new_asoc)); @@ -2210,10 +2229,12 @@ enum sctp_disposition sctp_sf_do_5_2_4_dupcook( void *arg, struct sctp_cmd_seq *commands) { + struct sctp_chunk *err_chk_p = NULL; struct sctp_association *new_asoc; + struct sctp_init_chunk *peer_init; struct sctp_chunk *chunk = arg; enum sctp_disposition retval; - struct sctp_chunk *err_chk_p; + enum sctp_cid cid; int error = 0; char action; @@ -2282,6 +2303,21 @@ enum sctp_disposition sctp_sf_do_5_2_4_dupcook( switch (action) { case 'A': /* Association restart. */ case 'B': /* Collision case B. */ + peer_init = (struct sctp_init_chunk *) + (chunk->subh.cookie_hdr + 1); + cid = peer_init->chunk_hdr.type; + if (!sctp_sk(ep->base.sk)->cookie_auth_enable && + !sctp_verify_init(net, ep, asoc, cid, peer_init, chunk, + &err_chk_p)) { + sctp_association_free(new_asoc); + if (err_chk_p) + sctp_chunk_free(err_chk_p); + return sctp_sf_pdiscard(net, ep, asoc, type, arg, + commands); + } + if (err_chk_p) + sctp_chunk_free(err_chk_p); + fallthrough; case 'D': /* Collision case D. */ /* Update socket peer label if first association. */ if (security_sctp_assoc_request((struct sctp_association *)asoc, @@ -2494,9 +2530,15 @@ enum sctp_disposition sctp_sf_cookie_echoed_err( * errors. */ sctp_walk_errors(err, chunk->chunk_hdr) { - if (SCTP_ERROR_STALE_COOKIE == err->cause) - return sctp_sf_do_5_2_6_stale(net, ep, asoc, type, - arg, commands); + if (err->cause != SCTP_ERROR_STALE_COOKIE) + continue; + /* The staleness is only meaningful if the cause is long + * enough to hold it; a shorter one is malformed. + */ + if (ntohs(err->length) < sizeof(*err) + sizeof(__be32)) + break; + return sctp_sf_do_5_2_6_stale(net, ep, asoc, type, + arg, commands, err); } /* It is possible to have malformed error causes, and that @@ -2538,13 +2580,13 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( const struct sctp_association *asoc, const union sctp_subtype type, void *arg, - struct sctp_cmd_seq *commands) + struct sctp_cmd_seq *commands, + struct sctp_errhdr *err) { int attempts = asoc->init_err_counter + 1; - struct sctp_chunk *chunk = arg, *reply; struct sctp_cookie_preserve_param bht; struct sctp_bind_addr *bp; - struct sctp_errhdr *err; + struct sctp_chunk *reply; u32 stale; if (attempts > asoc->max_init_attempts) { @@ -2555,8 +2597,6 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( return SCTP_DISPOSITION_DELETE_TCB; } - err = (struct sctp_errhdr *)(chunk->skb->data); - /* When calculating the time extension, an implementation * SHOULD use the RTT information measured based on the * previous COOKIE ECHO / ERROR exchange, and should add no diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 66e12fb0c646a..c7b9e325ec1cd 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -4111,8 +4111,9 @@ static int sctp_setsockopt_reset_streams(struct sock *sk, if (optlen < sizeof(*params)) return -EINVAL; /* srs_number_streams is u16, so optlen can't be bigger than this. */ - optlen = min_t(unsigned int, optlen, USHRT_MAX + - sizeof(__u16) * sizeof(*params)); + optlen = min_t(unsigned int, optlen, + struct_size_t(struct sctp_reset_streams, srs_stream_list, + USHRT_MAX)); if (params->srs_number_streams * sizeof(__u16) > optlen - sizeof(*params)) @@ -4598,8 +4599,8 @@ static int sctp_setsockopt(struct sock *sk, int level, int optname, if (optlen > 0) { /* Trim it to the biggest size sctp sockopt may need if necessary */ optlen = min_t(unsigned int, optlen, - PAGE_ALIGN(USHRT_MAX + - sizeof(__u16) * sizeof(struct sctp_reset_streams))); + PAGE_ALIGN(struct_size_t(struct sctp_reset_streams, + srs_stream_list, USHRT_MAX))); kopt = memdup_sockptr(optval, optlen); if (IS_ERR(kopt)) return PTR_ERR(kopt); @@ -5369,24 +5370,39 @@ struct sctp_transport *sctp_transport_get_idx(struct net *net, } int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), - void *p) { - int err = 0; - int hash = 0; - struct sctp_endpoint *ep; + struct net *net, int *pos, void *p) { + int err, hash = 0, idx = 0, start; struct sctp_hashbucket *head; + struct sctp_endpoint *ep; for (head = sctp_ep_hashtable; hash < sctp_ep_hashsize; hash++, head++) { + start = idx; +again: read_lock_bh(&head->lock); sctp_for_each_hentry(ep, &head->chain) { - err = cb(ep, p); - if (err) + if (sock_net(ep->base.sk) != net) + continue; + if (idx++ >= *pos) { + sctp_endpoint_hold(ep); break; + } } read_unlock_bh(&head->lock); + + if (ep) { + err = cb(ep, p); + sctp_endpoint_put(ep); + if (err) + return err; + (*pos)++; + + idx = start; + goto again; + } } - return err; + return 0; } EXPORT_SYMBOL_GPL(sctp_for_each_endpoint); diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c index 619b3bab38248..32d6d03df3214 100644 --- a/net/smc/smc_cdc.c +++ b/net/smc/smc_cdc.c @@ -470,9 +470,9 @@ static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) { struct smc_link *link = (struct smc_link *)wc->qp->qp_context; struct smc_cdc_msg *cdc = buf; + struct smc_sock *smc = NULL; struct smc_connection *conn; struct smc_link_group *lgr; - struct smc_sock *smc; if (wc->byte_len < offsetof(struct smc_cdc_msg, reserved)) return; /* short message */ @@ -483,21 +483,26 @@ static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) lgr = smc_get_lgr(link); read_lock_bh(&lgr->conns_lock); conn = smc_lgr_find_conn(ntohl(cdc->token), lgr); - read_unlock_bh(&lgr->conns_lock); - if (!conn || conn->out_of_sync) + if (!conn || conn->out_of_sync) { + read_unlock_bh(&lgr->conns_lock); return; + } smc = container_of(conn, struct smc_sock, conn); + sock_hold(&smc->sk); + read_unlock_bh(&lgr->conns_lock); if (cdc->prod_flags.failover_validation) { smc_cdc_msg_validate(smc, cdc, link); - return; + goto out; } if (smc_cdc_before(ntohs(cdc->seqno), conn->local_rx_ctrl.seqno)) /* received seqno is old */ - return; + goto out; smc_cdc_msg_recv(smc, cdc); +out: + sock_put(&smc->sk); } static struct smc_wr_rx_handler smc_cdc_rx_handlers[] = { diff --git a/net/socket.c b/net/socket.c index c2698a1441a7d..f51bdcbaa43f7 100644 --- a/net/socket.c +++ b/net/socket.c @@ -310,8 +310,10 @@ static int move_addr_to_user(struct sockaddr_storage *kaddr, int klen, static struct kmem_cache *sock_inode_cachep __ro_after_init; +static struct simple_xattr_cache sockfs_xa_cache; + struct sockfs_inode { - struct simple_xattrs *xattrs; + struct list_head xattrs; struct simple_xattr_limits xattr_limits; struct socket_alloc; }; @@ -328,7 +330,7 @@ static struct inode *sock_alloc_inode(struct super_block *sb) si = alloc_inode_sb(sb, sock_inode_cachep, GFP_KERNEL); if (!si) return NULL; - si->xattrs = NULL; + INIT_LIST_HEAD_RCU(&si->xattrs); simple_xattr_limits_init(&si->xattr_limits); init_waitqueue_head(&si->socket.wq.wait); @@ -347,12 +349,8 @@ static struct inode *sock_alloc_inode(struct super_block *sb) static void sock_evict_inode(struct inode *inode) { struct sockfs_inode *si = SOCKFS_I(inode); - struct simple_xattrs *xattrs = si->xattrs; - if (xattrs) { - simple_xattrs_free(xattrs, NULL); - kfree(xattrs); - } + simple_xattrs_free(&sockfs_xa_cache, &si->xattrs, NULL); clear_inode(inode); } @@ -443,13 +441,9 @@ static int sockfs_user_xattr_get(const struct xattr_handler *handler, const char *suffix, void *value, size_t size) { const char *name = xattr_full_name(handler, suffix); - struct simple_xattrs *xattrs; - - xattrs = READ_ONCE(SOCKFS_I(inode)->xattrs); - if (!xattrs) - return -ENODATA; + struct sockfs_inode *si = SOCKFS_I(inode); - return simple_xattr_get(xattrs, name, value, size); + return simple_xattr_get(&sockfs_xa_cache, &si->xattrs, name, value, size); } static int sockfs_user_xattr_set(const struct xattr_handler *handler, @@ -460,13 +454,8 @@ static int sockfs_user_xattr_set(const struct xattr_handler *handler, { const char *name = xattr_full_name(handler, suffix); struct sockfs_inode *si = SOCKFS_I(inode); - struct simple_xattrs *xattrs; - - xattrs = simple_xattrs_lazy_alloc(&si->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); - return simple_xattr_set_limited(xattrs, &si->xattr_limits, + return simple_xattr_set_limited(&sockfs_xa_cache, &si->xattrs, &si->xattr_limits, name, value, size, flags); } @@ -635,8 +624,7 @@ static ssize_t sockfs_listxattr(struct dentry *dentry, char *buffer, struct sockfs_inode *si = SOCKFS_I(d_inode(dentry)); ssize_t len, used; - len = simple_xattr_list(d_inode(dentry), READ_ONCE(si->xattrs), - buffer, size); + len = simple_xattr_list(d_inode(dentry), &si->xattrs, buffer, size); if (len < 0) return len; diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index bc8ca470718b7..efa26899bc7dc 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1026,8 +1026,23 @@ rpc_free_auth(struct rpc_clnt *clnt) return NULL; } -/* - * Release reference to the RPC client +/** + * rpc_hold_client - acquire a reference on an rpc_clnt + * @clnt: rpc_clnt to pin + * + * Pairs with rpc_release_client(). + */ +void rpc_hold_client(struct rpc_clnt *clnt) +{ + refcount_inc(&clnt->cl_count); +} + +/** + * rpc_release_client - release a reference on an rpc_clnt + * @clnt: rpc_clnt to release + * + * Pairs with rpc_hold_client(). The rpc_clnt's resources are + * freed once its reference count drops to zero. */ void rpc_release_client(struct rpc_clnt *clnt) diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 576fa42e7abf7..ae9ec4bf34f72 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -1598,6 +1598,12 @@ static void svc_release_rqst(struct svc_rqst *rqstp) if (procp && procp->pc_release) procp->pc_release(rqstp); + + /* + * A subsequent svc_release_rqst() on this rqstp must not + * re-invoke pc_release against released state. + */ + rqstp->rq_procinfo = NULL; } /** @@ -1616,6 +1622,9 @@ void svc_process(struct svc_rqst *rqstp) svc_xprt_deferred_close(rqstp->rq_xprt); #endif + /* Discard a stale release hook from a previous RPC. */ + rqstp->rq_procinfo = NULL; + /* * Setup response xdr_buf. * Initially it has just one page @@ -1672,6 +1681,7 @@ void svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp) int proc_error; /* Build the svc_rqst used by the common processing routine */ + rqstp->rq_procinfo = NULL; rqstp->rq_xid = req->rq_xid; rqstp->rq_prot = req->rq_xprt->prot; rqstp->rq_bc_net = req->rq_xprt->xprt_net; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 7be3de1a1aed9..eb747493db829 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -471,6 +471,7 @@ static void svc_tcp_handshake_done(void *data, int status, key_serial_t peerid) } clear_bit(XPT_HANDSHAKE, &xprt->xpt_flags); complete_all(&svsk->sk_handshake_done); + svc_xprt_put(xprt); } /** @@ -494,9 +495,13 @@ static void svc_tcp_handshake(struct svc_xprt *xprt) clear_bit(XPT_TLS_SESSION, &xprt->xpt_flags); init_completion(&svsk->sk_handshake_done); + /* Pin the transport across the asynchronous handshake callback. */ + svc_xprt_get(xprt); + ret = tls_server_hello_x509(&args, GFP_KERNEL); if (ret) { trace_svc_tls_not_started(xprt); + svc_xprt_put(xprt); goto out_failed; } @@ -505,8 +510,13 @@ static void svc_tcp_handshake(struct svc_xprt *xprt) if (ret <= 0) { if (tls_handshake_cancel(sk)) { trace_svc_tls_timed_out(xprt); + svc_xprt_put(xprt); goto out_close; } + /* Cancellation lost to handshake_complete(): the + * callback is in flight and should finish quickly. + */ + wait_for_completion(&svsk->sk_handshake_done); } if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) { diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index a90480f801548..e638b92b7ad1f 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -327,7 +327,7 @@ static ssize_t rpc_sysfs_xprt_switch_add_xprt_store(struct kobject *kobj, { struct rpc_xprt_switch *xprt_switch = rpc_sysfs_xprt_switch_kobj_get_xprt(kobj); - struct xprt_create xprt_create_args; + struct xprt_create xprt_create_args = {}; struct rpc_xprt *xprt, *new; if (!xprt_switch) @@ -348,7 +348,7 @@ static ssize_t rpc_sysfs_xprt_switch_add_xprt_store(struct kobject *kobj, xprt_create_args.reconnect_timeout = xprt->max_reconnect_timeout; new = xprt_create_transport(&xprt_create_args); - if (IS_ERR_OR_NULL(new)) { + if (IS_ERR(new)) { count = PTR_ERR(new); goto out_put_xprt; } diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index e83d5d0be78b7..4c41e83b7ca9d 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -152,6 +152,8 @@ unsigned int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, unsigned int count = 0; if (head->iov_len) { + if (unlikely(count >= bvec_size)) + goto bvec_overflow; bvec_set_virt(bvec++, head->iov_base, head->iov_len); ++count; } @@ -165,25 +167,27 @@ unsigned int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, while (remaining > 0) { len = min_t(unsigned int, remaining, PAGE_SIZE - offset); + if (unlikely(count >= bvec_size)) + goto bvec_overflow; bvec_set_page(bvec++, *pages++, len, offset); remaining -= len; offset = 0; - if (unlikely(++count > bvec_size)) - goto bvec_overflow; + ++count; } } if (tail->iov_len) { - bvec_set_virt(bvec, tail->iov_base, tail->iov_len); - if (unlikely(++count > bvec_size)) + if (unlikely(count >= bvec_size)) goto bvec_overflow; + bvec_set_virt(bvec, tail->iov_base, tail->iov_len); + ++count; } return count; bvec_overflow: pr_warn_once("%s: bio_vec array overflow\n", __func__); - return count - 1; + return count; } EXPORT_SYMBOL_GPL(xdr_buf_to_bvec); diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c index 2f0f9618dd058..e5b3463da25f9 100644 --- a/net/sunrpc/xprtrdma/backchannel.c +++ b/net/sunrpc/xprtrdma/backchannel.c @@ -159,9 +159,7 @@ void xprt_rdma_bc_free_rqst(struct rpc_rqst *rqst) rpcrdma_rep_put(&r_xprt->rx_buf, rep); req->rl_reply = NULL; - spin_lock(&xprt->bc_pa_lock); - list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list); - spin_unlock(&xprt->bc_pa_lock); + rpcrdma_req_put(req); xprt_put(xprt); } @@ -203,6 +201,7 @@ static struct rpc_rqst *rpcrdma_bc_rqst_get(struct rpcrdma_xprt *r_xprt) rqst->rq_xprt = xprt; __set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state); xdr_buf_init(&rqst->rq_snd_buf, rdmab_data(req->rl_sendbuf), size); + kref_init(&req->rl_kref); return rqst; } diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 7f79a0a2601e6..e5c71cf705a33 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -474,7 +474,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) ++num_wrs; } - if ((kref_read(&req->rl_kref) > 1) || num_wrs > ep->re_send_count) { + if (req->rl_sendctx->sc_unmap_count || num_wrs > ep->re_send_count) { send_wr->send_flags |= IB_SEND_SIGNALED; ep->re_send_count = min_t(unsigned int, ep->re_send_batch, num_wrs - ep->re_send_count); diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 0e0f219747109..e8b5fc86a01bb 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -467,29 +467,11 @@ static int rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, return 0; } -static void rpcrdma_sendctx_done(struct kref *kref) -{ - struct rpcrdma_req *req = - container_of(kref, struct rpcrdma_req, rl_kref); - struct rpcrdma_rep *rep = req->rl_reply; - - rpcrdma_complete_rqst(rep); - rep->rr_rxprt->rx_stats.reply_waits_for_send++; -} - -/** - * rpcrdma_sendctx_unmap - DMA-unmap Send buffer - * @sc: sendctx containing SGEs to unmap - * - */ -void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) +static void rpcrdma_sendctx_dma_unmap(struct rpcrdma_sendctx *sc) { struct rpcrdma_regbuf *rb = sc->sc_req->rl_sendbuf; struct ib_sge *sge; - if (!sc->sc_unmap_count) - return; - /* The first two SGEs contain the transport header and * the inline buffer. These are always left mapped so * they can be cheaply re-used. @@ -498,8 +480,29 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) ++sge, --sc->sc_unmap_count) ib_dma_unmap_page(rdmab_device(rb), sge->addr, sge->length, DMA_TO_DEVICE); +} - kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done); +/** + * rpcrdma_sendctx_unmap - DMA-unmap Send buffer and release Send owner + * @sc: sendctx containing SGEs to unmap + * + */ +void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) +{ + struct rpcrdma_req *req = sc->sc_req; + + rpcrdma_sendctx_dma_unmap(sc); + sc->sc_req = NULL; + rpcrdma_req_put(req); +} + +/* No Send was posted. Release DMA mappings prepared for this + * sendctx, but leave the request reference count alone. + */ +static void rpcrdma_sendctx_cancel(struct rpcrdma_sendctx *sc) +{ + rpcrdma_sendctx_dma_unmap(sc); + sc->sc_req = NULL; } /* Prepare an SGE for the RPC-over-RDMA transport header. @@ -691,8 +694,6 @@ static bool rpcrdma_prepare_noch_mapped(struct rpcrdma_xprt *r_xprt, tail->iov_len)) return false; - if (req->rl_sendctx->sc_unmap_count) - kref_get(&req->rl_kref); return true; } @@ -722,7 +723,6 @@ static bool rpcrdma_prepare_readch(struct rpcrdma_xprt *r_xprt, len -= len & 3; if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len)) return false; - kref_get(&req->rl_kref); } return true; @@ -743,6 +743,7 @@ inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdr, enum rpcrdma_chunktype rtype) { + struct rpcrdma_sendctx *sc; int ret; ret = -EAGAIN; @@ -751,7 +752,6 @@ inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, goto out_nosc; req->rl_sendctx->sc_unmap_count = 0; req->rl_sendctx->sc_req = req; - kref_init(&req->rl_kref); req->rl_wr.wr_cqe = &req->rl_sendctx->sc_cqe; req->rl_wr.sg_list = req->rl_sendctx->sc_sges; req->rl_wr.num_sge = 0; @@ -779,10 +779,16 @@ inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, goto out_unmap; } + /* The Send-side owner releases this reference when the + * Send has completed. + */ + kref_get(&req->rl_kref); return 0; out_unmap: - rpcrdma_sendctx_unmap(req->rl_sendctx); + sc = req->rl_sendctx; + rpcrdma_sendctx_cancel(sc); + rpcrdma_sendctx_unget_locked(r_xprt, sc); out_nosc: trace_xprtrdma_prepsend_failed(&req->rl_slot, ret); return ret; @@ -1081,6 +1087,8 @@ rpcrdma_is_bcall(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) /* Peek at stream contents without advancing. */ p = xdr_inline_decode(xdr, 0); + if ((char *)xdr->end - (char *)p < 5 * XDR_UNIT) + return false; /* Chunk lists */ if (xdr_item_is_present(p++)) @@ -1105,7 +1113,7 @@ rpcrdma_is_bcall(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) */ p = xdr_inline_decode(xdr, 3 * sizeof(*p)); if (unlikely(!p)) - return true; + return false; rpcrdma_bc_receive_call(r_xprt, rep); return true; @@ -1329,6 +1337,11 @@ void rpcrdma_complete_rqst(struct rpcrdma_rep *rep) struct rpc_rqst *rqst = rep->rr_rqst; int status; + /* I3: every registered MR has been invalidated and + * ib_dma_unmap_sg()'d before complete_rqst runs. + */ + WARN_ON_ONCE(!list_empty(&rpcr_to_rdmar(rqst)->rl_registered)); + switch (rep->rr_proc) { case rdma_msg: status = rpcrdma_decode_msg(r_xprt, rep, rqst); @@ -1360,13 +1373,69 @@ void rpcrdma_complete_rqst(struct rpcrdma_rep *rep) goto out; } -static void rpcrdma_reply_done(struct kref *kref) -{ - struct rpcrdma_req *req = - container_of(kref, struct rpcrdma_req, rl_kref); - - rpcrdma_complete_rqst(req->rl_reply); -} +/* Reply-side ownership invariants + * + * I1 (Receive WR ownership). A struct rpcrdma_rep is owned by the + * HCA between ib_post_recv() and the matching Receive completion. + * After ib_dma_sync_single_for_cpu() in rpcrdma_wc_receive() it is + * owned by the CPU until rpcrdma_rep_put() returns it to + * rb_free_reps; a rep on rb_free_reps is not re-posted until + * rpcrdma_post_recvs() pulls it off. Asserted: rpcrdma_post_recvs() + * WARNs that a pulled rep has rr_rqst == NULL. + * + * I2 (rep attachment). While req->rl_reply == rep, the rep cannot be + * re-posted. rpcrdma_reply_put() NULLs req->rl_reply before handing + * the rep to rpcrdma_rep_put(). Asserted: rpcrdma_reply_put() WARNs + * that rl_reply is NULL after the put. + * + * I3 (Registered-MR fence). On entry to rpcrdma_complete_rqst() every + * MR that was on req->rl_registered has had its rkey invalidated + * (remotely via IB_WC_WITH_INVALIDATE or locally via IB_WR_LOCAL_INV) + * and its pages ib_dma_unmap_sg()'d. The LocalInv chain is posted + * on a single QP; strong send-queue ordering makes the last + * completion (frwr_wc_localinv_done) observe the + * ib_dma_unmap_sg() that ran from each earlier completion's + * frwr_mr_put() before complete_rqst is called. The inline + * frwr_reminv() path unmaps its one MR synchronously before + * rpcrdma_reply_handler() reaches complete_rqst. Asserted: + * rpcrdma_complete_rqst() WARNs that rl_registered is empty. + * + * I4 (Send-buffer release). req->rl_kref carries two unconditional + * owners while a Send is outstanding: the RPC-layer reference (set + * at xprt_rdma_alloc_slot / xprt_rdma_bc_rqst_get / rpcrdma_req_release + * pool-entry) and the Send-side reference (kref_get() in + * rpcrdma_prepare_send_sges()). rpcrdma_req_release() runs only + * after both have dropped, so the req does not return to its free + * pool until rpcrdma_sendctx_unmap() has fired -- the HCA has + * released the send buffer before the req can be reused. Asserted: + * rpcrdma_req_release() WARNs that rl_sendctx is NULL. + * + * I5 (req lifecycle). A req is owned by the RPC layer between slot + * acquisition and the matching xprt_rdma_free_slot() (or, for the + * backchannel, xprt_rdma_bc_free_rqst()). While owned, rl_kref >= 1. + * The pools (rb_send_bufs, bc_pa_list, backlog wake target) never + * contain a req with outstanding Send-side or Reply-side work. + * + * Non-hazards. The following claims have been raised by adversarial + * review and are each closed by the invariants above: + * + * * "Reply completes the RPC while the HCA still holds the send + * buffer" -- excluded by I4. The Send-side kref reference is held + * until rpcrdma_sendctx_unmap() runs from Send completion. + * + * * "Signal-driven release races the in-flight Send" -- same + * resolution. xprt_rdma_free() does not touch rl_kref; the + * Send-side reference keeps the req out of its pool until Send + * completion fires. + * + * * "Receive completion races rep reuse" -- excluded by I1. A rep + * is on rb_free_reps only after rpcrdma_rep_put() has been called + * and rpcrdma_post_recvs() owns the next transition back to the HCA. + * + * * "Pages still DMA-mapped when call_decode reads them" -- excluded + * by I3. The matching ib_dma_unmap_sg() for every MR has run on + * the same CPU thread that calls rpcrdma_complete_rqst(). + */ /** * rpcrdma_reply_handler - Process received RPC/RDMA messages @@ -1402,6 +1471,14 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) credits = be32_to_cpu(*p++); rep->rr_proc = *p++; + /* The credit grant from the wire is not trustworthy; + * sanitize it before any code path consumes it. + */ + if (credits == 0) + credits = 1; /* don't deadlock */ + else if (credits > r_xprt->rx_ep->re_max_requests) + credits = r_xprt->rx_ep->re_max_requests; + if (rep->rr_vers != rpcrdma_version) goto out_badversion; @@ -1418,10 +1495,6 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) xprt_pin_rqst(rqst); spin_unlock(&xprt->queue_lock); - if (credits == 0) - credits = 1; /* don't deadlock */ - else if (credits > r_xprt->rx_ep->re_max_requests) - credits = r_xprt->rx_ep->re_max_requests; if (buf->rb_credits != credits) rpcrdma_update_cwnd(r_xprt, credits); @@ -1439,7 +1512,7 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) frwr_unmap_async(r_xprt, req); /* LocalInv completion will complete the RPC */ else - kref_put(&req->rl_kref, rpcrdma_reply_done); + rpcrdma_complete_rqst(rep); out_post: rpcrdma_post_recvs(r_xprt, @@ -1454,11 +1527,13 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) out_badversion: trace_xprtrdma_reply_vers_err(rep); - goto out; + rpcrdma_rep_put(buf, rep); + credits = buf->rb_credits; + goto out_post; out_shortreply: trace_xprtrdma_reply_short_err(rep); - -out: rpcrdma_rep_put(buf, rep); + credits = buf->rb_credits; + goto out_post; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index f8a0638eb095d..96594a0f8de6c 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -377,7 +377,7 @@ static void svc_rdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) trace_svcrdma_wc_recv_err(wc, &ctxt->rc_cid); dropped: svc_rdma_recv_ctxt_put(rdma, ctxt); - svc_xprt_deferred_close(&rdma->sc_xprt); + svc_rdma_xprt_deferred_close(rdma); } /** @@ -1001,7 +1001,7 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp) if (ret == -EINVAL) svc_rdma_send_error(rdma_xprt, ctxt, ret); svc_rdma_recv_ctxt_put(rdma_xprt, ctxt); - svc_xprt_deferred_close(xprt); + svc_rdma_xprt_deferred_close(rdma_xprt); return ret; } return 0; diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 402e2ceca4ff7..73d17ac83911d 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -313,7 +313,7 @@ static void svc_rdma_reply_done(struct ib_cq *cq, struct ib_wc *wc) trace_svcrdma_wc_reply_err(wc, &cc->cc_cid); } - svc_xprt_deferred_close(&rdma->sc_xprt); + svc_rdma_xprt_deferred_close(rdma); } /** @@ -345,7 +345,7 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) * some of the outgoing RPC message. Signal the loss * to the client by closing the connection. */ - svc_xprt_deferred_close(&rdma->sc_xprt); + svc_rdma_xprt_deferred_close(rdma); } /** @@ -390,7 +390,7 @@ static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc) */ svc_rdma_cc_release(rdma, cc, DMA_FROM_DEVICE); svc_rdma_recv_ctxt_put(rdma, ctxt); - svc_xprt_deferred_close(&rdma->sc_xprt); + svc_rdma_xprt_deferred_close(rdma); } /* @@ -754,216 +754,6 @@ int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, return xdr->len; } -/* - * Cap contiguous RDMA Read sink allocations at order-4. - * Higher orders risk allocation failure under - * __GFP_NORETRY, which would negate the benefit of the - * contiguous fast path. - */ -#define SVC_RDMA_CONTIG_MAX_ORDER 4 - -/** - * svc_rdma_alloc_read_pages - Allocate physically contiguous pages - * @nr_pages: number of pages needed - * @order: on success, set to the allocation order - * - * Attempts a higher-order allocation, falling back to smaller orders. - * The returned pages are split immediately so each sub-page has its - * own refcount and can be freed independently. - * - * Returns a pointer to the first page on success, or NULL if even - * order-1 allocation fails. - */ -static struct page * -svc_rdma_alloc_read_pages(unsigned int nr_pages, unsigned int *order) -{ - unsigned int o; - struct page *page; - - o = min(get_order(nr_pages << PAGE_SHIFT), - SVC_RDMA_CONTIG_MAX_ORDER); - - while (o >= 1) { - page = alloc_pages(GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN, - o); - if (page) { - split_page(page, o); - *order = o; - return page; - } - o--; - } - return NULL; -} - -/* - * svc_rdma_fill_contig_bvec - Replace rq_pages with a contiguous allocation - * @rqstp: RPC transaction context - * @head: context for ongoing I/O - * @bv: bvec entry to fill - * @pages_left: number of data pages remaining in the segment - * @len_left: bytes remaining in the segment - * - * On success, fills @bv with a bvec spanning the contiguous range and - * advances rc_curpage/rc_page_count. Returns the byte length covered, - * or zero if the allocation failed or would overrun rq_maxpages. - */ -static unsigned int -svc_rdma_fill_contig_bvec(struct svc_rqst *rqstp, - struct svc_rdma_recv_ctxt *head, - struct bio_vec *bv, unsigned int pages_left, - unsigned int len_left) -{ - unsigned int order, npages, chunk_pages, chunk_len, i; - struct page *page; - - page = svc_rdma_alloc_read_pages(pages_left, &order); - if (!page) - return 0; - npages = 1 << order; - - if (head->rc_curpage + npages > rqstp->rq_maxpages) { - for (i = 0; i < npages; i++) - __free_page(page + i); - return 0; - } - - /* - * Replace rq_pages[] entries with pages from the contiguous - * allocation. If npages exceeds chunk_pages, the extra pages - * stay in rq_pages[] for later reuse or normal rqst teardown. - */ - for (i = 0; i < npages; i++) { - svc_rqst_page_release(rqstp, - rqstp->rq_pages[head->rc_curpage + i]); - rqstp->rq_pages[head->rc_curpage + i] = page + i; - } - - chunk_pages = min(npages, pages_left); - chunk_len = min_t(unsigned int, chunk_pages << PAGE_SHIFT, len_left); - bvec_set_page(bv, page, chunk_len, 0); - head->rc_page_count += chunk_pages; - head->rc_curpage += chunk_pages; - return chunk_len; -} - -/* - * svc_rdma_fill_page_bvec - Add a single rq_page to the bvec array - * @head: context for ongoing I/O - * @ctxt: R/W context whose bvec array is being filled - * @cur: page to add - * @bvec_idx: pointer to current bvec index, not advanced on merge - * @len_left: bytes remaining in the segment - * - * If @cur is physically contiguous with the preceding bvec, it is - * merged by extending that bvec's length. Otherwise a new bvec - * entry is created. Returns the byte length covered. - */ -static unsigned int -svc_rdma_fill_page_bvec(struct svc_rdma_recv_ctxt *head, - struct svc_rdma_rw_ctxt *ctxt, struct page *cur, - unsigned int *bvec_idx, unsigned int len_left) -{ - unsigned int chunk_len = min_t(unsigned int, PAGE_SIZE, len_left); - - head->rc_page_count++; - head->rc_curpage++; - - if (*bvec_idx > 0) { - struct bio_vec *prev = &ctxt->rw_bvec[*bvec_idx - 1]; - - if (page_to_phys(prev->bv_page) + prev->bv_offset + - prev->bv_len == page_to_phys(cur)) { - prev->bv_len += chunk_len; - return chunk_len; - } - } - - bvec_set_page(&ctxt->rw_bvec[*bvec_idx], cur, chunk_len, 0); - (*bvec_idx)++; - return chunk_len; -} - -/** - * svc_rdma_build_read_segment_contig - Build RDMA Read WR with contiguous pages - * @rqstp: RPC transaction context - * @head: context for ongoing I/O - * @segment: co-ordinates of remote memory to be read - * - * Greedily allocates higher-order pages to cover the segment, - * building one bvec per contiguous chunk. Each allocation is - * split so sub-pages have independent refcounts. When a - * higher-order allocation fails, remaining pages are covered - * individually, merging adjacent pages into the preceding bvec - * when they are physically contiguous. The split sub-pages - * replace entries in rq_pages[] so downstream cleanup is - * unchanged. - * - * Returns: - * %0: the Read WR was constructed successfully - * %-ENOMEM: allocation failed - * %-EIO: a DMA mapping error occurred - */ -static int svc_rdma_build_read_segment_contig(struct svc_rqst *rqstp, - struct svc_rdma_recv_ctxt *head, - const struct svc_rdma_segment *segment) -{ - struct svcxprt_rdma *rdma = svc_rdma_rqst_rdma(rqstp); - struct svc_rdma_chunk_ctxt *cc = &head->rc_cc; - unsigned int nr_data_pages, bvec_idx; - struct svc_rdma_rw_ctxt *ctxt; - unsigned int len_left; - int ret; - - nr_data_pages = PAGE_ALIGN(segment->rs_length) >> PAGE_SHIFT; - if (head->rc_curpage + nr_data_pages > rqstp->rq_maxpages) - return -ENOMEM; - - ctxt = svc_rdma_get_rw_ctxt(rdma, nr_data_pages); - if (!ctxt) - return -ENOMEM; - - bvec_idx = 0; - len_left = segment->rs_length; - while (len_left) { - unsigned int pages_left = PAGE_ALIGN(len_left) >> PAGE_SHIFT; - unsigned int chunk_len = 0; - - if (pages_left >= 2) - chunk_len = svc_rdma_fill_contig_bvec(rqstp, head, - &ctxt->rw_bvec[bvec_idx], - pages_left, len_left); - if (chunk_len) { - bvec_idx++; - } else { - struct page *cur = - rqstp->rq_pages[head->rc_curpage]; - chunk_len = svc_rdma_fill_page_bvec(head, ctxt, cur, - &bvec_idx, - len_left); - } - - len_left -= chunk_len; - } - - ctxt->rw_nents = bvec_idx; - - head->rc_pageoff = offset_in_page(segment->rs_length); - if (head->rc_pageoff) - head->rc_curpage--; - - ret = svc_rdma_rw_ctx_init(rdma, ctxt, segment->rs_offset, - segment->rs_handle, segment->rs_length, - DMA_FROM_DEVICE); - if (ret < 0) - return -EIO; - percpu_counter_inc(&svcrdma_stat_read); - - list_add(&ctxt->rw_list, &cc->cc_rwctxts); - cc->cc_sqecount += ret; - return 0; -} - /** * svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment * @rqstp: RPC transaction context @@ -990,14 +780,6 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, if (check_add_overflow(head->rc_pageoff, len, &total)) return -EINVAL; nr_bvec = PAGE_ALIGN(total) >> PAGE_SHIFT; - - if (head->rc_pageoff == 0 && nr_bvec >= 2) { - ret = svc_rdma_build_read_segment_contig(rqstp, head, - segment); - if (ret != -ENOMEM) - return ret; - } - ctxt = svc_rdma_get_rw_ctxt(rdma, nr_bvec); if (!ctxt) return -ENOMEM; @@ -1343,11 +1125,6 @@ static void svc_rdma_clear_rqst_pages(struct svc_rqst *rqstp, { unsigned int i; - /* - * Move only pages containing RPC data into rc_pages[]. Pages - * from a contiguous allocation that were not used for the - * payload remain in rq_pages[] for subsequent reuse. - */ for (i = 0; i < head->rc_page_count; i++) { head->rc_pages[i] = rqstp->rq_pages[i]; rqstp->rq_pages[i] = NULL; diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 8b3f0c8c14b25..88ad6bbed69b8 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -395,7 +395,7 @@ int svc_rdma_post_send_err(struct svcxprt_rdma *rdma, int sqecount, int ret) { trace_svcrdma_sq_post_err(rdma, cid, ret); - svc_xprt_deferred_close(&rdma->sc_xprt); + svc_rdma_xprt_deferred_close(rdma); /* If even one WR was posted, a Send completion will * return the reserved SQ slots. @@ -437,7 +437,7 @@ static void svc_rdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) else trace_svcrdma_wc_send_flush(wc, &ctxt->sc_cid); svc_rdma_send_ctxt_put(rdma, ctxt); - svc_xprt_deferred_close(&rdma->sc_xprt); + svc_rdma_xprt_deferred_close(rdma); } /** @@ -1158,7 +1158,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) svc_rdma_send_ctxt_put(rdma, sctxt); drop_connection: trace_svcrdma_send_err(rqstp, ret); - svc_xprt_deferred_close(&rdma->sc_xprt); + svc_rdma_xprt_deferred_close(rdma); return -ENOTCONN; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index f18bc60d9f4ff..246f332db2f75 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -98,10 +98,27 @@ struct svc_xprt_class svc_rdma_class = { .xcl_ident = XPRT_TRANSPORT_RDMA, }; +/** + * svc_rdma_xprt_deferred_close - Close an RDMA transport (deferred) + * @rdma: transport to close + */ +void svc_rdma_xprt_deferred_close(struct svcxprt_rdma *rdma) +{ + svc_xprt_deferred_close(&rdma->sc_xprt); + + /* Release parked sc_sq_ticket_wait and sc_send_wait waiters. + * Once XPT_CLOSE is observed each returns -ENOTCONN. + */ + wake_up_all(&rdma->sc_sq_ticket_wait); + wake_up_all(&rdma->sc_send_wait); +} + /* QP event handler */ static void qp_event_handler(struct ib_event *event, void *context) { struct svc_xprt *xprt = context; + struct svcxprt_rdma *rdma = + container_of(xprt, struct svcxprt_rdma, sc_xprt); trace_svcrdma_qp_error(event, (struct sockaddr *)&xprt->xpt_remote); switch (event->event) { @@ -119,7 +136,7 @@ static void qp_event_handler(struct ib_event *event, void *context) case IB_EVENT_QP_ACCESS_ERR: case IB_EVENT_DEVICE_FATAL: default: - svc_xprt_deferred_close(xprt); + svc_rdma_xprt_deferred_close(rdma); break; } } @@ -340,7 +357,7 @@ static int svc_rdma_cma_handler(struct rdma_cm_id *cma_id, svc_xprt_enqueue(xprt); break; case RDMA_CM_EVENT_DISCONNECTED: - svc_xprt_deferred_close(xprt); + svc_rdma_xprt_deferred_close(rdma); break; default: break; @@ -597,6 +614,15 @@ static void svc_rdma_detach(struct svc_xprt *xprt) container_of(xprt, struct svcxprt_rdma, sc_xprt); rdma_disconnect(rdma->sc_cm_id); + + /* + * Most close paths go through svc_rdma_xprt_deferred_close(), + * which wakes the SQ waitqueues. svc_xprt_close() reaches + * detach without that helper, so wake any threads parked in + * svc_rdma_sq_wait() here as well. + */ + wake_up_all(&rdma->sc_sq_ticket_wait); + wake_up_all(&rdma->sc_send_wait); } /** diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 61706df5e485a..5ff8e5126a6c2 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -279,6 +279,13 @@ xprt_rdma_destroy(struct rpc_xprt *xprt) cancel_delayed_work_sync(&r_xprt->rx_connect_worker); rpcrdma_xprt_disconnect(r_xprt); + + /* The disconnect's sendctx drain can return bc_prealloc reqs + * to bc_pa_list after xprt_destroy_backchannel() emptied it. + */ +#if defined(CONFIG_SUNRPC_BACKCHANNEL) + xprt_rdma_bc_destroy(xprt, 0); +#endif rpcrdma_buffer_destroy(&r_xprt->rx_buf); xprt_rdma_free_addresses(xprt); @@ -487,6 +494,50 @@ xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task) queue_delayed_work(system_long_wq, &r_xprt->rx_connect_worker, delay); } +/* rl_kref has two owners while a Send is outstanding: the rpc_rqst + * owner and the sendctx. Replies complete the RPC but do not drop + * either reference. The req returns to its free pool only after + * xprt_rdma_free_slot() or xprt_rdma_bc_free_rqst() has dropped the + * RPC-layer reference and rpcrdma_sendctx_unmap() has dropped the + * Send-side reference. + */ +static void rpcrdma_req_release(struct kref *kref) +{ + struct rpcrdma_req *req = + container_of(kref, struct rpcrdma_req, rl_kref); + struct rpc_rqst *rqst = &req->rl_slot; + struct rpc_xprt *xprt = rqst->rq_xprt; + struct rpcrdma_xprt *r_xprt; + + /* I4: both the RPC-layer and Send-side owners have dropped, + * so rpcrdma_sendctx_unmap() has cleared rl_sendctx. + */ + WARN_ON_ONCE(req->rl_sendctx); + + kref_init(&req->rl_kref); + +#if defined(CONFIG_SUNRPC_BACKCHANNEL) + if (bc_prealloc(rqst)) { + spin_lock(&xprt->bc_pa_lock); + list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list); + spin_unlock(&xprt->bc_pa_lock); + return; + } +#endif + + if (xprt_wake_up_backlog(xprt, rqst)) + return; + + r_xprt = rpcx_to_rdmax(xprt); + memset(rqst, 0, sizeof(*rqst)); + rpcrdma_buffer_put(&r_xprt->rx_buf, req); +} + +void rpcrdma_req_put(struct rpcrdma_req *req) +{ + kref_put(&req->rl_kref, rpcrdma_req_release); +} + /** * xprt_rdma_alloc_slot - allocate an rpc_rqst * @xprt: controlling RPC transport @@ -505,6 +556,7 @@ xprt_rdma_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) req = rpcrdma_buffer_get(&r_xprt->rx_buf); if (!req) goto out_sleep; + kref_init(&req->rl_kref); task->tk_rqstp = &req->rl_slot; task->tk_status = 0; return; @@ -520,6 +572,7 @@ xprt_rdma_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) if (req) { struct rpc_rqst *rqst = &req->rl_slot; + kref_init(&req->rl_kref); if (!xprt_wake_up_backlog(xprt, rqst)) { memset(rqst, 0, sizeof(*rqst)); rpcrdma_buffer_put(&r_xprt->rx_buf, req); @@ -540,10 +593,7 @@ xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst) container_of(xprt, struct rpcrdma_xprt, rx_xprt); rpcrdma_reply_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst)); - if (!xprt_wake_up_backlog(xprt, rqst)) { - memset(rqst, 0, sizeof(*rqst)); - rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst)); - } + rpcrdma_req_put(rpcr_to_rdmar(rqst)); } static bool rpcrdma_check_regbuf(struct rpcrdma_xprt *r_xprt, @@ -607,10 +657,10 @@ xprt_rdma_free(struct rpc_task *task) frwr_unmap_sync(rpcx_to_rdmax(rqst->rq_xprt), req); } - /* XXX: If the RPC is completing because of a signal and - * not because a reply was received, we ought to ensure - * that the Send completion has fired, so that memory - * involved with the Send is not still visible to the NIC. + /* The Send-side rl_kref owner keeps req out of its free pool + * until rpcrdma_sendctx_unmap() has fired -- see I4 above + * rpcrdma_reply_handler() -- so signal-driven release here + * does not let the HCA touch a recycled send buffer. */ } @@ -716,7 +766,7 @@ void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) r_xprt->rx_stats.mrs_allocated, r_xprt->rx_stats.local_inv_needed, r_xprt->rx_stats.empty_sendctx_q, - r_xprt->rx_stats.reply_waits_for_send); + 0LU); /* was reply_waits_for_send; column preserved */ } static int diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index aecf9c0a153f3..0ac82f26bf6ec 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -65,6 +65,8 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt); +static unsigned long rpcrdma_sendctx_next(struct rpcrdma_buffer *buf, + unsigned long item); static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, struct rpcrdma_sendctx *sc); static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt); @@ -79,6 +81,8 @@ rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction, int node); static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction); +static bool rpcrdma_regbuf_realloc_node(struct rpcrdma_regbuf *rb, + size_t size, gfp_t flags, int node); static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb); static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); @@ -243,8 +247,17 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) complete(&ep->re_done); return 0; case RDMA_CM_EVENT_ADDR_CHANGE: - ep->re_connect_status = -ENODEV; - goto disconnected; + switch (xchg(&ep->re_connect_status, -ENODEV)) { + case 0: + goto wake_connect_worker; + case 1: + /* The later DISCONNECTED event balances the + * ESTABLISHED get; do not put here. + */ + rpcrdma_force_disconnect(ep); + return 0; + } + return 0; case RDMA_CM_EVENT_ESTABLISHED: rpcrdma_ep_get(ep); ep->re_connect_status = 1; @@ -267,7 +280,6 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) return 0; case RDMA_CM_EVENT_DISCONNECTED: ep->re_connect_status = -ECONNABORTED; -disconnected: rpcrdma_force_disconnect(ep); return rpcrdma_ep_put(ep); default: @@ -324,6 +336,7 @@ static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt, if (rc) goto out; + ep->re_id = id; rc = rpcrdma_rn_register(id->device, &ep->re_rn, rpcrdma_ep_removal_done); if (rc) goto out; @@ -396,7 +409,6 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) } __module_get(THIS_MODULE); device = id->device; - ep->re_id = id; reinit_completion(&ep->re_done); ep->re_max_requests = r_xprt->rx_xprt.max_reqs; @@ -539,7 +551,17 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) goto out; } rpcrdma_mrs_create(r_xprt); - frwr_wp_create(r_xprt); + + /* + * rpcrdma_encode_write_list() dereferences the write-pad + * MR with no NULL check, so fail the connect rather than + * publish a transport whose write-pad MR is NULL. + */ + rc = frwr_wp_create(r_xprt); + if (rc) { + rc = -ENOTCONN; + goto out; + } out: trace_xprtrdma_connect(r_xprt, rc); @@ -571,9 +593,9 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) rpcrdma_xprt_drain(r_xprt); rpcrdma_reps_unmap(r_xprt); + rpcrdma_sendctxs_destroy(r_xprt); rpcrdma_reqs_reset(r_xprt); rpcrdma_mrs_destroy(r_xprt); - rpcrdma_sendctxs_destroy(r_xprt); if (rpcrdma_ep_put(ep)) rdma_destroy_id(id); @@ -605,6 +627,25 @@ static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt) if (!buf->rb_sc_ctxs) return; + + /* The QP is drained, but the final unsignaled Sends might not + * have been walked by a signaled Send completion. Release those + * Send owners before request buffers are reset. + * + * Unlike the completion sweep, this walk can visit slots with + * no Send posted: after a partial rpcrdma_sendctxs_create() + * failure on reconnect, rb_sc_head and rb_sc_tail are stale, + * and slots between them can be NULL or have sc_req clear. + */ + for (i = rpcrdma_sendctx_next(buf, buf->rb_sc_tail); + i != rpcrdma_sendctx_next(buf, buf->rb_sc_head); + i = rpcrdma_sendctx_next(buf, i)) { + struct rpcrdma_sendctx *sc = buf->rb_sc_ctxs[i]; + + if (sc && sc->sc_req) + rpcrdma_sendctx_unmap(sc); + } + for (i = 0; i <= buf->rb_sc_last; i++) kfree(buf->rb_sc_ctxs[i]); kfree(buf->rb_sc_ctxs); @@ -667,6 +708,12 @@ static unsigned long rpcrdma_sendctx_next(struct rpcrdma_buffer *buf, return likely(item < buf->rb_sc_last) ? item + 1 : 0; } +static unsigned long rpcrdma_sendctx_prev(struct rpcrdma_buffer *buf, + unsigned long item) +{ + return item > 0 ? item - 1 : buf->rb_sc_last; +} + /** * rpcrdma_sendctx_get_locked - Acquire a send context * @r_xprt: controlling transport instance @@ -723,6 +770,29 @@ struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt) return NULL; } +/** + * rpcrdma_sendctx_unget_locked - Release an unposted send context + * @r_xprt: controlling transport instance + * @sc: send context to release + * + * Usage: Called when no Send is posted for the sendctx most + * recently returned by rpcrdma_sendctx_get_locked(). + * + * The caller serializes calls to this function and to + * rpcrdma_sendctx_get_locked() (per transport). + */ +void rpcrdma_sendctx_unget_locked(struct rpcrdma_xprt *r_xprt, + struct rpcrdma_sendctx *sc) +{ + struct rpcrdma_buffer *buf = &r_xprt->rx_buf; + + if (WARN_ON_ONCE(buf->rb_sc_ctxs[buf->rb_sc_head] != sc)) + return; + + buf->rb_sc_head = rpcrdma_sendctx_prev(buf, buf->rb_sc_head); + xprt_write_space(&r_xprt->rx_xprt); +} + /** * rpcrdma_sendctx_put_locked - Release a send context * @r_xprt: controlling transport instance @@ -739,15 +809,18 @@ static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, struct rpcrdma_buffer *buf = &r_xprt->rx_buf; unsigned long next_tail; - /* Unmap SGEs of previously completed but unsignaled - * Sends by walking up the queue until @sc is found. + /* Release previously completed but unsignaled Sends by walking + * up the queue until @sc is found. */ next_tail = buf->rb_sc_tail; do { + struct rpcrdma_sendctx *cur; + next_tail = rpcrdma_sendctx_next(buf, next_tail); /* ORDER: item must be accessed _before_ tail is updated */ - rpcrdma_sendctx_unmap(buf->rb_sc_ctxs[next_tail]); + cur = buf->rb_sc_ctxs[next_tail]; + rpcrdma_sendctx_unmap(cur); } while (buf->rb_sc_ctxs[next_tail] != sc); @@ -1211,6 +1284,10 @@ void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) rpcrdma_rep_put(buffers, req->rl_reply); req->rl_reply = NULL; } + /* I2: rl_reply NULL after the put closes the + * 'rep on rb_free_reps still referenced by req' window. + */ + WARN_ON_ONCE(req->rl_reply); } /** @@ -1291,10 +1368,16 @@ rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction) * returned, @rb is left untouched. */ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) +{ + return rpcrdma_regbuf_realloc_node(rb, size, flags, NUMA_NO_NODE); +} + +static bool rpcrdma_regbuf_realloc_node(struct rpcrdma_regbuf *rb, + size_t size, gfp_t flags, int node) { void *buf; - buf = kmalloc(size, flags); + buf = kmalloc_node(size, flags, node); if (!buf) return false; @@ -1306,6 +1389,23 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) return true; } +static bool rpcrdma_rep_resize(struct rpcrdma_xprt *r_xprt, + struct rpcrdma_rep *rep) +{ + struct rpcrdma_regbuf *rb = rep->rr_rdmabuf; + struct rpcrdma_ep *ep = r_xprt->rx_ep; + size_t size = ep->re_inline_recv; + + if (likely(rdmab_length(rb) >= size)) + return true; + if (!rpcrdma_regbuf_realloc_node(rb, size, XPRTRDMA_GFP_FLAGS, + ibdev_to_node(ep->re_id->device))) + return false; + + xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rb), rdmab_length(rb)); + return true; +} + /** * __rpcrdma_regbuf_dma_map - DMA-map a regbuf * @r_xprt: controlling transport instance @@ -1387,6 +1487,12 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) rep = rpcrdma_rep_create(r_xprt); if (!rep) break; + /* I1: a rep on rb_free_reps must carry no rqst pointer. */ + WARN_ON_ONCE(rep->rr_rqst); + if (!rpcrdma_rep_resize(r_xprt, rep)) { + rpcrdma_rep_put(buf, rep); + break; + } if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) { rpcrdma_rep_put(buf, rep); break; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index f53a774727245..57be5776aaff1 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -427,7 +427,6 @@ struct rpcrdma_stats { /* accessed when receiving a reply */ unsigned long long total_rdma_reply; unsigned long long fixup_copy_count; - unsigned long reply_waits_for_send; unsigned long local_inv_needed; unsigned long nomsg_call_count; unsigned long bcall_count; @@ -496,6 +495,8 @@ void rpcrdma_req_destroy(struct rpcrdma_req *req); int rpcrdma_buffer_create(struct rpcrdma_xprt *); void rpcrdma_buffer_destroy(struct rpcrdma_buffer *); struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt); +void rpcrdma_sendctx_unget_locked(struct rpcrdma_xprt *r_xprt, + struct rpcrdma_sendctx *sc); struct rpcrdma_mr *rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt); void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt); @@ -505,6 +506,7 @@ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req); void rpcrdma_rep_put(struct rpcrdma_buffer *buf, struct rpcrdma_rep *rep); void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req); +void rpcrdma_req_put(struct rpcrdma_req *req); bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags); diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 2e1fe60133615..359407aae03e6 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -2734,8 +2734,11 @@ static void xs_tcp_tls_setup_socket(struct work_struct *work) lower_xprt = rcu_dereference(lower_clnt->cl_xprt); rcu_read_unlock(); - if (wait_on_bit_lock(&lower_xprt->state, XPRT_LOCKED, TASK_KILLABLE)) + if (wait_on_bit_lock(&lower_xprt->state, XPRT_LOCKED, TASK_KILLABLE)) { + /* XPRT_LOCKED was never acquired. */ + rpc_shutdown_client(lower_clnt); goto out_unlock; + } status = xs_tls_handshake_sync(lower_xprt, &upper_xprt->xprtsec); if (status) { @@ -2758,6 +2761,7 @@ static void xs_tcp_tls_setup_socket(struct work_struct *work) out_unlock: current_restore_flags(pflags, PF_MEMALLOC); upper_transport->clnt = NULL; + rpc_release_client(upper_clnt); xprt_unlock_connect(upper_xprt, upper_transport); return; @@ -2805,7 +2809,15 @@ static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task) } else dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); - transport->clnt = task->tk_client; + /* + * Only the TLS connect_worker reads transport->clnt; pinning + * the upper rpc_clnt unconditionally would form a cycle with + * cl_xprt and prevent xprt destruction. + */ + if (xprt->xprtsec.policy != RPC_XPRTSEC_NONE) { + rpc_hold_client(task->tk_client); + transport->clnt = task->tk_client; + } queue_delayed_work(xprtiod_workqueue, &transport->connect_worker, delay); diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c index 76a1585d3f6b6..10d1ec5930846 100644 --- a/net/tipc/bcast.c +++ b/net/tipc/bcast.c @@ -497,12 +497,13 @@ void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, */ int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr, - struct sk_buff_head *retrq) + struct sk_buff_head *retrq, bool *valid) { struct sk_buff_head *inputq = &tipc_bc_base(net)->inputq; struct tipc_gap_ack_blks *ga; struct sk_buff_head xmitq; int rc = 0; + u16 glen; __skb_queue_head_init(&xmitq); @@ -510,13 +511,18 @@ int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, if (msg_type(hdr) != STATE_MSG) { tipc_link_bc_init_rcv(l, hdr); } else if (!msg_bc_ack_invalid(hdr)) { - tipc_get_gap_ack_blks(&ga, l, hdr, false); - if (!sysctl_tipc_bc_retruni) - retrq = &xmitq; - rc = tipc_link_bc_ack_rcv(l, msg_bcast_ack(hdr), - msg_bc_gap(hdr), ga, &xmitq, - retrq); - rc |= tipc_link_bc_sync_rcv(l, hdr, &xmitq); + glen = tipc_get_gap_ack_blks(&ga, l, hdr, false); + if (glen > msg_data_sz(hdr)) { + /* Malformed Gap ACK blocks; caller drops the msg */ + *valid = false; + } else { + if (!sysctl_tipc_bc_retruni) + retrq = &xmitq; + rc = tipc_link_bc_ack_rcv(l, msg_bcast_ack(hdr), + msg_bc_gap(hdr), ga, &xmitq, + retrq); + rc |= tipc_link_bc_sync_rcv(l, hdr, &xmitq); + } } tipc_bcast_unlock(net); diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h index 2d9352dc7b0e6..55d17b5413e16 100644 --- a/net/tipc/bcast.h +++ b/net/tipc/bcast.h @@ -97,7 +97,7 @@ void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr); int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr, - struct sk_buff_head *retrq); + struct sk_buff_head *retrq, bool *valid); int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg, struct tipc_link *bcl); int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]); diff --git a/net/tipc/bearer.c b/net/tipc/bearer.c index a3bd1ef17558a..05dcd2f9e887a 100644 --- a/net/tipc/bearer.c +++ b/net/tipc/bearer.c @@ -482,6 +482,7 @@ void tipc_disable_l2_media(struct tipc_bearer *b) dev = (struct net_device *)rtnl_dereference(b->media_ptr); dev_remove_pack(&b->pt); RCU_INIT_POINTER(dev->tipc_ptr, NULL); + RCU_INIT_POINTER(b->media_ptr, NULL); synchronize_net(); dev_put(dev); } diff --git a/net/tipc/core.c b/net/tipc/core.c index 434e70eabe081..315975c3be818 100644 --- a/net/tipc/core.c +++ b/net/tipc/core.c @@ -45,6 +45,7 @@ #include "crypto.h" #include +#include /* configurable TIPC parameters */ unsigned int tipc_net_id __read_mostly; @@ -118,8 +119,7 @@ static void __net_exit tipc_exit_net(struct net *net) #ifdef CONFIG_TIPC_CRYPTO tipc_crypto_stop(&tipc_net(net)->crypto_tx); #endif - while (atomic_read(&tn->wq_count)) - cond_resched(); + wait_var_event(&tn->wq_count, atomic_read(&tn->wq_count) == 0); } static void __net_exit tipc_pernet_pre_exit(struct net *net) @@ -218,6 +218,11 @@ static void __exit tipc_exit(void) unregister_pernet_device(&tipc_net_ops); tipc_unregister_sysctl(); + /* TODO: Wait for all timers that called call_rcu() to finish before + * calling rcu_barrier(). + */ + rcu_barrier(); + pr_info("Deactivated\n"); } diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index 6d3b6b89b1d17..16f1ed1f6b1b5 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -941,12 +941,20 @@ static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead, goto exit; } + /* Get net to avoid freed tipc_crypto when delete namespace */ + if (!maybe_get_net(net)) { + tipc_bearer_put(b); + rc = -ENODEV; + goto exit; + } + /* Now, do decrypt */ rc = crypto_aead_decrypt(req); if (rc == -EINPROGRESS || rc == -EBUSY) return rc; tipc_bearer_put(b); + put_net(net); exit: kfree(ctx); @@ -984,6 +992,7 @@ static void tipc_aead_decrypt_done(void *data, int err) } tipc_bearer_put(b); + put_net(net); } static inline int tipc_ehdr_size(struct tipc_ehdr *ehdr) diff --git a/net/tipc/discover.c b/net/tipc/discover.c index 3e54d2df5683a..b9d06595b0673 100644 --- a/net/tipc/discover.c +++ b/net/tipc/discover.c @@ -58,6 +58,7 @@ * @skb: request message to be (repeatedly) sent * @timer: timer governing period between requests * @timer_intv: current interval between requests (in ms) + * @rcu: RCU head for deferred freeing */ struct tipc_discoverer { u32 bearer_id; @@ -69,6 +70,7 @@ struct tipc_discoverer { struct sk_buff *skb; struct timer_list timer; unsigned long timer_intv; + struct rcu_head rcu; }; /** @@ -382,6 +384,15 @@ int tipc_disc_create(struct net *net, struct tipc_bearer *b, return 0; } +static void tipc_disc_free_rcu(struct rcu_head *rp) +{ + struct tipc_discoverer *d = container_of(rp, struct tipc_discoverer, + rcu); + + kfree_skb(d->skb); + kfree(d); +} + /** * tipc_disc_delete - destroy object sending periodic link setup requests * @d: ptr to link dest structure @@ -389,8 +400,7 @@ int tipc_disc_create(struct net *net, struct tipc_bearer *b, void tipc_disc_delete(struct tipc_discoverer *d) { timer_shutdown_sync(&d->timer); - kfree_skb(d->skb); - kfree(d); + call_rcu(&d->rcu, tipc_disc_free_rcu); } /** diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c index 190b49c5cbc3e..ba4f4906e13b7 100644 --- a/net/tipc/name_distr.c +++ b/net/tipc/name_distr.c @@ -280,12 +280,21 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i, u32 node, u32 dtype) { struct publication *p = NULL; + u32 lower = ntohl(i->lower); + u32 upper = ntohl(i->upper); struct tipc_socket_addr sk; - struct tipc_uaddr ua; u32 key = ntohl(i->key); + struct tipc_uaddr ua; + + /* A peer-advertised binding with lower > upper can never be matched + * or withdrawn and would leak the publication; the local bind path + * rejects such ranges, so reject ranges learned from the network too. + */ + if (lower > upper) + return false; tipc_uaddr(&ua, TIPC_SERVICE_RANGE, TIPC_CLUSTER_SCOPE, - ntohl(i->type), ntohl(i->lower), ntohl(i->upper)); + ntohl(i->type), lower, upper); sk.ref = ntohl(i->port); sk.node = node; diff --git a/net/tipc/netlink.c b/net/tipc/netlink.c index 1a9a5bdaccf4f..8336a9664703f 100644 --- a/net/tipc/netlink.c +++ b/net/tipc/netlink.c @@ -152,11 +152,13 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_BEARER_DISABLE, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_bearer_disable, }, { .cmd = TIPC_NL_BEARER_ENABLE, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_bearer_enable, }, { @@ -168,11 +170,13 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_BEARER_ADD, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_bearer_add, }, { .cmd = TIPC_NL_BEARER_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_bearer_set, }, { @@ -197,11 +201,13 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_LINK_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_node_set_link, }, { .cmd = TIPC_NL_LINK_RESET_STATS, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_node_reset_link_stats, }, { @@ -213,6 +219,7 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_MEDIA_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_media_set, }, { @@ -228,6 +235,7 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_NET_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_net_set, }, { @@ -238,6 +246,7 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_MON_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_node_set_monitor, }, { @@ -255,6 +264,7 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_PEER_REMOVE, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_peer_rm, }, #ifdef CONFIG_TIPC_MEDIA_UDP @@ -269,11 +279,13 @@ static const struct genl_ops tipc_genl_v2_ops[] = { { .cmd = TIPC_NL_KEY_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_node_set_key, }, { .cmd = TIPC_NL_KEY_FLUSH, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_UNS_ADMIN_PERM, .doit = tipc_nl_node_flush_key, }, #endif diff --git a/net/tipc/node.c b/net/tipc/node.c index 97aa970a0d83c..8e4ef2630ae4f 100644 --- a/net/tipc/node.c +++ b/net/tipc/node.c @@ -1831,12 +1831,15 @@ static void tipc_node_mcast_rcv(struct tipc_node *n) } static void tipc_node_bc_sync_rcv(struct tipc_node *n, struct tipc_msg *hdr, - int bearer_id, struct sk_buff_head *xmitq) + int bearer_id, struct sk_buff_head *xmitq, + bool *valid) { struct tipc_link *ucl; int rc; - rc = tipc_bcast_sync_rcv(n->net, n->bc_entry.link, hdr, xmitq); + rc = tipc_bcast_sync_rcv(n->net, n->bc_entry.link, hdr, xmitq, valid); + if (!*valid) + return; if (rc & TIPC_LINK_DOWN_EVT) { tipc_node_reset_links(n); @@ -2140,12 +2143,18 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b) /* Ensure broadcast reception is in synch with peer's send state */ if (unlikely(usr == LINK_PROTOCOL)) { + bool valid = true; + if (unlikely(skb_linearize(skb))) { tipc_node_put(n); goto discard; } hdr = buf_msg(skb); - tipc_node_bc_sync_rcv(n, hdr, bearer_id, &xmitq); + tipc_node_bc_sync_rcv(n, hdr, bearer_id, &xmitq, &valid); + if (!valid) { + tipc_node_put(n); + goto discard; + } } else if (unlikely(tipc_link_acked(n->bc_entry.link) != bc_ack)) { tipc_bcast_ack_rcv(net, n->bc_entry.link, hdr); } diff --git a/net/tipc/socket.c b/net/tipc/socket.c index 9329919fb07f0..490f30899b5a9 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -1362,6 +1362,9 @@ static void tipc_sk_conn_proto_rcv(struct tipc_sock *tsk, struct sk_buff *skb, __skb_queue_tail(xmitq, skb); return; } else if (mtyp == CONN_ACK) { + if (tsk->snt_unacked < msg_conn_ack(hdr)) + goto exit; + was_cong = tsk_conn_cong(tsk); tipc_sk_push_backlog(tsk, msg_nagle_ack(hdr)); tsk->snt_unacked -= msg_conn_ack(hdr); @@ -2452,17 +2455,17 @@ static void tipc_sk_enqueue(struct sk_buff_head *inputq, struct sock *sk, atomic_set(dcnt, 0); lim = rcvbuf_limit(sk, skb) + atomic_read(dcnt); if (likely(!sk_add_backlog(sk, skb, lim))) { - trace_tipc_sk_overlimit1(sk, skb, TIPC_DUMP_ALL, + trace_tipc_sk_overlimit1(sk, skb, TIPC_DUMP_SK_BKLGQ, "bklg & rcvq >90% allocated!"); continue; } - trace_tipc_sk_dump(sk, skb, TIPC_DUMP_ALL, "err_overload!"); + trace_tipc_sk_dump(sk, skb, TIPC_DUMP_SK_BKLGQ, "err_overload!"); /* Overload => reject message back to sender */ onode = tipc_own_addr(sock_net(sk)); sk_drops_inc(sk); if (tipc_msg_reverse(onode, &skb, TIPC_ERR_OVERLOAD)) { - trace_tipc_sk_rej_msg(sk, skb, TIPC_DUMP_ALL, + trace_tipc_sk_rej_msg(sk, skb, TIPC_DUMP_SK_BKLGQ, "@sk_enqueue!"); __skb_queue_tail(xmitq, skb); } diff --git a/net/tipc/udp_media.c b/net/tipc/udp_media.c index 2c66b356025a7..62ae7f5b58409 100644 --- a/net/tipc/udp_media.c +++ b/net/tipc/udp_media.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -89,14 +90,14 @@ struct udp_replicast { /** * struct udp_bearer - ip/udp bearer data structure * @bearer: associated generic tipc bearer - * @ubsock: bearer associated socket + * @sk: bearer associated socket * @ifindex: local address scope * @work: used to schedule deferred work on a bearer * @rcast: associated udp_replicast container */ struct udp_bearer { struct tipc_bearer __rcu *bearer; - struct socket *ubsock; + struct sock *sk; u32 ifindex; struct work_struct work; struct udp_replicast rcast; @@ -194,7 +195,7 @@ static int tipc_udp_xmit(struct net *net, struct sk_buff *skb, } ttl = ip4_dst_hoplimit(&rt->dst); - udp_tunnel_xmit_skb(rt, ub->ubsock->sk, skb, src->ipv4.s_addr, + udp_tunnel_xmit_skb(rt, ub->sk, skb, src->ipv4.s_addr, dst->ipv4.s_addr, 0, ttl, 0, src->port, dst->port, false, true, 0); #if IS_ENABLED(CONFIG_IPV6) @@ -206,7 +207,7 @@ static int tipc_udp_xmit(struct net *net, struct sk_buff *skb, .saddr = src->ipv6, .flowi6_proto = IPPROTO_UDP }; - ndst = ip6_dst_lookup_flow(net, ub->ubsock->sk, + ndst = ip6_dst_lookup_flow(net, ub->sk, &fl6, NULL); if (IS_ERR(ndst)) { err = PTR_ERR(ndst); @@ -215,7 +216,7 @@ static int tipc_udp_xmit(struct net *net, struct sk_buff *skb, dst_cache_set_ip6(cache, ndst, &fl6.saddr); } ttl = ip6_dst_hoplimit(ndst); - udp_tunnel6_xmit_skb(ndst, ub->ubsock->sk, skb, NULL, + udp_tunnel6_xmit_skb(ndst, ub->sk, skb, NULL, &src->ipv6, &dst->ipv6, 0, ttl, 0, src->port, dst->port, false, 0); #endif @@ -405,9 +406,9 @@ static int tipc_udp_recv(struct sock *sk, struct sk_buff *skb) static int enable_mcast(struct udp_bearer *ub, struct udp_media_addr *remote) { - int err = 0; + struct sock *sk = ub->sk; struct ip_mreqn mreqn; - struct sock *sk = ub->ubsock->sk; + int err = 0; if (ntohs(remote->proto) == ETH_P_IP) { mreqn.imr_multiaddr = remote->ipv4; @@ -670,6 +671,7 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b, struct nlattr *opts[TIPC_NLA_UDP_MAX + 1]; u8 node_id[NODE_ID_LEN] = {0,}; struct net_device *dev; + struct socket *sock; int rmcast = 0; ub = kzalloc_obj(*ub, GFP_ATOMIC); @@ -764,14 +766,16 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b, goto err; } udp_conf.local_udp_port = local.port; - err = udp_sock_create(net, &udp_conf, &ub->ubsock); + err = udp_sock_create(net, &udp_conf, &sock); if (err) goto err; + + ub->sk = sock->sk; tuncfg.sk_user_data = ub; tuncfg.encap_type = 1; tuncfg.encap_rcv = tipc_udp_recv; tuncfg.encap_destroy = NULL; - setup_udp_tunnel_sock(net, ub->ubsock, &tuncfg); + setup_udp_tunnel_sock(net, ub->sk, &tuncfg); err = dst_cache_init(&ub->rcast.dst_cache, GFP_ATOMIC); if (err) @@ -793,12 +797,21 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b, free: dst_cache_destroy(&ub->rcast.dst_cache); - udp_tunnel_sock_release(ub->ubsock); + udp_tunnel_sock_release(ub->sk); + synchronize_rcu(); err: kfree(ub); return err; } +static void rcast_free_rcu(struct rcu_head *rcu) +{ + struct udp_replicast *rcast = container_of(rcu, struct udp_replicast, rcu); + + dst_cache_destroy(&rcast->dst_cache); + kfree(rcast); +} + /* cleanup_bearer - break the socket/bearer association */ static void cleanup_bearer(struct work_struct *work) { @@ -807,19 +820,19 @@ static void cleanup_bearer(struct work_struct *work) struct tipc_net *tn; list_for_each_entry_safe(rcast, tmp, &ub->rcast.list, list) { - dst_cache_destroy(&rcast->dst_cache); list_del_rcu(&rcast->list); - kfree_rcu(rcast, rcu); + call_rcu_hurry(&rcast->rcu, rcast_free_rcu); } - tn = tipc_net(sock_net(ub->ubsock->sk)); + tn = tipc_net(sock_net(ub->sk)); - dst_cache_destroy(&ub->rcast.dst_cache); - udp_tunnel_sock_release(ub->ubsock); + udp_tunnel_sock_release(ub->sk); - /* Note: could use a call_rcu() to avoid another synchronize_net() */ synchronize_net(); - atomic_dec(&tn->wq_count); + + dst_cache_destroy(&ub->rcast.dst_cache); + if (atomic_dec_and_test(&tn->wq_count)) + wake_up_var(&tn->wq_count); kfree(ub); } @@ -833,11 +846,11 @@ static void tipc_udp_disable(struct tipc_bearer *b) pr_err("UDP bearer instance not found\n"); return; } - sock_set_flag(ub->ubsock->sk, SOCK_DEAD); + sock_set_flag(ub->sk, SOCK_DEAD); RCU_INIT_POINTER(ub->bearer, NULL); /* sock_release need to be done outside of rtnl lock */ - atomic_inc(&tipc_net(sock_net(ub->ubsock->sk))->wq_count); + atomic_inc(&tipc_net(sock_net(ub->sk))->wq_count); INIT_WORK(&ub->work, cleanup_bearer); schedule_work(&ub->work); } diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 964ebc268ee46..ba95f9d927a35 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -623,6 +623,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, struct scatterlist *sge, *osge, *nsge; u32 orig_size = msg_opl->sg.size; struct scatterlist tmp = { }; + u32 tmp_i = 0; struct sk_msg *msg_npl; struct tls_rec *new; int ret; @@ -644,6 +645,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, if (sge->length > apply) { u32 len = sge->length - apply; + tmp_i = i; get_page(sg_page(sge)); sg_set_page(&tmp, sg_page(sge), len, sge->offset + apply); @@ -675,6 +677,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, nsge = sk_msg_elem(msg_npl, j); if (tmp.length) { memcpy(nsge, &tmp, sizeof(*nsge)); + sk_msg_sg_copy_assign(msg_npl, j, msg_opl, tmp_i); sk_msg_iter_var_next(j); nsge = sk_msg_elem(msg_npl, j); } @@ -682,6 +685,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, osge = sk_msg_elem(msg_opl, i); while (osge->length) { memcpy(nsge, osge, sizeof(*nsge)); + sk_msg_sg_copy_assign(msg_npl, j, msg_opl, i); sg_unmark_end(nsge); sk_msg_iter_var_next(i); sk_msg_iter_var_next(j); @@ -2420,6 +2424,17 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, goto read_sock_requeue; } + /* An empty data record (legal in TLS 1.3) gives a zero + * read_actor return, indistinguishable from the consumer + * stalling; the used <= 0 path would requeue it at the + * head of rx_list and block all later records. Consume it + * here instead. + */ + if (rxm->full_len == 0) { + consume_skb(skb); + continue; + } + used = read_actor(desc, skb, rxm->offset, rxm->full_len); if (used <= 0) { if (!copied) diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 76c537a6e8b52..21e43e08b5d79 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -5112,7 +5112,7 @@ static int nl80211_validate_key_link_id(struct genl_info *info, if (wdev->valid_links) { if (link_id == -1) { GENL_SET_ERR_MSG(info, - "link ID must for MLO group key"); + "link ID must be set for MLO group key"); return -EINVAL; } if (!(wdev->valid_links & BIT(link_id))) { diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c index d6cd0de64d1f8..0ca93fd7d46f8 100644 --- a/net/wireless/pmsr.c +++ b/net/wireless/pmsr.c @@ -195,7 +195,9 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, return -EOPNOTSUPP; } - if (out->ftm.rsta && !out->ftm.lmr_feedback) { + if (out->ftm.rsta && + (out->ftm.non_trigger_based || out->ftm.trigger_based) && + !out->ftm.lmr_feedback) { NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_FTM_REQ_ATTR_RSTA], "FTM: RSTA set without LMR feedback"); diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 27a56ee2e8f0b..d9f2a77e127ce 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -326,8 +326,11 @@ cfg80211_gen_new_ie(const u8 *ie, size_t ielen, /* For ML probe response, match the MLE in the frame body with * MLD id being 'bssid_index' */ - if (parent->id == WLAN_EID_EXTENSION && parent->datalen > 1 && + if (parent->id == WLAN_EID_EXTENSION && parent->data[0] == WLAN_EID_EXT_EHT_MULTI_LINK && + ieee80211_mle_type_ok(parent->data + 1, + IEEE80211_ML_CONTROL_TYPE_BASIC, + parent->datalen - 1) && bssid_index == ieee80211_mle_get_mld_id(parent->data + 1)) { if (!cfg80211_copy_elem_with_frags(parent, ie, ielen, diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index d9035546375eb..374e1b9644383 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -212,43 +212,23 @@ static int espintcp_sendskmsg_locked(struct sock *sk, struct sk_msg *skmsg = &emsg->skmsg; bool more = flags & MSG_MORE; struct scatterlist *sg; - int done = 0; int ret; - sg = &skmsg->sg.data[skmsg->sg.start]; do { struct bio_vec bvec; - size_t size = sg->length - emsg->offset; - int offset = sg->offset + emsg->offset; - struct page *p; - - emsg->offset = 0; + sg = &skmsg->sg.data[skmsg->sg.start]; if (sg_is_last(sg) && !more) msghdr.msg_flags &= ~MSG_MORE; - p = sg_page(sg); -retry: - bvec_set_page(&bvec, p, size, offset); - iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, size); - ret = tcp_sendmsg_locked(sk, &msghdr, size); - if (ret < 0) { - emsg->offset = offset - sg->offset; - skmsg->sg.start += done; + bvec_set_page(&bvec, sg_page(sg), sg->length, sg->offset); + iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, sg->length); + ret = tcp_sendmsg_locked(sk, &msghdr, sg->length); + if (ret < 0) return ret; - } - - if (ret != size) { - offset += ret; - size -= ret; - goto retry; - } - done++; - put_page(p); - sk_mem_uncharge(sk, sg->length); - sg = sg_next(sg); - } while (sg); + sk_msg_free_partial(sk, skmsg, ret); + } while (skmsg->sg.size); memset(emsg, 0, sizeof(*emsg)); diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index e4c2cd24936d3..eecab337bd0a7 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c @@ -467,6 +467,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) { const struct xfrm_state_afinfo *afinfo; struct net *net = dev_net(skb->dev); + struct net_device *dev = skb->dev; int err; __be32 seq; __be32 seq_hi; @@ -493,7 +494,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) LINUX_MIB_XFRMINSTATEINVALID); if (encap_type == -1) - dev_put(skb->dev); + dev_put(dev); goto drop; } @@ -655,16 +656,16 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) if (!crypto_done) { spin_unlock(&x->lock); - dev_hold(skb->dev); + dev_hold(dev); nexthdr = x->type->input(x, skb); if (nexthdr == -EINPROGRESS) { if (async) - dev_put(skb->dev); + dev_put(dev); return 0; } - dev_put(skb->dev); + dev_put(dev); spin_lock(&x->lock); } resume: @@ -699,7 +700,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) err = xfrm_inner_mode_input(x, skb); if (err == -EINPROGRESS) { if (async) - dev_put(skb->dev); + dev_put(dev); return 0; } else if (err) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); @@ -726,9 +727,12 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) crypto_done = false; } while (!err); + rcu_read_lock(); err = xfrm_rcv_cb(skb, family, x->type->proto, 0); - if (err) + if (err) { + rcu_read_unlock(); goto drop; + } nf_reset_ct(skb); @@ -739,8 +743,9 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) if (skb_valid_dst(skb)) skb_dst_drop(skb); if (async) - dev_put(skb->dev); + dev_put(dev); gro_cells_receive(&gro_cells, skb); + rcu_read_unlock(); return 0; } else { xo = xfrm_offload(skb); @@ -748,23 +753,21 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) xfrm_gro = xo->flags & XFRM_GRO; err = -EAFNOSUPPORT; - rcu_read_lock(); afinfo = xfrm_state_afinfo_get_rcu(x->props.family); if (likely(afinfo)) err = afinfo->transport_finish(skb, xfrm_gro || async); - rcu_read_unlock(); if (xfrm_gro) { sp = skb_sec_path(skb); if (sp) sp->olen = 0; if (skb_valid_dst(skb)) skb_dst_drop(skb); - if (async) - dev_put(skb->dev); gro_cells_receive(&gro_cells, skb); - return err; } + if (async) + dev_put(dev); + rcu_read_unlock(); return err; } @@ -772,7 +775,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) spin_unlock(&x->lock); drop: if (async) - dev_put(skb->dev); + dev_put(dev); xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); kfree_skb(skb); return 0; diff --git a/net/xfrm/xfrm_interface_core.c b/net/xfrm/xfrm_interface_core.c index 330a05286a56f..688306bf62c5a 100644 --- a/net/xfrm/xfrm_interface_core.c +++ b/net/xfrm/xfrm_interface_core.c @@ -869,6 +869,9 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[], struct net *net = xi->net; struct xfrm_if_parms p = {}; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + xfrmi_netlink_parms(data, &p); if (!p.if_id) { NL_SET_ERR_MSG(extack, "if_id must be non zero"); diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index 458931062a04e..eb1b6f67739e1 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -55,8 +55,10 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb, ka->encap_sport, sock_net_uid(net, NULL)); rt = ip_route_output_key(net, &fl4); - if (IS_ERR(rt)) + if (IS_ERR(rt)) { + kfree_skb(skb); return PTR_ERR(rt); + } skb_dst_set(skb, &rt->dst); @@ -101,6 +103,7 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb, dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); if (IS_ERR(dst)) { local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock); + kfree_skb(skb); return PTR_ERR(dst); } @@ -118,7 +121,6 @@ static void nat_keepalive_send(struct nat_keepalive *ka) sizeof(struct ipv6hdr)) + sizeof(struct udphdr); const u8 nat_ka_payload = 0xFF; - int err = -EAFNOSUPPORT; struct sk_buff *skb; struct udphdr *uh; @@ -140,16 +142,17 @@ static void nat_keepalive_send(struct nat_keepalive *ka) switch (ka->family) { case AF_INET: - err = nat_keepalive_send_ipv4(skb, ka); + nat_keepalive_send_ipv4(skb, ka); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: - err = nat_keepalive_send_ipv6(skb, ka, uh); + nat_keepalive_send_ipv6(skb, ka, uh); break; #endif - } - if (err) + default: kfree_skb(skb); + break; + } } struct nat_keepalive_work_ctx { diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 9595444256929..639934f300167 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -242,6 +242,9 @@ __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl, unsigned short family) { + if (family != sel->family && sel->family != AF_UNSPEC) + return false; + switch (family) { case AF_INET: return __xfrm4_selector_match(sel, fl); @@ -685,7 +688,7 @@ static void xfrm_byidx_resize(struct net *net) static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total) { - unsigned int cnt = net->xfrm.policy_count[dir]; + unsigned int cnt = READ_ONCE(net->xfrm.policy_count[dir]); unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; if (total) @@ -711,12 +714,12 @@ static inline int xfrm_byidx_should_resize(struct net *net, int total) void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si) { - si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; - si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; - si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; - si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; - si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; - si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; + si->incnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_IN]); + si->outcnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT]); + si->fwdcnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_FWD]); + si->inscnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]); + si->outscnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]); + si->fwdscnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]); si->spdhcnt = net->xfrm.policy_idx_hmask; si->spdhmcnt = xfrm_policy_hashmax; } @@ -2318,7 +2321,7 @@ static void __xfrm_policy_link(struct xfrm_policy *pol, int dir) } list_add(&pol->walk.all, &net->xfrm.policy_all); - net->xfrm.policy_count[dir]++; + WRITE_ONCE(net->xfrm.policy_count[dir], net->xfrm.policy_count[dir] + 1); xfrm_pol_hold(pol); } @@ -2337,7 +2340,7 @@ static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, } list_del_init(&pol->walk.all); - net->xfrm.policy_count[dir]--; + WRITE_ONCE(net->xfrm.policy_count[dir], net->xfrm.policy_count[dir] - 1); return pol; } @@ -3222,7 +3225,7 @@ struct dst_entry *xfrm_lookup_with_ifid(struct net *net, /* To accelerate a bit... */ if (!if_id && ((dst_orig->flags & DST_NOXFRM) || - !net->xfrm.policy_count[XFRM_POLICY_OUT])) + !READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT]))) goto nopol; xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id); @@ -3296,7 +3299,7 @@ struct dst_entry *xfrm_lookup_with_ifid(struct net *net, nopol: if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && - net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { + READ_ONCE(net->xfrm.policy_default[dir]) == XFRM_USERPOLICY_BLOCK) { err = -EPERM; goto error; } @@ -3750,7 +3753,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, const bool is_crypto_offload = sp && (xfrm_input_state(skb)->xso.type == XFRM_DEV_OFFLOAD_CRYPTO); - if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { + if (READ_ONCE(net->xfrm.policy_default[dir]) == XFRM_USERPOLICY_BLOCK) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS); return 0; } diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 589c3b6e46791..ec9ecf72c76ba 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -1207,9 +1207,11 @@ struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark, struct hlist_head *state_cache_input; struct xfrm_state *x = NULL; + /* BH is always disabled on the input path. */ + lockdep_assert_in_softirq(); + state_cache_input = raw_cpu_ptr(net->xfrm.state_cache_input); - rcu_read_lock(); hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) { if (x->props.family != family || x->id.spi != spi || @@ -1227,20 +1229,25 @@ struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark, xfrm_hash_ptrs_get(net, &state_ptrs); x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family); - - if (x && x->km.state == XFRM_STATE_VALID) { - spin_lock_bh(&net->xfrm.xfrm_state_lock); - if (hlist_unhashed(&x->state_cache_input)) { + if (x) { + spin_lock(&net->xfrm.xfrm_state_lock); + if (x->km.state != XFRM_STATE_VALID) { + /* + * The state is about to be destroyed. + * + * Don't add it to the cache but still + * return it to the caller. + */ + } else if (hlist_unhashed(&x->state_cache_input)) { hlist_add_head_rcu(&x->state_cache_input, state_cache_input); } else { hlist_del_rcu(&x->state_cache_input); hlist_add_head_rcu(&x->state_cache_input, state_cache_input); } - spin_unlock_bh(&net->xfrm.xfrm_state_lock); + spin_unlock(&net->xfrm.xfrm_state_lock); } out: - rcu_read_unlock(); return x; } EXPORT_SYMBOL(xfrm_input_state_lookup); @@ -2020,6 +2027,8 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, if (!x->encap) goto error; + x->mapping_maxage = orig->mapping_maxage; + x->nat_keepalive_interval = orig->nat_keepalive_interval; } if (orig->security) @@ -2054,7 +2063,6 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, x->km.seq = orig->km.seq; x->replay = orig->replay; x->preplay = orig->preplay; - x->mapping_maxage = orig->mapping_maxage; x->lastused = orig->lastused; x->new_mapping = 0; x->new_mapping_sport = 0; @@ -2976,7 +2984,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (IS_ERR(data)) return PTR_ERR(data); - if (in_compat_syscall()) { + if (IS_ENABLED(CONFIG_COMPAT_FOR_U64_ALIGNMENT) && in_compat_syscall()) { struct xfrm_translator *xtr = xfrm_get_translator(); if (!xtr) { diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 71a4b7278eba9..61eb5de33b87e 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -2485,9 +2485,9 @@ static int xfrm_notify_userpolicy(struct net *net) } up = nlmsg_data(nlh); - up->in = net->xfrm.policy_default[XFRM_POLICY_IN]; - up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD]; - up->out = net->xfrm.policy_default[XFRM_POLICY_OUT]; + up->in = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_IN]); + up->fwd = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_FWD]); + up->out = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT]); nlmsg_end(skb, nlh); @@ -2511,13 +2511,13 @@ static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh, struct xfrm_userpolicy_default *up = nlmsg_data(nlh); if (xfrm_userpolicy_is_valid(up->in)) - net->xfrm.policy_default[XFRM_POLICY_IN] = up->in; + WRITE_ONCE(net->xfrm.policy_default[XFRM_POLICY_IN], up->in); if (xfrm_userpolicy_is_valid(up->fwd)) - net->xfrm.policy_default[XFRM_POLICY_FWD] = up->fwd; + WRITE_ONCE(net->xfrm.policy_default[XFRM_POLICY_FWD], up->fwd); if (xfrm_userpolicy_is_valid(up->out)) - net->xfrm.policy_default[XFRM_POLICY_OUT] = up->out; + WRITE_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT], up->out); rt_genid_bump_all(net); @@ -2547,9 +2547,9 @@ static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh, } r_up = nlmsg_data(r_nlh); - r_up->in = net->xfrm.policy_default[XFRM_POLICY_IN]; - r_up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD]; - r_up->out = net->xfrm.policy_default[XFRM_POLICY_OUT]; + r_up->in = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_IN]); + r_up->fwd = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_FWD]); + r_up->out = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT]); nlmsg_end(r_skb, r_nlh); return nlmsg_unicast(xfrm_net_nlsk(net, skb), r_skb, portid); @@ -3472,7 +3472,7 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, if (!netlink_net_capable(skb, CAP_NET_ADMIN)) return -EPERM; - if (in_compat_syscall()) { + if (IS_ENABLED(CONFIG_COMPAT_FOR_U64_ALIGNMENT) && in_compat_syscall()) { struct xfrm_translator *xtr = xfrm_get_translator(); if (!xtr) diff --git a/rust/Makefile b/rust/Makefile index b9e9f512cec31..bec9726f256ce 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -517,7 +517,7 @@ $(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE $(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE $(call if_changed,exports) -quiet_cmd_rustc_procmacrolibrary = $(RUSTC_OR_CLIPPY_QUIET) PL $@ +quiet_cmd_rustc_procmacrolibrary = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) PL $@ cmd_rustc_procmacrolibrary = \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c index d9625b9ee0466..6e59cc9c665cc 100644 --- a/rust/helpers/uaccess.c +++ b/rust/helpers/uaccess.c @@ -14,7 +14,7 @@ rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) return copy_to_user(to, from, n); } -#ifdef INLINE_COPY_FROM_USER +#ifdef INLINE_COPY_USER __rust_helper unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n) { diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 6438385e4322e..0f7b43e81030c 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -614,7 +614,7 @@ where /// /// v.reserve(10, GFP_KERNEL)?; /// let cap = v.capacity(); - /// assert!(cap >= 10); + /// assert!(cap >= v.len() + 10); /// /// v.reserve(10, GFP_KERNEL)?; /// let new_cap = v.capacity(); @@ -866,7 +866,7 @@ impl Vec { spare[n - 1].write(value); // SAFETY: - // - `self.len() + n < self.capacity()` due to the call to reserve above, + // - `self.len() + n <= self.capacity()` due to the call to reserve above, // - the loop and the line above initialized the next `n` elements. unsafe { self.inc_len(n) }; diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs index 912cb805caf51..fc97dd8739746 100644 --- a/rust/kernel/block/mq/gen_disk.rs +++ b/rust/kernel/block/mq/gen_disk.rs @@ -150,6 +150,19 @@ impl GenDiskBuilder { // SAFETY: `gendisk` is a valid pointer as we initialized it above unsafe { (*gendisk).fops = &TABLE }; + let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| { + // SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and + // has not been added to the VFS on this cleanup path. + unsafe { bindings::put_disk(gendisk) }; + // SAFETY: `data` came from `into_foreign()` above and has not been + // converted back on this cleanup path. + drop(unsafe { T::QueueData::from_foreign(data) }); + }); + + // The failure guard now owns both pieces of cleanup; the early guard + // must not run on this path anymore. + recover_data.dismiss(); + let mut writer = NullTerminatedFormatter::new( // SAFETY: `gendisk` points to a valid and initialized instance. We // have exclusive access, since the disk is not added to the VFS @@ -172,7 +185,7 @@ impl GenDiskBuilder { }, )?; - recover_data.dismiss(); + cleanup_failure.dismiss(); // INVARIANT: `gendisk` was initialized above. // INVARIANT: `gendisk` was added to the VFS via `device_add_disk` above. @@ -215,6 +228,11 @@ impl Drop for GenDisk { // to the VFS. unsafe { bindings::del_gendisk(self.gendisk) }; + // SAFETY: By type invariant, `self.gendisk` was added to the VFS, so + // `put_disk()` must follow `del_gendisk()` to drop the final gendisk + // reference and trigger the remaining release path. + unsafe { bindings::put_disk(self.gendisk) }; + // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with // a call to `ForeignOwnable::into_foreign` to create `queuedata`. // `ForeignOwnable::from_foreign` is only called here. diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index d8d26870bea2e..a20bd5006f384 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -1323,7 +1323,7 @@ impl Registration { // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; - PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f)) + PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).unwrap_or(0)) } /// Driver's `update_limit` callback. diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 9e5f93aed20cf..2e258d31a45ca 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -122,7 +122,7 @@ struct Inner { /// # Ok(()) /// # } /// ``` -pub struct Devres { +pub struct Devres { dev: ARef, inner: Arc>, } @@ -184,7 +184,7 @@ mod base { } } -impl Devres { +impl Devres { /// Creates a new [`Devres`] instance of the given `data`. /// /// The `data` encapsulated within the returned `Devres` instance' `data` will be @@ -349,7 +349,7 @@ unsafe impl Send for Devres {} // SAFETY: `Devres` can be shared with any task, if `T: Sync`. unsafe impl Sync for Devres {} -impl Drop for Devres { +impl Drop for Devres { fn drop(&mut self) { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index ae78676c927ff..3ce21482b0797 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -153,7 +153,7 @@ pub struct Bar { } impl Bar { - pub(super) fn new(pdev: &Device, num: u32, name: &CStr) -> Result { + pub(super) fn new(pdev: &Device, num: u32, name: &'static CStr) -> Result { let len = pdev.resource_len(num)?; if len == 0 { return Err(ENOMEM); @@ -252,7 +252,7 @@ impl Device { pub fn iomap_region_sized<'a, const SIZE: usize>( &'a self, bar: u32, - name: &'a CStr, + name: &'static CStr, ) -> impl PinInit>, Error> + 'a { Devres::new(self.as_ref(), Bar::::new(self, bar, name)) } @@ -261,7 +261,7 @@ impl Device { pub fn iomap_region<'a>( &'a self, bar: u32, - name: &'a CStr, + name: &'static CStr, ) -> impl PinInit, Error> + 'a { self.iomap_region_sized::<0>(bar, name) } diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 8311d91549e15..4517c1bc547af 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -3,14 +3,27 @@ //! String representations. use crate::{ - alloc::{flags::*, AllocError, KVec}, - error::{to_result, Result}, - fmt::{self, Write}, - prelude::*, + alloc::{ + AllocError, + KVec, // + }, + error::{ + to_result, + Result, // + }, + fmt::{ + self, + Write, // + }, + prelude::*, // }; use core::{ marker::PhantomData, - ops::{Deref, DerefMut, Index}, + ops::{ + Deref, + DerefMut, + Index, // + }, // }; pub use crate::prelude::CStr; diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index 775838a23d935..7f84367ad3e58 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -120,6 +120,9 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) addr.end = promote ? node1_end_addr : node0_end_addr; } + if (addr.start >= addr.end) + goto free_out; + region = damon_new_region(addr.start, addr.end); if (!region) goto free_out; diff --git a/samples/trace_events/trace-events-sample.c b/samples/trace_events/trace-events-sample.c index ecc7db237f2ef..0b7a6efdb247a 100644 --- a/samples/trace_events/trace-events-sample.c +++ b/samples/trace_events/trace-events-sample.c @@ -107,6 +107,10 @@ int foo_bar_reg(void) * for consistency sake, we still take the thread_mutex. */ simple_tsk_fn = kthread_run(simple_thread_fn, NULL, "event-sample-fn"); + if (IS_ERR_OR_NULL(simple_tsk_fn)) { + pr_err("Failed to create simple_thread_fn\n"); + simple_tsk_fn = NULL; + } out: mutex_unlock(&thread_mutex); return 0; diff --git a/scripts/kconfig/conf.c b/scripts/kconfig/conf.c index a7b44cd8ae140..c368bec5ab601 100644 --- a/scripts/kconfig/conf.c +++ b/scripts/kconfig/conf.c @@ -297,9 +297,7 @@ static int conf_askvalue(struct symbol *sym, const char *def) line[1] = 0; if (!sym_is_changeable(sym)) { - printf("%s\n", def); - line[0] = '\n'; - line[1] = 0; + printf("%s\n", def ?: ""); return 0; } @@ -307,7 +305,7 @@ static int conf_askvalue(struct symbol *sym, const char *def) case oldconfig: case syncconfig: if (sym_has_value(sym)) { - printf("%s\n", def); + printf("%s\n", def ?: ""); return 0; } /* fall through */ diff --git a/scripts/livepatch/klp-build b/scripts/livepatch/klp-build index 0ad7e66313142..ab5ea13c87004 100755 --- a/scripts/livepatch/klp-build +++ b/scripts/livepatch/klp-build @@ -306,7 +306,12 @@ set_kernelversion() { stash_file "$file" - kernelrelease="$(cd "$SRC" && make syncconfig &>/dev/null && make -s kernelrelease)" + if [[ -n "$(make -s listnewconfig 2>/dev/null)" ]]; then + die ".config mismatch, check your .config or run 'make olddefconfig'" + fi + make syncconfig &>/dev/null || die "make syncconfig failed" + + kernelrelease="$(make -s kernelrelease)" [[ -z "$kernelrelease" ]] && die "failed to get kernel version" sed -i "2i echo $kernelrelease; exit 0" scripts/setlocalversion @@ -376,15 +381,15 @@ apply_patch() { warn "${patch} applied with fuzz" fi - patch -d "$SRC" -p1 --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" --silent < "$patch" APPLIED_PATCHES+=("$patch") + patch -d "$SRC" -p1 --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" --silent < "$patch" } revert_patch() { local patch="$1" local tmp=() - patch -d "$SRC" -p1 -R --silent --no-backup-if-mismatch -r /dev/null < "$patch" + patch -d "$SRC" -p1 -R --force --no-backup-if-mismatch -r /dev/null &> /dev/null < "$patch" || true for p in "${APPLIED_PATCHES[@]}"; do [[ "$p" == "$patch" ]] && continue @@ -722,13 +727,29 @@ diff_checksums() { ) for func in ${funcs[$file]}; do - diff <( grep0 -E "^DEBUG: .*checksum: $func " "$orig_log" | sed "s|$ORIG_DIR/||") \ - <( grep0 -E "^DEBUG: .*checksum: $func " "$patched_log" | sed "s|$PATCHED_DIR/||") \ - | gawk '/^< DEBUG: / { - gsub(/:/, "") - printf "%s: %s: %s\n", $3, $5, $6 - exit - }' || true + local -a orig patched + paste <(grep0 -E "^DEBUG: .*checksum: $func " "$orig_log") \ + <(grep0 -E "^DEBUG: .*checksum: $func " "$patched_log") | + while IFS= read -r line; do + read -ra orig <<< "${line%%$'\t'*}" + read -ra patched <<< "${line#*$'\t'}" + + if [[ ${#patched[@]} -eq 0 ]]; then + printf "%s: %s: %s (removed)\n" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}" + break + elif [[ ${#orig[@]} -eq 0 ]]; then + printf "%s: %s: %s (added)\n" "${patched[1]%:}" "${patched[3]}" "${patched[-2]}" + break + fi + + [[ "${orig[-1]}" == "${patched[-1]}" ]] && continue + + printf "%s: %s: %s" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}" + [[ "${orig[-2]}" != "${patched[-2]}" ]] && \ + printf " (patched: %s)" "${patched[-2]}" + printf "\n" + break + done || true done done } diff --git a/scripts/rustdoc_test_builder.rs b/scripts/rustdoc_test_builder.rs index f7540bcf595ac..df864437cef70 100644 --- a/scripts/rustdoc_test_builder.rs +++ b/scripts/rustdoc_test_builder.rs @@ -28,7 +28,7 @@ fn main() { // // ``` // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_28_0() { - // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl ::core::fmt::Debug> { + // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl core::fmt::Debug> { // ``` // // It should be unlikely that doctest code matches such lines (when code is formatted properly). @@ -47,12 +47,16 @@ fn main() { }) .expect("No test function found in `rustdoc`'s output."); - // Qualify `Result` to avoid the collision with our own `Result` coming from the prelude. + // Replicate `rustdoc` 1.87+ behaviour [1] by fully qualifying `Result` to avoid the collision + // with our own `Result` coming from the prelude. + // + // [1]: https://github.com/rust-lang/rust/pull/137807 + // + // TODO: Remove this when MSRV is bumped above 1.87. let body = body.replace( - &format!("{rustdoc_function_name}() -> Result<(), impl ::core::fmt::Debug> {{"), - &format!( - "{rustdoc_function_name}() -> ::core::result::Result<(), impl ::core::fmt::Debug> {{" - ), + &format!("{rustdoc_function_name}() -> Result<(), impl core::fmt::Debug> {{"), + // This intentionally does not use absolute paths to match `rustdoc` 1.87 behaviour. + &format!("{rustdoc_function_name}() -> core::result::Result<(), impl core::fmt::Debug> {{"), ); // For tests that get generated with `Result`, like above, `rustdoc` generates an `unwrap()` on diff --git a/security/apparmor/af_unix.c b/security/apparmor/af_unix.c index fdb4a9f212c3b..834a3b1c2f0af 100644 --- a/security/apparmor/af_unix.c +++ b/security/apparmor/af_unix.c @@ -674,9 +674,11 @@ static void update_sk_ctx(struct sock *sk, struct aa_label *label, old = rcu_dereference_protected(ctx->peer, lockdep_is_held(&unix_sk(sk)->lock)); if (old == plabel) { - rcu_assign_pointer(ctx->peer_lastupdate, plabel); + rcu_assign_pointer(ctx->peer_lastupdate, + aa_get_label(plabel)); } else if (aa_label_is_subset(plabel, old)) { - rcu_assign_pointer(ctx->peer_lastupdate, plabel); + rcu_assign_pointer(ctx->peer_lastupdate, + aa_get_label(plabel)); rcu_assign_pointer(ctx->peer, aa_get_label(plabel)); aa_put_label(old); } /* else race or a subset - don't update */ @@ -748,42 +750,47 @@ int aa_unix_file_perm(const struct cred *subj_cred, struct aa_label *label, if (!peer_sk) goto out; - peer_addr = aa_sunaddr(unix_sk(peer_sk), &peer_addrlen); - - struct path peer_path; - - peer_path = unix_sk(peer_sk)->path; - if (!is_sk_fs && is_unix_fs(peer_sk)) { - last_error(error, - unix_fs_perm(op, request, subj_cred, label, - is_unix_fs(peer_sk) ? &peer_path : NULL)); - } else if (!is_sk_fs) { - struct aa_label *plabel; - struct aa_sk_ctx *pctx = aa_sock(peer_sk); - - rcu_read_lock(); - plabel = aa_get_label_rcu(&pctx->label); - rcu_read_unlock(); - /* no fs check of aa_unix_peer_perm because conditions above - * ensure they will never be done - */ - last_error(error, - xcheck(unix_peer_perm(subj_cred, label, op, + if (!is_sk_fs) { + bool is_peer_fs = is_unix_fs(peer_sk); + + peer_addr = aa_sunaddr(unix_sk(peer_sk), &peer_addrlen); + if (is_peer_fs) { + struct path peer_path; + + unix_state_lock(peer_sk); + peer_path = unix_sk(peer_sk)->path; + if (peer_path.dentry) + path_get(&peer_path); + unix_state_unlock(peer_sk); + + last_error(error, + unix_fs_perm(op, request, subj_cred, label, + &peer_path)); + if (peer_path.dentry) + path_put(&peer_path); + } else { + struct aa_sk_ctx *pctx = aa_sock(peer_sk); + + rcu_read_lock(); + plabel = aa_get_newest_label(pctx->label); + rcu_read_unlock(); + /* no fs check of aa_unix_peer_perm because conditions + * above ensure they will never be done + */ + last_error(error, + xcheck(unix_peer_perm(subj_cred, label, op, MAY_READ | MAY_WRITE, sock->sk, is_sk_fs ? &path : NULL, peer_addr, peer_addrlen, - is_unix_fs(peer_sk) ? - &peer_path : NULL, - plabel), - unix_peer_perm(file->f_cred, plabel, op, + NULL, plabel), + unix_peer_perm(file->f_cred, plabel, op, MAY_READ | MAY_WRITE, peer_sk, - is_unix_fs(peer_sk) ? - &peer_path : NULL, - addr, addrlen, + NULL, addr, addrlen, is_sk_fs ? &path : NULL, label))); - if (!error && !__aa_subj_label_is_cached(plabel, label)) - update_peer_ctx(peer_sk, pctx, label); + if (!error && !__aa_subj_label_is_cached(plabel, label)) + update_peer_ctx(peer_sk, pctx, label); + } } sock_put(peer_sk); diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index ededaf46f3caa..95deb867a76b5 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -71,10 +71,10 @@ struct rawdata_f_data { struct aa_loaddata *loaddata; + DECLARE_FLEX_ARRAY(char, data); }; #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY -#define RAWDATA_F_DATA_BUF(p) (char *)(p + 1) static void rawdata_f_data_free(struct rawdata_f_data *private) { @@ -174,6 +174,7 @@ static struct aa_proxy *get_proxy_common_ref(struct aa_common_ref *ref) return NULL; } +#ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY static struct aa_loaddata *get_loaddata_common_ref(struct aa_common_ref *ref) { if (ref) @@ -181,6 +182,7 @@ static struct aa_loaddata *get_loaddata_common_ref(struct aa_common_ref *ref) count)); return NULL; } +#endif static void aa_put_common_ref(struct aa_common_ref *ref) { @@ -1434,7 +1436,7 @@ static ssize_t rawdata_read(struct file *file, char __user *buf, size_t size, struct rawdata_f_data *private = file->private_data; return simple_read_from_buffer(buf, size, ppos, - RAWDATA_F_DATA_BUF(private), + private->data, private->loaddata->size); } @@ -1467,8 +1469,7 @@ static int rawdata_open(struct inode *inode, struct file *file) private->loaddata = loaddata; error = decompress_zstd(loaddata->data, loaddata->compressed_size, - RAWDATA_F_DATA_BUF(private), - loaddata->size); + private->data, loaddata->size); if (error) goto fail_decompress; @@ -1756,6 +1757,80 @@ static const struct inode_operations rawdata_link_abi_iops = { static const struct inode_operations rawdata_link_data_iops = { .get_link = rawdata_get_link_data, }; + +/* + * Requires: @profile->ns->lock held + */ +void __aa_remove_rawdata_symlink_dents(struct aa_profile *profile) +{ + aafs_remove(profile->dents[AAFS_PROF_RAW_HASH]); + profile->dents[AAFS_PROF_RAW_HASH] = NULL; + aafs_remove(profile->dents[AAFS_PROF_RAW_ABI]); + profile->dents[AAFS_PROF_RAW_ABI] = NULL; + aafs_remove(profile->dents[AAFS_PROF_RAW_DATA]); + profile->dents[AAFS_PROF_RAW_DATA] = NULL; +} + +static inline int create_symlink_dent(struct aa_profile *profile, + const char *name, + enum aafs_prof_type type, + const struct inode_operations *iops) +{ + struct dentry *dent = NULL; + struct dentry *dir = prof_dir(profile); + + if (profile->dents[type]) + return 0; + + dent = aafs_create(name, S_IFLNK | 0444, dir, + &profile->label.proxy->count, NULL, NULL, iops); + if (IS_ERR(dent)) + return PTR_ERR(dent); + + profile->dents[type] = dent; + return 0; +} + +/* + * Requires: @profile->ns->lock held + */ +int __aa_create_rawdata_symlink_dents(struct aa_profile *profile) +{ + int error; + + if (!profile || + (profile->dents[AAFS_PROF_RAW_HASH] && + profile->dents[AAFS_PROF_RAW_ABI] && + profile->dents[AAFS_PROF_RAW_DATA])) + return 0; + + if (!profile->rawdata) + return 0; + + if (aa_g_hash_policy) { + error = create_symlink_dent(profile, "raw_sha256", + AAFS_PROF_RAW_HASH, + &rawdata_link_sha256_iops); + if (error) + return error; + } + + error = create_symlink_dent(profile, "raw_abi", + AAFS_PROF_RAW_ABI, + &rawdata_link_abi_iops); + if (error) + return error; + + + error = create_symlink_dent(profile, "raw_data", + AAFS_PROF_RAW_DATA, + &rawdata_link_data_iops); + if (error) + return error; + + return 0; +} + #endif /* CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ /* @@ -1831,31 +1906,9 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent) profile->dents[AAFS_PROF_HASH] = dent; } -#ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY - if (profile->rawdata) { - if (aa_g_hash_policy) { - dent = aafs_create("raw_sha256", S_IFLNK | 0444, dir, - &profile->label.proxy->count, NULL, - NULL, &rawdata_link_sha256_iops); - if (IS_ERR(dent)) - goto fail; - profile->dents[AAFS_PROF_RAW_HASH] = dent; - } - dent = aafs_create("raw_abi", S_IFLNK | 0444, dir, - &profile->label.proxy->count, NULL, NULL, - &rawdata_link_abi_iops); - if (IS_ERR(dent)) - goto fail; - profile->dents[AAFS_PROF_RAW_ABI] = dent; - - dent = aafs_create("raw_data", S_IFLNK | 0444, dir, - &profile->label.proxy->count, NULL, NULL, - &rawdata_link_data_iops); - if (IS_ERR(dent)) - goto fail; - profile->dents[AAFS_PROF_RAW_DATA] = dent; - } -#endif /*CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ + error = __aa_create_rawdata_symlink_dents(profile); + if (error) + goto fail2; list_for_each_entry(child, &profile->base.profiles, base.list) { error = __aafs_profile_mkdir(child, prof_child_dir(profile)); @@ -1922,7 +1975,7 @@ static struct dentry *ns_mkdir_op(struct mnt_idmap *idmap, struct inode *dir, mutex_unlock(&parent->lock); aa_put_ns(parent); - return ERR_PTR(error); + return error ? ERR_PTR(error) : NULL; } static int ns_rmdir_op(struct inode *dir, struct dentry *dentry) diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index f02bf770f6385..c2652165a5886 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1109,6 +1110,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, int count, int flags) { struct aa_profile *profile, *root, *hat = NULL; + struct aa_ns *ns, *new_ns; struct aa_label *new; struct label_it it; bool sibling = false; @@ -1119,6 +1121,32 @@ static struct aa_label *change_hat(const struct cred *subj_cred, AA_BUG(!hats); AA_BUG(count < 1); + /* + * Acquire the newest label and then hold the lock until we choose a + * hat, so that profile replacement doesn't atomically truncate the + * list of potential hats. Because we are getting the namespaces from + * the profiles and label, we can rely on the namespaces being live + * and avoid incrementing their refcounts while grabbing the lock. + */ + label = aa_get_label(label); + ns = labels_ns(label); + +retry: + mutex_lock_nested(&ns->lock, ns->level); + if (label_is_stale(label)) { + new = aa_get_newest_label(label); + new_ns = labels_ns(new); + if (new_ns != ns) { + aa_put_label(new); + mutex_unlock(&ns->lock); + ns = new_ns; + label = new; + goto retry; + } + aa_put_label(label); + label = new; + } + if (PROFILE_IS_HAT(labels_profile(label))) sibling = true; @@ -1127,7 +1155,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, name = hats[i]; label_for_each_in_scope(it, labels_ns(label), label, profile) { if (sibling && PROFILE_IS_HAT(profile)) { - root = aa_get_profile_rcu(&profile->parent); + root = aa_get_profile(profile->parent); } else if (!sibling && !PROFILE_IS_HAT(profile)) { root = aa_get_profile(profile); } else { /* conflicting change type */ @@ -1187,6 +1215,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, GLOBAL_ROOT_UID, info, error); } } + mutex_unlock(&ns->lock); return ERR_PTR(error); build: @@ -1199,7 +1228,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, error = -ENOMEM; goto fail; } /* else if (IS_ERR) build_change_hat has logged error so return new */ - + mutex_unlock(&ns->lock); return new; } @@ -1527,6 +1556,8 @@ int aa_change_profile(const char *fqname, int flags) new = fn_label_build_in_scope(label, profile, GFP_KERNEL, aa_get_label(target), aa_get_label(&profile->label)); + if (IS_ERR_OR_NULL(new)) + goto build_fail; /* * no new privs prevents domain transitions that would * reduce restrictions. @@ -1545,16 +1576,8 @@ int aa_change_profile(const char *fqname, int flags) /* only transition profiles in the current ns */ if (stack) new = aa_label_merge(label, target, GFP_KERNEL); - if (IS_ERR_OR_NULL(new)) { - info = "failed to build target label"; - if (!new) - error = -ENOMEM; - else - error = PTR_ERR(new); - new = NULL; - perms.allow = 0; - goto audit; - } + if (IS_ERR_OR_NULL(new)) + goto build_fail; error = aa_replace_current_label(new); } else { if (new) { @@ -1566,6 +1589,17 @@ int aa_change_profile(const char *fqname, int flags) aa_set_current_onexec(target, stack); } + goto audit; + +build_fail: + info = "failed to build target label"; + if (!new) + error = -ENOMEM; + else + error = PTR_ERR(new); + new = NULL; + perms.allow = 0; + audit: error = fn_for_each_in_scope(label, profile, aa_audit_file(subj_cred, diff --git a/security/apparmor/file.c b/security/apparmor/file.c index 694e157149e85..c9d55fe1086f9 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -157,10 +157,10 @@ static int path_name(const char *op, const struct cred *subj_cred, /* don't reaudit files closed during inheritance */ if (unlikely(path->dentry == aa_null.dentry)) - error = -EACCES; - else - error = aa_path_name(path, flags, buffer, name, &info, - labels_profile(label)->disconnected); + return -EACCES; + + error = aa_path_name(path, flags, buffer, name, &info, + labels_profile(label)->disconnected); if (error) { fn_for_each_confined(label, profile, aa_audit_file(subj_cred, @@ -250,7 +250,7 @@ static int profile_path_perm(const char *op, const struct cred *subj_cred, struct path_cond *cond, int flags, struct aa_perms *perms) { - const char *name; + const char *name = NULL; int error; if (profile_unconfined(profile)) @@ -328,7 +328,7 @@ static int profile_path_link(const struct cred *subj_cred, struct path_cond *cond) { struct aa_ruleset *rules = profile->label.rules[0]; - const char *lname, *tname = NULL; + const char *lname = NULL, *tname = NULL; struct aa_perms lperms = {}, perms; const char *info = NULL; u32 request = AA_MAY_LINK; diff --git a/security/apparmor/include/apparmorfs.h b/security/apparmor/include/apparmorfs.h index dd580594dfb70..33243d11fd106 100644 --- a/security/apparmor/include/apparmorfs.h +++ b/security/apparmor/include/apparmorfs.h @@ -120,6 +120,8 @@ struct aa_loaddata; #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY void __aa_fs_remove_rawdata(struct aa_loaddata *rawdata); int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata); +void __aa_remove_rawdata_symlink_dents(struct aa_profile *profile); +int __aa_create_rawdata_symlink_dents(struct aa_profile *profile); #else static inline void __aa_fs_remove_rawdata(struct aa_loaddata *rawdata) { @@ -131,6 +133,16 @@ static inline int __aa_fs_create_rawdata(struct aa_ns *ns, { return 0; } + +static inline void __aa_remove_rawdata_symlink_dents(struct aa_profile *profile) +{ + /* empty stub */ +} + +static inline int __aa_create_rawdata_symlink_dents(struct aa_profile *profile) +{ + return 0; +} #endif /* CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ #endif /* __AA_APPARMORFS_H */ diff --git a/security/apparmor/include/policy_unpack.h b/security/apparmor/include/policy_unpack.h index e5a95dc4da1f7..b9de0fdf9ee52 100644 --- a/security/apparmor/include/policy_unpack.h +++ b/security/apparmor/include/policy_unpack.h @@ -163,6 +163,25 @@ aa_get_profile_loaddata(struct aa_loaddata *data) return data; } +/** + * aa_get_profile_loaddata_not0 - get a profile reference count if not zero + * @data: reference to get a count on + * + * Like aa_get_profile_loaddata(), but safe to call on an entry that may + * be on a list (e.g. ns->rawdata_list) where the last pcount has already + * dropped and the deferred cleanup has not yet run. + * + * Returns: pointer to reference, or %NULL if @data is NULL or its + * profile refcount has already reached zero. + */ +static inline struct aa_loaddata * +aa_get_profile_loaddata_not0(struct aa_loaddata *data) +{ + if (data && kref_get_unless_zero(&data->pcount)) + return data; + return NULL; +} + void __aa_loaddata_update(struct aa_loaddata *data, long revision); bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r); void aa_loaddata_kref(struct kref *kref); diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 3a721fdf18339..c6a96355e8d9e 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -458,7 +458,7 @@ struct aa_label *aa_label_alloc(int size, struct aa_proxy *proxy, gfp_t gfp) return new; fail: - kfree(new); + aa_label_free(new); return NULL; } diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 3491e9f601943..1f2bbc3175d20 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -1422,7 +1422,21 @@ static int aa_sock_msg_perm(const char *op, u32 request, struct socket *sock, static int apparmor_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) { - return aa_sock_msg_perm(OP_SENDMSG, AA_MAY_SEND, sock, msg, size); + int error = aa_sock_msg_perm(OP_SENDMSG, AA_MAY_SEND, sock, msg, size); + + if (error) + return error; + + /* TCP fast open carries connect() semantics in sendmsg(); mediate + * the implicit connect so it cannot bypass the connect permission. + */ + if ((msg->msg_flags & MSG_FASTOPEN) && msg->msg_name && + (sk_is_tcp(sock->sk) || + (sk_is_inet(sock->sk) && sock->sk->sk_type == SOCK_STREAM && + sock->sk->sk_protocol == IPPROTO_MPTCP))) + error = aa_sk_perm(OP_CONNECT, AA_MAY_CONNECT, sock->sk); + + return error; } static int apparmor_socket_recvmsg(struct socket *sock, @@ -2129,7 +2143,7 @@ static int param_set_mode(const char *val, const struct kernel_param *kp) */ static void cache_hold_inc(unsigned int *hold) { - if (*hold > MAX_HOLD_COUNT) + if (*hold < MAX_HOLD_COUNT) (*hold)++; } diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 44c04102062f3..cf590dd08540c 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -22,12 +22,14 @@ struct aa_sfs_entry aa_sfs_entry_network[] = { AA_SFS_FILE_STRING("af_mask", AA_SFS_AF_MASK), + AA_SFS_FILE_BOOLEAN("tcp-fast-open", 1), { } }; struct aa_sfs_entry aa_sfs_entry_networkv9[] = { AA_SFS_FILE_STRING("af_mask", AA_SFS_AF_MASK), AA_SFS_FILE_BOOLEAN("af_unix", 1), + AA_SFS_FILE_BOOLEAN("tcp-fast-open", 1), { } }; @@ -354,6 +356,7 @@ static int apparmor_secmark_init(struct aa_secmark *secmark) return PTR_ERR(label); secmark->secid = label->secid; + aa_put_label(label); return 0; } diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index b6a5eb4021dbd..567f6a1c2d659 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -1223,8 +1223,12 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, if (aa_rawdata_eq(rawdata_ent, udata)) { struct aa_loaddata *tmp; - tmp = aa_get_profile_loaddata(rawdata_ent); - /* check we didn't fail the race */ + /* + * Entries remain on rawdata_list with + * pcount == 0 until do_ploaddata_rmfs() + * runs; only take a live profile ref. + */ + tmp = aa_get_profile_loaddata_not0(rawdata_ent); if (tmp) { aa_put_profile_loaddata(udata); udata = tmp; @@ -1342,6 +1346,16 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, goto skip; } + if (!aa_g_export_binary) { + if (ent->old && ent->old->rawdata && + ent->old->dents[AAFS_LOADDATA_DIR]) { + /* remove rawdata symlinks because the symlink + * target will be removed + */ + __aa_remove_rawdata_symlink_dents(ent->old); + } + } + /* * TODO: finer dedup based on profile range in data. Load set * can differ but profile may remain unchanged @@ -1352,6 +1366,11 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, if (ent->old) { share_name(ent->old, ent->new); __replace_profile(ent->old, ent->new); + if (aa_g_export_binary) { + /* recreate rawdata symlinks */ + if (!ent->old->rawdata) + __aa_create_rawdata_symlink_dents(ent->new); + } } else { struct list_head *lh; @@ -1372,12 +1391,15 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, out: aa_put_ns(ns); + + ssize_t udata_sz = udata->size; + aa_put_profile_loaddata(udata); kfree(ns_name); if (error) return error; - return udata->size; + return udata_sz; fail_lock: mutex_unlock(&ns->lock); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 9f45d5513d2ca..d9dcff167c480 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -1045,7 +1045,7 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, } /* accept2 is in some cases being allocated, even with perms */ - if (pdb->perms && !pdb->dfa->tables[YYTD_ID_ACCEPT2]) { + if (pdb->dfa && pdb->perms && !pdb->dfa->tables[YYTD_ID_ACCEPT2]) { /* add dfa flags table missing in v2 */ u32 noents = pdb->dfa->tables[YYTD_ID_ACCEPT]->td_lolen; u16 tdflags = pdb->dfa->tables[YYTD_ID_ACCEPT]->td_flags; @@ -1054,7 +1054,8 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, pdb->dfa->tables[YYTD_ID_ACCEPT2] = kvzalloc(tsize, GFP_KERNEL); if (!pdb->dfa->tables[YYTD_ID_ACCEPT2]) { *info = "failed to alloc dfa flags table"; - goto out; + error = -ENOMEM; + goto fail; } pdb->dfa->tables[YYTD_ID_ACCEPT2]->td_lolen = noents; pdb->dfa->tables[YYTD_ID_ACCEPT2]->td_flags = tdflags; @@ -1079,7 +1080,6 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, * - move free of unneeded trans table here, has to be done * after perm mapping. */ -out: *policy = pdb; return 0; diff --git a/security/apparmor/procattr.c b/security/apparmor/procattr.c index ce40f15d4952d..c07b6e8fd9c93 100644 --- a/security/apparmor/procattr.c +++ b/security/apparmor/procattr.c @@ -54,6 +54,8 @@ int aa_getprocattr(struct aa_label *label, char **string, bool newline) FLAG_SHOW_MODE | FLAG_VIEW_SUBNS | FLAG_HIDDEN_UNCONFINED); if (len < 0) { + kfree(*string); + *string = NULL; aa_put_ns(current_ns); return len; } diff --git a/security/apparmor/task.c b/security/apparmor/task.c index 0db0e81b46001..b9fb3738124e0 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -314,7 +314,7 @@ static const char *get_current_exe_path(char *buffer, int buffer_size) path_get(&p); if (aa_path_name(&p, FLAG_VIEW_SUBNS, buffer, &path_str, NULL, NULL)) - return ERR_PTR(-ENOMEM); + path_str = ERR_PTR(-ENOMEM); fput(exe_file); path_put(&p); diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c index 40efde233f3ab..7b98f5d1e2be8 100644 --- a/security/bpf/hooks.c +++ b/security/bpf/hooks.c @@ -7,6 +7,8 @@ #include #include +bool bpf_lsm_initialized __ro_after_init; + static struct security_hook_list bpf_lsm_hooks[] __ro_after_init = { #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ LSM_HOOK_INIT(NAME, bpf_lsm_##NAME), @@ -24,6 +26,7 @@ static int __init bpf_lsm_init(void) { security_add_hooks(bpf_lsm_hooks, ARRAY_SIZE(bpf_lsm_hooks), &bpf_lsmid); + bpf_lsm_initialized = true; pr_info("LSM support for eBPF active\n"); return 0; } diff --git a/security/integrity/evm/evm_secfs.c b/security/integrity/evm/evm_secfs.c index acd840461902f..4baf5e23bc97e 100644 --- a/security/integrity/evm/evm_secfs.c +++ b/security/integrity/evm/evm_secfs.c @@ -127,8 +127,8 @@ static ssize_t evm_read_xattrs(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { char *temp; - int offset = 0; - ssize_t rc, size = 0; + size_t offset = 0, size = 0; + ssize_t rc; struct xattr_list *xattr; if (*ppos != 0) @@ -151,16 +151,22 @@ static ssize_t evm_read_xattrs(struct file *filp, char __user *buf, return -ENOMEM; } + temp[size] = '\0'; + + /* + * No truncation possible: size is computed over the same enabled + * xattrs under xattr_list_mutex, so offset never exceeds size. + */ list_for_each_entry(xattr, &evm_config_xattrnames, list) { if (!xattr->enabled) continue; - sprintf(temp + offset, "%s\n", xattr->name); - offset += strlen(xattr->name) + 1; + offset += snprintf(temp + offset, size + 1 - offset, "%s\n", + xattr->name); } mutex_unlock(&xattr_list_mutex); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); + rc = simple_read_from_buffer(buf, count, ppos, temp, offset); kfree(temp); diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index de963b9f36344..2dd231567710a 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -195,8 +195,9 @@ enum hash_algo ima_get_hash_algo(const struct evm_ima_xattr_data *xattr_value, return sig->hash_algo; case EVM_IMA_XATTR_DIGSIG: sig = (typeof(sig))xattr_value; - if (sig->version != 2 || xattr_len <= sizeof(*sig) - || sig->hash_algo >= HASH_ALGO__LAST) + if ((sig->version != 2 && sig->version != 3) || + xattr_len <= sizeof(*sig) || + sig->hash_algo >= HASH_ALGO__LAST) return ima_hash_algo; return sig->hash_algo; case IMA_XATTR_DIGEST_NG: diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index f7f940a769223..b1c010e8eb138 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -1313,7 +1313,8 @@ static bool ima_validate_rule(struct ima_rule_entry *entry) IMA_GID | IMA_EGID | IMA_FGROUP | IMA_DIGSIG_REQUIRED | IMA_PERMIT_DIRECTIO | IMA_MODSIG_ALLOWED | - IMA_CHECK_BLACKLIST | IMA_VALIDATE_ALGOS)) + IMA_CHECK_BLACKLIST | IMA_VALIDATE_ALGOS | + IMA_SIGV3_REQUIRED)) return false; break; diff --git a/security/keys/internal.h b/security/keys/internal.h index 2cffa6dc82557..b7b622bc36a13 100644 --- a/security/keys/internal.h +++ b/security/keys/internal.h @@ -208,6 +208,8 @@ extern struct key *request_key_auth_new(struct key *target, const void *callout_info, size_t callout_len, struct key *dest_keyring); +struct request_key_auth *request_key_auth_get(struct key *authkey); +void request_key_auth_put(struct request_key_auth *rka); extern struct key *key_get_instantiation_authkey(key_serial_t target_id); diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index ef855d69c97a9..d14ace88e529c 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -1197,9 +1197,13 @@ static long keyctl_instantiate_key_common(key_serial_t id, if (!instkey) goto error; - rka = instkey->payload.data[0]; - if (rka->target_key->serial != id) + rka = request_key_auth_get(instkey); + if (!rka) { + ret = -EKEYREVOKED; goto error; + } + if (rka->target_key->serial != id) + goto error_put_rka; /* pull the payload in if one was supplied */ payload = NULL; @@ -1208,7 +1212,7 @@ static long keyctl_instantiate_key_common(key_serial_t id, ret = -ENOMEM; payload = kvmalloc(plen, GFP_KERNEL); if (!payload) - goto error; + goto error_put_rka; ret = -EFAULT; if (!copy_from_iter_full(payload, plen, from)) @@ -1234,6 +1238,8 @@ static long keyctl_instantiate_key_common(key_serial_t id, error2: kvfree_sensitive(payload, plen); +error_put_rka: + request_key_auth_put(rka); error: return ret; } @@ -1358,15 +1364,19 @@ long keyctl_reject_key(key_serial_t id, unsigned timeout, unsigned error, if (!instkey) goto error; - rka = instkey->payload.data[0]; - if (rka->target_key->serial != id) + rka = request_key_auth_get(instkey); + if (!rka) { + ret = -EKEYREVOKED; goto error; + } + if (rka->target_key->serial != id) + goto error_put_rka; /* find the destination keyring if present (which must also be * writable) */ ret = get_instantiation_keyring(ringid, rka, &dest_keyring); if (ret < 0) - goto error; + goto error_put_rka; /* instantiate the key and link it into a keyring */ ret = key_reject_and_link(rka->target_key, timeout, error, @@ -1379,6 +1389,8 @@ long keyctl_reject_key(key_serial_t id, unsigned timeout, unsigned error, if (ret == 0) keyctl_change_reqkey_auth(NULL); +error_put_rka: + request_key_auth_put(rka); error: return ret; } diff --git a/security/keys/keyctl_pkey.c b/security/keys/keyctl_pkey.c index 97bc27bbf0797..ba150ee2d4a37 100644 --- a/security/keys/keyctl_pkey.c +++ b/security/keys/keyctl_pkey.c @@ -138,28 +138,35 @@ static int keyctl_pkey_params_get_2(const struct keyctl_pkey_params __user *_par if (uparams.in_len > info.max_dec_size || uparams.out_len > info.max_enc_size) return -EINVAL; + + params->out_len = info.max_enc_size; break; case KEYCTL_PKEY_DECRYPT: if (uparams.in_len > info.max_enc_size || uparams.out_len > info.max_dec_size) return -EINVAL; + + params->out_len = info.max_dec_size; break; case KEYCTL_PKEY_SIGN: if (uparams.in_len > info.max_data_size || uparams.out_len > info.max_sig_size) return -EINVAL; + + params->out_len = info.max_sig_size; break; case KEYCTL_PKEY_VERIFY: if (uparams.in_len > info.max_data_size || uparams.in2_len > info.max_sig_size) return -EINVAL; + + params->out_len = info.max_sig_size; break; default: BUG(); } params->in_len = uparams.in_len; - params->out_len = uparams.out_len; /* Note: same as in2_len */ return 0; } diff --git a/security/keys/keyring.c b/security/keys/keyring.c index 5a9887d6b7be3..7a2ee0ded7c93 100644 --- a/security/keys/keyring.c +++ b/security/keys/keyring.c @@ -576,7 +576,7 @@ static int keyring_search_iterator(const void *object, void *iterator_data) struct keyring_search_context *ctx = iterator_data; const struct key *key = keyring_ptr_to_key(object); unsigned long kflags = READ_ONCE(key->flags); - short state = READ_ONCE(key->state); + short state = key_read_state(key); kenter("{%d}", key->serial); diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c index a7d7538c1f705..282e09d8fa46c 100644 --- a/security/keys/request_key_auth.c +++ b/security/keys/request_key_auth.c @@ -23,6 +23,7 @@ static void request_key_auth_describe(const struct key *, struct seq_file *); static void request_key_auth_revoke(struct key *); static void request_key_auth_destroy(struct key *); static long request_key_auth_read(const struct key *, char *, size_t); +static void request_key_auth_rcu_disposal(struct rcu_head *); /* * The request-key authorisation key type definition. @@ -115,6 +116,31 @@ static void free_request_key_auth(struct request_key_auth *rka) kfree(rka); } +/* + * Take a reference to the request-key authorisation payload so callers can + * drop authkey->sem before doing operations that may sleep. + */ +struct request_key_auth *request_key_auth_get(struct key *authkey) +{ + struct request_key_auth *rka; + + down_read(&authkey->sem); + rka = dereference_key_locked(authkey); + if (rka && !test_bit(KEY_FLAG_REVOKED, &authkey->flags)) + refcount_inc(&rka->usage); + else + rka = NULL; + up_read(&authkey->sem); + + return rka; +} + +void request_key_auth_put(struct request_key_auth *rka) +{ + if (rka && refcount_dec_and_test(&rka->usage)) + call_rcu(&rka->rcu, request_key_auth_rcu_disposal); +} + /* * Dispose of the request_key_auth record under RCU conditions */ @@ -136,8 +162,10 @@ static void request_key_auth_revoke(struct key *key) struct request_key_auth *rka = dereference_key_locked(key); kenter("{%d}", key->serial); + if (!rka) + return; rcu_assign_keypointer(key, NULL); - call_rcu(&rka->rcu, request_key_auth_rcu_disposal); + request_key_auth_put(rka); } /* @@ -150,7 +178,7 @@ static void request_key_auth_destroy(struct key *key) kenter("{%d}", key->serial); if (rka) { rcu_assign_keypointer(key, NULL); - call_rcu(&rka->rcu, request_key_auth_rcu_disposal); + request_key_auth_put(rka); } } @@ -174,6 +202,7 @@ struct key *request_key_auth_new(struct key *target, const char *op, rka = kzalloc_obj(*rka); if (!rka) goto error; + refcount_set(&rka->usage, 1); rka->callout_info = kmemdup(callout_info, callout_len, GFP_KERNEL); if (!rka->callout_info) goto error_free_rka; diff --git a/security/landlock/domain.c b/security/landlock/domain.c index 06b6bd845060f..04b69b43fbf56 100644 --- a/security/landlock/domain.c +++ b/security/landlock/domain.c @@ -90,11 +90,12 @@ static struct landlock_details *get_current_details(void) return ERR_CAST(buffer); /* - * Create the new details according to the path's length. Do not - * allocate with GFP_KERNEL_ACCOUNT because it is independent from the - * caller. + * Create the new details according to the path's length. Account to + * the calling task's memcg, like the other Landlock per-domain + * allocations, even if it may not control the related size. */ - details = kzalloc_flex(*details, exe_path, path_size); + details = + kzalloc_flex(*details, exe_path, path_size, GFP_KERNEL_ACCOUNT); if (!details) return ERR_PTR(-ENOMEM); diff --git a/security/landlock/domain.h b/security/landlock/domain.h index a9d57db0120dc..35cac8f6daee7 100644 --- a/security/landlock/domain.h +++ b/security/landlock/domain.h @@ -33,10 +33,7 @@ enum landlock_log_status { * Rarely accessed, mainly when logging the first domain's denial. * * The contained pointers are initialized at the domain creation time and never - * changed again. Contrary to most other Landlock object types, this one is - * not allocated with GFP_KERNEL_ACCOUNT because its size may not be under the - * caller's control (e.g. unknown exe_path) and the data is not explicitly - * requested nor used by tasks. + * changed again. */ struct landlock_details { /** diff --git a/security/landlock/fs.c b/security/landlock/fs.c index c1ecfe2390326..664962a416d7a 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -1900,6 +1900,14 @@ static bool control_current_fowner(struct fown_struct *const fown) */ lockdep_assert_held(&fown->lock); + /* + * A process-group or session owner (PIDTYPE_PGID/PIDTYPE_SID) fans the + * signal out to every member at delivery time, so record the domain and + * let hook_file_send_sigiotask() check the live scope per recipient. + */ + if (fown->pid_type != PIDTYPE_PID && fown->pid_type != PIDTYPE_TGID) + return true; + /* * Some callers (e.g. fcntl_dirnotify) may not be in an RCU read-side * critical section. @@ -1916,6 +1924,7 @@ static void hook_file_set_fowner(struct file *file) { struct landlock_ruleset *prev_dom; struct landlock_cred_security fown_subject = {}; + struct pid *prev_tg, *fown_tg = NULL; size_t fown_layer = 0; if (control_current_fowner(file_f_owner(file))) { @@ -1928,21 +1937,26 @@ static void hook_file_set_fowner(struct file *file) if (new_subject) { landlock_get_ruleset(new_subject->domain); fown_subject = *new_subject; + fown_tg = get_pid(task_tgid(current)); } } prev_dom = landlock_file(file)->fown_subject.domain; + prev_tg = landlock_file(file)->fown_tg; landlock_file(file)->fown_subject = fown_subject; + landlock_file(file)->fown_tg = fown_tg; #ifdef CONFIG_AUDIT landlock_file(file)->fown_layer = fown_layer; #endif /* CONFIG_AUDIT*/ /* May be called in an RCU read-side critical section. */ landlock_put_ruleset_deferred(prev_dom); + put_pid(prev_tg); } static void hook_file_free_security(struct file *file) { + put_pid(landlock_file(file)->fown_tg); landlock_put_ruleset_deferred(landlock_file(file)->fown_subject.domain); } diff --git a/security/landlock/fs.h b/security/landlock/fs.h index bf9948941f2fb..911b83669e207 100644 --- a/security/landlock/fs.h +++ b/security/landlock/fs.h @@ -78,6 +78,16 @@ struct landlock_file_security { * euid. */ struct landlock_cred_security fown_subject; + /** + * @fown_tg: Thread group of the task that set the file owner, pinned + * while @fown_subject holds a domain. It lets + * hook_file_send_sigiotask() always allow a SIGIO delivered to the + * owner's own process -- e.g. the thread-group leader reached through a + * process-group owner -- matching the same-process exemption of + * hook_task_kill(). NULL when no domain is recorded. Protected by + * file->f_owner->lock, like @fown_subject. + */ + struct pid *fown_tg; }; #ifdef CONFIG_AUDIT diff --git a/security/landlock/net.c b/security/landlock/net.c index c368649985c53..4ee4002a8f567 100644 --- a/security/landlock/net.c +++ b/security/landlock/net.c @@ -46,6 +46,7 @@ static int current_check_access_socket(struct socket *const sock, const int addrlen, access_mask_t access_request) { + unsigned short sock_family; __be16 port; struct layer_access_masks layer_masks = {}; const struct landlock_rule *rule; @@ -66,6 +67,12 @@ static int current_check_access_socket(struct socket *const sock, if (addrlen < offsetofend(typeof(*address), sa_family)) return -EINVAL; + /* + * The socket is not locked, so sk_family can change concurrently due to + * e.g. setsockopt(IPV6_ADDRFORM). + */ + sock_family = READ_ONCE(sock->sk->sk_family); + switch (address->sa_family) { case AF_UNSPEC: if (access_request == LANDLOCK_ACCESS_NET_CONNECT_TCP) { @@ -102,7 +109,7 @@ static int current_check_access_socket(struct socket *const sock, * these checks, but it is safer to return a proper * error and test consistency thanks to kselftest. */ - if (sock->sk->__sk_common.skc_family == AF_INET) { + if (sock_family == AF_INET) { const struct sockaddr_in *const sockaddr = (struct sockaddr_in *)address; @@ -180,7 +187,7 @@ static int current_check_access_socket(struct socket *const sock, * check, but it is safer to return a proper error and test * consistency thanks to kselftest. */ - if (address->sa_family != sock->sk->__sk_common.skc_family && + if (address->sa_family != sock_family && address->sa_family != AF_UNSPEC) return -EINVAL; @@ -198,6 +205,7 @@ static int current_check_access_socket(struct socket *const sock, return 0; audit_net.family = address->sa_family; + audit_net.sk = sock->sk; landlock_log_denial(subject, &(struct landlock_request){ .type = LANDLOCK_REQUEST_NET_ACCESS, diff --git a/security/landlock/task.c b/security/landlock/task.c index 6d46042132ce1..7ddf211f75c37 100644 --- a/security/landlock/task.c +++ b/security/landlock/task.c @@ -411,6 +411,17 @@ static int hook_file_send_sigiotask(struct task_struct *tsk, if (!subject->domain) return 0; + /* + * Always allow delivery to the file owner's own process, including a + * thread-group leader reached through a process-group owner. This + * mirrors hook_task_kill()'s same-process exemption and preserves the + * guarantee of commit 18eb75f3af40 ("landlock: Always allow signals + * between threads of the same process"), which the registration-time + * check cannot honor for a process-group target. + */ + if (task_tgid(tsk) == landlock_file(fown->file)->fown_tg) + return 0; + scoped_guard(rcu) { is_scoped = domain_is_scoped(subject->domain, diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 0f704380a8c81..e810f39291677 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3973,9 +3973,9 @@ static int selinux_file_ioctl_compat(struct file *file, unsigned int cmd, static int default_noexec __ro_after_init; -static int __file_map_prot_check(const struct cred *cred, - const struct file *file, unsigned long prot, - bool shared, bool bf_user_file) +static int __file_map_prot_check(const struct file *file, unsigned long prot, + bool shared, bool mounter_check, + bool bf_user_file) { struct inode *inode = NULL; bool prot_exec = prot & PROT_EXEC; @@ -3988,10 +3988,10 @@ static int __file_map_prot_check(const struct cred *cred, inode = file_inode(file); } - if (default_noexec && prot_exec && + if (!mounter_check && default_noexec && prot_exec && (!file || IS_PRIVATE(inode) || (!shared && prot_write))) { int rc; - u32 sid = cred_sid(cred); + u32 sid = current_sid(); /* * We are making executable an anonymous mapping or a private @@ -4004,6 +4004,8 @@ static int __file_map_prot_check(const struct cred *cred, } if (file) { + const struct cred *cred = mounter_check ? + file->f_cred : current_cred(); /* "read" always possible, "write" only if shared */ u32 av = FILE__READ; if (shared && prot_write) @@ -4017,11 +4019,11 @@ static int __file_map_prot_check(const struct cred *cred, return 0; } -static inline int file_map_prot_check(const struct cred *cred, - const struct file *file, - unsigned long prot, bool shared) +static inline int file_map_prot_check(const struct file *file, + unsigned long prot, bool shared, + bool mounter_check) { - return __file_map_prot_check(cred, file, prot, shared, false); + return __file_map_prot_check(file, prot, shared, mounter_check, false); } static int selinux_mmap_addr(unsigned long addr) @@ -4037,12 +4039,14 @@ static int selinux_mmap_addr(unsigned long addr) return rc; } -static int selinux_mmap_file_common(const struct cred *cred, struct file *file, - unsigned long prot, bool shared) +static int selinux_mmap_file_common(struct file *file, unsigned long prot, + bool shared, bool mounter_check) { if (file) { int rc; struct common_audit_data ad; + const struct cred *cred = mounter_check ? + file->f_cred : current_cred(); ad.type = LSM_AUDIT_DATA_FILE; ad.u.file = file; @@ -4051,15 +4055,16 @@ static int selinux_mmap_file_common(const struct cred *cred, struct file *file, return rc; } - return file_map_prot_check(cred, file, prot, shared); + return file_map_prot_check(file, prot, shared, mounter_check); } static int selinux_mmap_file(struct file *file, unsigned long reqprot __always_unused, unsigned long prot, unsigned long flags) { - return selinux_mmap_file_common(current_cred(), file, prot, - (flags & MAP_TYPE) == MAP_SHARED); + return selinux_mmap_file_common(file, prot, + (flags & MAP_TYPE) == MAP_SHARED, + false); } /** @@ -4091,8 +4096,9 @@ static int selinux_mmap_backing_file(struct vm_area_struct *vma, if (vma->vm_flags & VM_EXEC) prot |= PROT_EXEC; - return selinux_mmap_file_common(backing_file->f_cred, backing_file, - prot, vma->vm_flags & VM_SHARED); + return selinux_mmap_file_common(backing_file, prot, + vma->vm_flags & VM_SHARED, + true); } static int selinux_file_mprotect(struct vm_area_struct *vma, @@ -4153,11 +4159,11 @@ static int selinux_file_mprotect(struct vm_area_struct *vma, } } - rc = __file_map_prot_check(cred, file, prot, shared, backing_file); + rc = __file_map_prot_check(file, prot, shared, false, backing_file); if (rc) return rc; if (backing_file) { - rc = file_map_prot_check(file->f_cred, file, prot, shared); + rc = file_map_prot_check(file, prot, shared, true); if (rc) return rc; } @@ -4998,9 +5004,8 @@ static int selinux_socket_socketpair(struct socket *socka, Need to determine whether we should perform a name_bind permission check between the socket and the port number. */ -static int selinux_socket_bind(struct socket *sock, struct sockaddr *address, int addrlen) +static int __selinux_socket_bind(struct sock *sk, struct sockaddr *address, int addrlen) { - struct sock *sk = sock->sk; struct sk_security_struct *sksec = selinux_sock(sk); u16 family; int err; @@ -5130,13 +5135,17 @@ static int selinux_socket_bind(struct socket *sock, struct sockaddr *address, in return -EAFNOSUPPORT; } +static int selinux_socket_bind(struct socket *sock, struct sockaddr *address, int addrlen) +{ + return __selinux_socket_bind(sock->sk, address, addrlen); +} + /* This supports connect(2) and SCTP connect services such as sctp_connectx(3) * and sctp_sendmsg(3) as described in Documentation/security/SCTP.rst */ -static int selinux_socket_connect_helper(struct socket *sock, +static int selinux_socket_connect_helper(struct sock *sk, struct sockaddr *address, int addrlen) { - struct sock *sk = sock->sk; struct sk_security_struct *sksec = selinux_sock(sk); int err; @@ -5225,7 +5234,7 @@ static int selinux_socket_connect(struct socket *sock, int err; struct sock *sk = sock->sk; - err = selinux_socket_connect_helper(sock, address, addrlen); + err = selinux_socket_connect_helper(sk, address, addrlen); if (err) return err; @@ -5266,7 +5275,24 @@ static int selinux_socket_accept(struct socket *sock, struct socket *newsock) static int selinux_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) { - return sock_has_perm(sock->sk, SOCKET__WRITE); + int rc; + struct sockaddr *const addr = msg->msg_name; + const int addrlen = msg->msg_namelen; + + rc = sock_has_perm(sock->sk, SOCKET__WRITE); + if (rc) + return rc; + + if (addr && (msg->msg_flags & MSG_FASTOPEN) && + (sk_is_tcp(sock->sk) || + (sk_is_inet(sock->sk) && sock->sk->sk_type == SOCK_STREAM && + sock->sk->sk_protocol == IPPROTO_MPTCP))) { + rc = selinux_socket_connect(sock, addr, addrlen); + if (rc) + return rc; + } + + return 0; } static int selinux_socket_recvmsg(struct socket *sock, struct msghdr *msg, @@ -5710,13 +5736,11 @@ static int selinux_sctp_bind_connect(struct sock *sk, int optname, int len, err = 0, walk_size = 0; void *addr_buf; struct sockaddr *addr; - struct socket *sock; if (!selinux_policycap_extsockclass()) return 0; /* Process one or more addresses that may be IPv4 or IPv6 */ - sock = sk->sk_socket; addr_buf = address; while (walk_size < addrlen) { @@ -5745,14 +5769,14 @@ static int selinux_sctp_bind_connect(struct sock *sk, int optname, case SCTP_PRIMARY_ADDR: case SCTP_SET_PEER_PRIMARY_ADDR: case SCTP_SOCKOPT_BINDX_ADD: - err = selinux_socket_bind(sock, addr, len); + err = __selinux_socket_bind(sk, addr, len); break; /* Connect checks */ case SCTP_SOCKOPT_CONNECTX: case SCTP_PARAM_SET_PRIMARY: case SCTP_PARAM_ADD_IP: case SCTP_SENDMSG_CONNECT: - err = selinux_socket_connect_helper(sock, addr, len); + err = selinux_socket_connect_helper(sk, addr, len); if (err) return err; diff --git a/sound/aoa/fabrics/layout.c b/sound/aoa/fabrics/layout.c index c3ebb6de47891..7bb541577a263 100644 --- a/sound/aoa/fabrics/layout.c +++ b/sound/aoa/fabrics/layout.c @@ -948,6 +948,8 @@ static void layout_attached_codec(struct aoa_codec *codec) if (lineout == 1) ldev->gpio.methods->set_lineout(codec->gpio, 1); ctl = snd_ctl_new1(&lineout_ctl, codec->gpio); + if (!ctl) + return; if (cc->connected & CC_LINEOUT_LABELLED_HEADPHONE) strscpy(ctl->id.name, "Headphone Switch"); ldev->lineout_ctrl = ctl; @@ -961,12 +963,16 @@ static void layout_attached_codec(struct aoa_codec *codec) if (ldev->have_lineout_detect) { ctl = snd_ctl_new1(&lineout_detect_choice, ldev); + if (!ctl) + return; if (cc->connected & CC_LINEOUT_LABELLED_HEADPHONE) strscpy(ctl->id.name, "Headphone Detect Autoswitch"); aoa_snd_ctl_add(ctl); ctl = snd_ctl_new1(&lineout_detected, ldev); + if (!ctl) + return; if (cc->connected & CC_LINEOUT_LABELLED_HEADPHONE) strscpy(ctl->id.name, "Headphone Detected"); diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index fd63d219bf866..ea699491f0c36 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -1083,15 +1083,18 @@ static int snd_compr_task_new(struct snd_compr_stream *stream, struct snd_compr_ file descriptors are allocated before fd_install() */ if (!task->input || !task->input->file || !task->output || !task->output->file) { retval = -EINVAL; - goto cleanup; + goto free_driver_task; } fd_i = get_unused_fd_flags(O_WRONLY|O_CLOEXEC); - if (fd_i < 0) - goto cleanup; + if (fd_i < 0) { + retval = fd_i; + goto free_driver_task; + } fd_o = get_unused_fd_flags(O_RDONLY|O_CLOEXEC); if (fd_o < 0) { + retval = fd_o; put_unused_fd(fd_i); - goto cleanup; + goto free_driver_task; } /* keep dmabuf reference until freed with task free ioctl */ get_dma_buf(task->input); @@ -1103,6 +1106,8 @@ static int snd_compr_task_new(struct snd_compr_stream *stream, struct snd_compr_ list_add_tail(&task->list, &stream->runtime->tasks); stream->runtime->total_tasks++; return 0; +free_driver_task: + stream->ops->task_free(stream, task); cleanup: snd_compr_task_free(task); return retval; diff --git a/sound/core/init.c b/sound/core/init.c index 0c316189e9476..9bff101af6478 100644 --- a/sound/core/init.c +++ b/sound/core/init.c @@ -1139,7 +1139,7 @@ EXPORT_SYMBOL(snd_card_file_remove); * typically around calling control ops. * * The caller needs to pull down the refcount via snd_power_unref() later - * no matter whether the error is returned from this function or not. + * when this function returns 0. * * Return: Zero if successful, or a negative error code. */ @@ -1152,7 +1152,11 @@ int snd_power_ref_and_wait(struct snd_card *card) card->shutdown || snd_power_get_state(card) == SNDRV_CTL_POWER_D0, snd_power_unref(card), snd_power_ref(card)); - return card->shutdown ? -ENODEV : 0; + if (card->shutdown) { + snd_power_unref(card); + return -ENODEV; + } + return 0; } EXPORT_SYMBOL_GPL(snd_power_ref_and_wait); @@ -1169,7 +1173,8 @@ int snd_power_wait(struct snd_card *card) int ret; ret = snd_power_ref_and_wait(card); - snd_power_unref(card); + if (!ret) + snd_power_unref(card); return ret; } EXPORT_SYMBOL(snd_power_wait); diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c index 5313f50f17da5..55ecf87586c4d 100644 --- a/sound/core/pcm_compat.c +++ b/sound/core/pcm_compat.c @@ -293,7 +293,7 @@ static int snd_pcm_ioctl_xferi_compat(struct snd_pcm_substream *substream, return -ENOTTY; if (substream->stream != dir) return -EINVAL; - if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) + if (snd_pcm_get_state(substream) == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (get_user(buf, &data32->buf) || @@ -338,7 +338,7 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, return -ENOTTY; if (substream->stream != dir) return -EINVAL; - if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) + if (snd_pcm_get_state(substream) == SNDRV_PCM_STATE_OPEN) return -EBADFD; ch = substream->runtime->channels; diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 302643c1c1921..aa33441696855 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -3303,10 +3303,9 @@ static int snd_pcm_xferi_frames_ioctl(struct snd_pcm_substream *substream, struct snd_xferi __user *_xferi) { struct snd_xferi xferi; - struct snd_pcm_runtime *runtime = substream->runtime; snd_pcm_sframes_t result; - if (runtime->state == SNDRV_PCM_STATE_OPEN) + if (snd_pcm_get_state(substream) == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (put_user(0, &_xferi->result)) return -EFAULT; @@ -3329,7 +3328,7 @@ static int snd_pcm_xfern_frames_ioctl(struct snd_pcm_substream *substream, void *bufs __free(kfree) = NULL; snd_pcm_sframes_t result; - if (runtime->state == SNDRV_PCM_STATE_OPEN) + if (snd_pcm_get_state(substream) == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (runtime->channels > 128) return -EINVAL; @@ -3392,7 +3391,7 @@ static int snd_pcm_common_ioctl(struct file *file, if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; - if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED) + if (snd_pcm_get_state(substream) == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; res = snd_power_wait(substream->pcm->card); diff --git a/sound/core/seq/oss/seq_oss_event.c b/sound/core/seq/oss/seq_oss_event.c index 76fb81077eef7..122735862044d 100644 --- a/sound/core/seq/oss/seq_oss_event.c +++ b/sound/core/seq/oss/seq_oss_event.c @@ -39,8 +39,10 @@ static int set_echo_event(struct seq_oss_devinfo *dp, union evrec *rec, struct s */ int -snd_seq_oss_process_event(struct seq_oss_devinfo *dp, union evrec *q, struct snd_seq_event *ev) +snd_seq_oss_process_event(struct seq_oss_devinfo *dp, union evrec *q, + struct snd_seq_event *ev, snd_use_lock_t **lockp) { + *lockp = NULL; switch (q->s.code) { case SEQ_EXTENDED: return extended_event(dp, q, ev); @@ -69,7 +71,7 @@ snd_seq_oss_process_event(struct seq_oss_devinfo *dp, union evrec *q, struct snd if (snd_seq_oss_midi_open(dp, q->s.dev, SNDRV_SEQ_OSS_FILE_WRITE)) break; if (snd_seq_oss_midi_filemode(dp, q->s.dev) & SNDRV_SEQ_OSS_FILE_WRITE) - return snd_seq_oss_midi_putc(dp, q->s.dev, q->s.parm1, ev); + return snd_seq_oss_midi_putc(dp, q->s.dev, q->s.parm1, ev, lockp); break; case SEQ_ECHO: diff --git a/sound/core/seq/oss/seq_oss_event.h b/sound/core/seq/oss/seq_oss_event.h index b4f723949a170..a4524e51d0e9d 100644 --- a/sound/core/seq/oss/seq_oss_event.h +++ b/sound/core/seq/oss/seq_oss_event.h @@ -91,7 +91,8 @@ union evrec { #define ev_is_long(ev) ((ev)->s.code >= 128) #define ev_length(ev) ((ev)->s.code >= 128 ? LONG_EVENT_SIZE : SHORT_EVENT_SIZE) -int snd_seq_oss_process_event(struct seq_oss_devinfo *dp, union evrec *q, struct snd_seq_event *ev); +int snd_seq_oss_process_event(struct seq_oss_devinfo *dp, union evrec *q, + struct snd_seq_event *ev, snd_use_lock_t **lockp); int snd_seq_oss_process_timer_event(struct seq_oss_timer *rec, union evrec *q); int snd_seq_oss_event_input(struct snd_seq_event *ev, int direct, void *private_data, int atomic, int hop); diff --git a/sound/core/seq/oss/seq_oss_ioctl.c b/sound/core/seq/oss/seq_oss_ioctl.c index ccf682689ec95..ce7a69d52b308 100644 --- a/sound/core/seq/oss/seq_oss_ioctl.c +++ b/sound/core/seq/oss/seq_oss_ioctl.c @@ -45,14 +45,17 @@ static int snd_seq_oss_oob_user(struct seq_oss_devinfo *dp, void __user *arg) { unsigned char ev[8]; struct snd_seq_event tmpev; + snd_use_lock_t *lock = NULL; if (copy_from_user(ev, arg, 8)) return -EFAULT; memset(&tmpev, 0, sizeof(tmpev)); snd_seq_oss_fill_addr(dp, &tmpev, dp->addr.client, dp->addr.port); tmpev.time.tick = 0; - if (! snd_seq_oss_process_event(dp, (union evrec *)ev, &tmpev)) { + if (!snd_seq_oss_process_event(dp, (union evrec *)ev, &tmpev, &lock)) { snd_seq_oss_dispatch(dp, &tmpev, 0, 0); + if (lock) + snd_use_lock_free(lock); } return 0; } diff --git a/sound/core/seq/oss/seq_oss_midi.c b/sound/core/seq/oss/seq_oss_midi.c index b50a49ca42ff3..70f94df651446 100644 --- a/sound/core/seq/oss/seq_oss_midi.c +++ b/sound/core/seq/oss/seq_oss_midi.c @@ -593,7 +593,8 @@ send_midi_event(struct seq_oss_devinfo *dp, struct snd_seq_event *ev, struct seq * non-zero : invalid - ignored */ int -snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, struct snd_seq_event *ev) +snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, + struct snd_seq_event *ev, snd_use_lock_t **lockp) { struct seq_oss_midi *mdev __free(seq_oss_midi) = get_mididev(dp, dev); @@ -602,6 +603,9 @@ snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, stru return -ENODEV; if (snd_midi_event_encode_byte(mdev->coder, c, ev)) { snd_seq_oss_fill_addr(dp, ev, mdev->client, mdev->port); + /* the caller must release this later */ + *lockp = &mdev->use_lock; + snd_use_lock_use(*lockp); return 0; } return -EINVAL; diff --git a/sound/core/seq/oss/seq_oss_midi.h b/sound/core/seq/oss/seq_oss_midi.h index bcc1683773dfb..4819d4170bf6d 100644 --- a/sound/core/seq/oss/seq_oss_midi.h +++ b/sound/core/seq/oss/seq_oss_midi.h @@ -26,7 +26,7 @@ void snd_seq_oss_midi_open_all(struct seq_oss_devinfo *dp, int file_mode); int snd_seq_oss_midi_close(struct seq_oss_devinfo *dp, int dev); void snd_seq_oss_midi_reset(struct seq_oss_devinfo *dp, int dev); int snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, - struct snd_seq_event *ev); + struct snd_seq_event *ev, snd_use_lock_t **lockp); int snd_seq_oss_midi_input(struct snd_seq_event *ev, int direct, void *private); int snd_seq_oss_midi_filemode(struct seq_oss_devinfo *dp, int dev); int snd_seq_oss_midi_make_info(struct seq_oss_devinfo *dp, int dev, struct midi_info *inf); diff --git a/sound/core/seq/oss/seq_oss_readq.c b/sound/core/seq/oss/seq_oss_readq.c index c880d47711698..06e12707db213 100644 --- a/sound/core/seq/oss/seq_oss_readq.c +++ b/sound/core/seq/oss/seq_oss_readq.c @@ -73,13 +73,17 @@ snd_seq_oss_readq_delete(struct seq_oss_readq *q) void snd_seq_oss_readq_clear(struct seq_oss_readq *q) { - if (q->qlen) { - q->qlen = 0; - q->head = q->tail = 0; + scoped_guard(spinlock_irqsave, &q->lock) { + if (q->qlen) { + q->qlen = 0; + q->head = 0; + q->tail = 0; + } + q->input_time = (unsigned long)-1; } + /* if someone sleeping, wake'em up */ wake_up(&q->midi_sleep); - q->input_time = (unsigned long)-1; } /* @@ -136,11 +140,11 @@ int snd_seq_oss_readq_sysex(struct seq_oss_readq *q, int dev, /* * copy an event to input queue: * return zero if enqueued + * caller must hold lock */ -int -snd_seq_oss_readq_put_event(struct seq_oss_readq *q, union evrec *ev) +static int snd_seq_oss_readq_put_event_locked(struct seq_oss_readq *q, + union evrec *ev) { - guard(spinlock_irqsave)(&q->lock); if (q->qlen >= q->maxlen - 1) return -ENOMEM; @@ -148,12 +152,27 @@ snd_seq_oss_readq_put_event(struct seq_oss_readq *q, union evrec *ev) q->tail = (q->tail + 1) % q->maxlen; q->qlen++; - /* wake up sleeper */ - wake_up(&q->midi_sleep); - return 0; } +/* + * copy an event to input queue: + * return zero if enqueued + */ +int +snd_seq_oss_readq_put_event(struct seq_oss_readq *q, union evrec *ev) +{ + int rc; + + scoped_guard(spinlock_irqsave, &q->lock) { + rc = snd_seq_oss_readq_put_event_locked(q, ev); + if (!rc) + wake_up(&q->midi_sleep); + } + + return rc; +} + /* * pop queue @@ -209,23 +228,31 @@ snd_seq_oss_readq_poll(struct seq_oss_readq *q, struct file *file, poll_table *w int snd_seq_oss_readq_put_timestamp(struct seq_oss_readq *q, unsigned long curt, int seq_mode) { - if (curt != q->input_time) { - union evrec rec; - memset(&rec, 0, sizeof(rec)); - switch (seq_mode) { - case SNDRV_SEQ_OSS_MODE_SYNTH: - rec.echo = (curt << 8) | SEQ_WAIT; - snd_seq_oss_readq_put_event(q, &rec); - break; - case SNDRV_SEQ_OSS_MODE_MUSIC: - rec.t.code = EV_TIMING; - rec.t.cmd = TMR_WAIT_ABS; - rec.t.time = curt; - snd_seq_oss_readq_put_event(q, &rec); - break; + int queued = 0; + + scoped_guard(spinlock_irqsave, &q->lock) { + if (curt != q->input_time) { + union evrec rec; + + memset(&rec, 0, sizeof(rec)); + switch (seq_mode) { + case SNDRV_SEQ_OSS_MODE_SYNTH: + rec.echo = (curt << 8) | SEQ_WAIT; + queued = !snd_seq_oss_readq_put_event_locked(q, &rec); + break; + case SNDRV_SEQ_OSS_MODE_MUSIC: + rec.t.code = EV_TIMING; + rec.t.cmd = TMR_WAIT_ABS; + rec.t.time = curt; + queued = !snd_seq_oss_readq_put_event_locked(q, &rec); + break; + } + q->input_time = curt; } - q->input_time = curt; } + if (queued) + wake_up(&q->midi_sleep); + return 0; } diff --git a/sound/core/seq/oss/seq_oss_rw.c b/sound/core/seq/oss/seq_oss_rw.c index 307ef98c44c7b..111c792bc72ca 100644 --- a/sound/core/seq/oss/seq_oss_rw.c +++ b/sound/core/seq/oss/seq_oss_rw.c @@ -153,6 +153,7 @@ insert_queue(struct seq_oss_devinfo *dp, union evrec *rec, struct file *opt) { int rc = 0; struct snd_seq_event event; + snd_use_lock_t *lock = NULL; /* if this is a timing event, process the current time */ if (snd_seq_oss_process_timer_event(dp->timer, rec)) @@ -164,7 +165,7 @@ insert_queue(struct seq_oss_devinfo *dp, union evrec *rec, struct file *opt) event.type = SNDRV_SEQ_EVENT_NOTEOFF; snd_seq_oss_fill_addr(dp, &event, dp->addr.client, dp->addr.port); - if (snd_seq_oss_process_event(dp, rec, &event)) + if (snd_seq_oss_process_event(dp, rec, &event, &lock)) return 0; /* invalid event - no need to insert queue */ event.time.tick = snd_seq_oss_timer_cur_tick(dp->timer); @@ -173,6 +174,8 @@ insert_queue(struct seq_oss_devinfo *dp, union evrec *rec, struct file *opt) else rc = snd_seq_kernel_client_enqueue(dp->cseq, &event, opt, !is_nonblock_mode(dp->file_mode)); + if (lock) + snd_use_lock_free(lock); return rc; } diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 5719637575a91..9481fa0ca0f12 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -441,6 +441,7 @@ static ssize_t snd_seq_read(struct file *file, char __user *buf, size_t count, memcpy(&tmpev, &cell->event, aligned_size); tmpev.data.ext.len &= ~SNDRV_SEQ_EXT_MASK; + tmpev.data.ext.ptr = NULL; if (copy_to_user(buf, &tmpev, aligned_size)) { err = -EFAULT; break; @@ -537,16 +538,38 @@ static int bounce_error_event(struct snd_seq_client *client, /* set up quoted error */ memset(&bounce_ev, 0, sizeof(bounce_ev)); - bounce_ev.type = SNDRV_SEQ_EVENT_KERNEL_ERROR; - bounce_ev.flags = SNDRV_SEQ_EVENT_LENGTH_FIXED; + + if (client->type == USER_CLIENT) { + /* + * For user clients, send SNDRV_SEQ_EVENT_BOUNCE with the + * original event embedded as variable-length data. This + * avoids exposing data.quote.event (a kernel pointer) to + * userspace. The variable-length path in snd_seq_event_dup() + * copies the event data from data.ext.ptr into chained cells, + * and snd_seq_expand_var_event() copies only the data content + * -- never the pointer -- to userspace. + */ + bounce_ev.type = SNDRV_SEQ_EVENT_BOUNCE; + bounce_ev.flags = SNDRV_SEQ_EVENT_LENGTH_VARIABLE; + bounce_ev.data.ext.len = sizeof(struct snd_seq_event); + bounce_ev.data.ext.ptr = (char *)event; + } else { + /* + * For kernel clients, quote the event pointer directly. + * Kernel consumers can safely dereference the pointer. + */ + bounce_ev.type = SNDRV_SEQ_EVENT_KERNEL_ERROR; + bounce_ev.flags = SNDRV_SEQ_EVENT_LENGTH_FIXED; + bounce_ev.data.quote.origin = event->dest; + bounce_ev.data.quote.event = event; + bounce_ev.data.quote.value = -err; /* use positive value */ + } + bounce_ev.queue = SNDRV_SEQ_QUEUE_DIRECT; bounce_ev.source.client = SNDRV_SEQ_CLIENT_SYSTEM; bounce_ev.source.port = SNDRV_SEQ_PORT_SYSTEM_ANNOUNCE; bounce_ev.dest.client = client->number; bounce_ev.dest.port = event->source.port; - bounce_ev.data.quote.origin = event->dest; - bounce_ev.data.quote.event = event; - bounce_ev.data.quote.value = -err; /* use positive value */ result = snd_seq_deliver_single_event(NULL, &bounce_ev, atomic, hop + 1); if (result < 0) { client->event_lost++; diff --git a/sound/core/seq/seq_fifo.c b/sound/core/seq/seq_fifo.c index ebe1394c18a96..cffa8d4303747 100644 --- a/sound/core/seq/seq_fifo.c +++ b/sound/core/seq/seq_fifo.c @@ -101,13 +101,17 @@ int snd_seq_fifo_event_in(struct snd_seq_fifo *f, struct snd_seq_event *event) { struct snd_seq_event_cell *cell; + struct snd_seq_pool *pool; + bool linked; int err; if (snd_BUG_ON(!f)) return -EINVAL; guard(snd_seq_fifo)(f); - err = snd_seq_event_dup(f->pool, event, &cell, 1, NULL, NULL); /* always non-blocking */ +retry: + pool = READ_ONCE(f->pool); + err = snd_seq_event_dup(pool, event, &cell, 1, NULL, NULL); /* always non-blocking */ if (err < 0) { if ((err == -ENOMEM) || (err == -EAGAIN)) atomic_inc(&f->overflow); @@ -115,14 +119,24 @@ int snd_seq_fifo_event_in(struct snd_seq_fifo *f, } /* append new cells to fifo */ + linked = false; scoped_guard(spinlock_irqsave, &f->lock) { - if (f->tail != NULL) - f->tail->next = cell; - f->tail = cell; - if (f->head == NULL) - f->head = cell; - cell->next = NULL; - f->cells++; + if (cell->pool == f->pool) { + if (f->tail) + f->tail->next = cell; + f->tail = cell; + if (!f->head) + f->head = cell; + cell->next = NULL; + f->cells++; + linked = true; + } + } + + if (!linked) { + /* Retry against the replacement pool after resize publishes it. */ + snd_seq_cell_free(cell); + goto retry; } /* wakeup client */ @@ -194,13 +208,21 @@ int snd_seq_fifo_cell_out(struct snd_seq_fifo *f, void snd_seq_fifo_cell_putback(struct snd_seq_fifo *f, struct snd_seq_event_cell *cell) { + bool linked = false; + if (cell) { - guard(spinlock_irqsave)(&f->lock); - cell->next = f->head; - f->head = cell; - if (!f->tail) - f->tail = cell; - f->cells++; + scoped_guard(spinlock_irqsave, &f->lock) { + if (cell->pool == f->pool) { + cell->next = f->head; + f->head = cell; + if (!f->tail) + f->tail = cell; + f->cells++; + linked = true; + } + } + if (!linked) + snd_seq_cell_free(cell); } } @@ -237,7 +259,7 @@ int snd_seq_fifo_resize(struct snd_seq_fifo *f, int poolsize) oldpool = f->pool; oldhead = f->head; /* exchange pools */ - f->pool = newpool; + WRITE_ONCE(f->pool, newpool); f->head = NULL; f->tail = NULL; f->cells = 0; diff --git a/sound/core/seq/seq_memory.c b/sound/core/seq/seq_memory.c index aaf808316c30d..209b08c2a9408 100644 --- a/sound/core/seq/seq_memory.c +++ b/sound/core/seq/seq_memory.c @@ -211,7 +211,7 @@ int snd_seq_expand_var_event_at(const struct snd_seq_event *event, int count, len -= offset; if (len > count) len = count; - err = expand_var_event(event, offset, count, buf, true); + err = expand_var_event(event, offset, len, buf, true); if (err < 0) return err; return len; @@ -364,7 +364,7 @@ int snd_seq_event_dup(struct snd_seq_pool *pool, struct snd_seq_event *event, size = snd_seq_event_packet_size(event); memcpy(&cell->ump, event, size); #if IS_ENABLED(CONFIG_SND_SEQ_UMP) - if (size < sizeof(cell->event)) + if (size < sizeof(cell->ump)) cell->ump.raw.extra = 0; #endif diff --git a/sound/core/seq/seq_midi.c b/sound/core/seq/seq_midi.c index ca3f5fc309927..2eb12199c92f9 100644 --- a/sound/core/seq/seq_midi.c +++ b/sound/core/seq/seq_midi.c @@ -24,6 +24,7 @@ Possible options for midisynth module: #include #include #include +#include "seq_lock.h" MODULE_AUTHOR("Frank van de Pol , Jaroslav Kysela "); MODULE_DESCRIPTION("Advanced Linux Sound Architecture sequencer MIDI synth."); @@ -42,6 +43,8 @@ struct seq_midisynth { int device; int subdevice; struct snd_rawmidi_file input_rfile; + spinlock_t output_lock; /* protects output_rfile publication */ + snd_use_lock_t output_use_lock; /* in-flight event_input users */ struct snd_rawmidi_file output_rfile; int seq_client; int seq_port; @@ -125,31 +128,42 @@ static int event_process_midi(struct snd_seq_event *ev, int direct, struct seq_midisynth *msynth = private_data; unsigned char msg[10]; /* buffer for constructing midi messages */ struct snd_rawmidi_substream *substream; + int err = 0; int len; if (snd_BUG_ON(!msynth)) return -EINVAL; - substream = msynth->output_rfile.output; - if (substream == NULL) - return -ENODEV; + + scoped_guard(spinlock_irqsave, &msynth->output_lock) { + substream = msynth->output_rfile.output; + if (!substream) + return -ENODEV; + snd_use_lock_use(&msynth->output_use_lock); + } + if (ev->type == SNDRV_SEQ_EVENT_SYSEX) { /* special case, to save space */ if ((ev->flags & SNDRV_SEQ_EVENT_LENGTH_MASK) != SNDRV_SEQ_EVENT_LENGTH_VARIABLE) { /* invalid event */ pr_debug("ALSA: seq_midi: invalid sysex event flags = 0x%x\n", ev->flags); - return 0; + goto out; } snd_seq_dump_var_event(ev, __dump_midi, substream); snd_midi_event_reset_decode(msynth->parser); } else { - if (msynth->parser == NULL) - return -EIO; + if (!msynth->parser) { + err = -EIO; + goto out; + } len = snd_midi_event_decode(msynth->parser, msg, sizeof(msg), ev); if (len < 0) - return 0; + goto out; if (dump_midi(substream, msg, len) < 0) snd_midi_event_reset_decode(msynth->parser); } - return 0; + +out: + snd_use_lock_free(&msynth->output_use_lock); + return err; } @@ -163,6 +177,8 @@ static int snd_seq_midisynth_new(struct seq_midisynth *msynth, msynth->card = card; msynth->device = device; msynth->subdevice = subdevice; + spin_lock_init(&msynth->output_lock); + snd_use_lock_init(&msynth->output_use_lock); return 0; } @@ -215,12 +231,13 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info { int err; struct seq_midisynth *msynth = private_data; + struct snd_rawmidi_file rfile = {}; struct snd_rawmidi_params params; /* open midi port */ err = snd_rawmidi_kernel_open(msynth->rmidi, msynth->subdevice, SNDRV_RAWMIDI_LFLG_OUTPUT, - &msynth->output_rfile); + &rfile); if (err < 0) { pr_debug("ALSA: seq_midi: midi output open failed!!!\n"); return err; @@ -229,12 +246,14 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info params.avail_min = 1; params.buffer_size = output_buffer_size; params.no_active_sensing = 1; - err = snd_rawmidi_output_params(msynth->output_rfile.output, ¶ms); + err = snd_rawmidi_output_params(rfile.output, ¶ms); if (err < 0) { - snd_rawmidi_kernel_release(&msynth->output_rfile); + snd_rawmidi_kernel_release(&rfile); return err; } snd_midi_event_reset_decode(msynth->parser); + scoped_guard(spinlock_irqsave, &msynth->output_lock) + msynth->output_rfile = rfile; return 0; } @@ -242,11 +261,19 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info static int midisynth_unuse(void *private_data, struct snd_seq_port_subscribe *info) { struct seq_midisynth *msynth = private_data; + struct snd_rawmidi_file rfile = {}; - if (snd_BUG_ON(!msynth->output_rfile.output)) + scoped_guard(spinlock_irqsave, &msynth->output_lock) { + rfile = msynth->output_rfile; + msynth->output_rfile = (struct snd_rawmidi_file){}; + } + + if (snd_BUG_ON(!rfile.output)) return -EINVAL; - snd_rawmidi_drain_output(msynth->output_rfile.output); - return snd_rawmidi_kernel_release(&msynth->output_rfile); + + snd_use_lock_sync(&msynth->output_use_lock); + snd_rawmidi_drain_output(rfile.output); + return snd_rawmidi_kernel_release(&rfile); } /* delete given midi synth port */ diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c index a37a1695f51c7..3f8488716a082 100644 --- a/sound/drivers/aloop.c +++ b/sound/drivers/aloop.c @@ -728,7 +728,6 @@ static void loopback_jiffies_timer_function(struct timer_list *t) if (dpcm->period_update_pending) { dpcm->period_update_pending = 0; period_elapsed = true; - break; } } } diff --git a/sound/firewire/isight.c b/sound/firewire/isight.c index 2b7f071d593b9..33c9dd48b3b0b 100644 --- a/sound/firewire/isight.c +++ b/sound/firewire/isight.c @@ -179,7 +179,8 @@ static void isight_packet(struct fw_iso_context *context, u32 cycle, if (likely(length >= 16 && payload->signature == cpu_to_be32(0x73676874/*"sght"*/))) { count = be32_to_cpu(payload->sample_count); - if (likely(count <= (length - 16) / 4)) { + if (likely(count <= (length - 16) / 4 && + count <= MAX_FRAMES_PER_PACKET)) { total = be32_to_cpu(payload->sample_total); if (unlikely(total != isight->total_samples)) { if (!isight->first_packet) diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index e3b6aaabe3a9c..9c6142ab57214 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -162,9 +162,6 @@ static void cx_fixup_headset_recog(struct hda_codec *codec) { unsigned int mic_present; - /* fix some headset type recognize fail issue, such as EDIFIER headset */ - /* set micbias output current comparator threshold from 66% to 55%. */ - snd_hda_codec_write(codec, 0x1c, 0, 0x320, 0x010); /* set OFF voltage for DFET from -1.2V to -0.8V, set headset micbias register * value adjustment trim from 2.2K ohms to 2.0K ohms. */ diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c index f20d1715da62c..1f4d646724edc 100644 --- a/sound/hda/codecs/hdmi/hdmi.c +++ b/sound/hda/codecs/hdmi/hdmi.c @@ -1549,6 +1549,7 @@ static const struct snd_pci_quirk force_connect_list[] = { SND_PCI_QUIRK(0x103c, 0x83e2, "HP EliteDesk 800 G4", 1), SND_PCI_QUIRK(0x103c, 0x83ef, "HP MP9 G4 Retail System AMS", 1), SND_PCI_QUIRK(0x103c, 0x845a, "HP EliteDesk 800 G4 DM 65W", 1), + SND_PCI_QUIRK(0x103c, 0x8595, "HP EliteDesk 800 G5 Mini", 1), SND_PCI_QUIRK(0x103c, 0x83f3, "HP ProDesk 400", 1), SND_PCI_QUIRK(0x103c, 0x870f, "HP", 1), SND_PCI_QUIRK(0x103c, 0x871a, "HP", 1), @@ -2285,6 +2286,7 @@ EXPORT_SYMBOL_NS_GPL(snd_hda_hdmi_acomp_init, "SND_HDA_CODEC_HDMI"); enum { MODEL_GENERIC, MODEL_GF, + MODEL_LOONGSON, }; static int generichdmi_probe(struct hda_codec *codec, @@ -2302,6 +2304,11 @@ static int generichdmi_probe(struct hda_codec *codec, if (id->driver_data == MODEL_GF) codec->no_sticky_stream = 1; + if (id->driver_data == MODEL_LOONGSON) { + if (codec->bus && codec->bus->pci->revision == 0x2) + codec->eld_jack_detect = 1; /* Jack-detection by ELD */ + } + return 0; } @@ -2319,7 +2326,7 @@ static const struct hda_codec_ops generichdmi_codec_ops = { /* */ static const struct hda_device_id snd_hda_id_generichdmi[] = { - HDA_CODEC_ID_MODEL(0x00147a47, "Loongson HDMI", MODEL_GENERIC), + HDA_CODEC_ID_MODEL(0x00147a47, "Loongson HDMI", MODEL_LOONGSON), HDA_CODEC_ID_MODEL(0x10951390, "SiI1390 HDMI", MODEL_GENERIC), HDA_CODEC_ID_MODEL(0x10951392, "SiI1392 HDMI", MODEL_GENERIC), HDA_CODEC_ID_MODEL(0x11069f84, "VX11 HDMI/DP", MODEL_GENERIC), diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index dcbc669842e05..5b385a82e7dc4 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -4076,6 +4076,7 @@ enum { ALC287_FIXUP_YOGA7_14ITL_SPEAKERS, ALC298_FIXUP_LENOVO_C940_DUET7, ALC287_FIXUP_LENOVO_YOGA_BOOK_9I, + ALC287_FIXUP_LENOVO_YOGA_PRO7, ALC287_FIXUP_13S_GEN2_SPEAKERS, ALC256_FIXUP_SET_COEF_DEFAULTS, ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE, @@ -6101,6 +6102,13 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC285_FIXUP_THINKPAD_HEADSET_JACK, }, + [ALC287_FIXUP_LENOVO_YOGA_PRO7] = { + .type = HDA_FIXUP_FUNC, + /* Reuse the DAC routing selected for ThinkPad X1 Gen7 */ + .v.func = alc285_fixup_thinkpad_x1_gen7, + .chained = true, + .chain_id = ALC269_FIXUP_LENOVO_XPAD_ACPI, + }, [ALC623_FIXUP_LENOVO_THINKSTATION_P340] = { .type = HDA_FIXUP_FUNC, .v.func = alc_fixup_no_shutup, @@ -7567,6 +7575,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1558, 0x51b3, "Clevo NS70AU", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x5630, "Clevo NP50RNJS", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x5700, "Clevo X560WN[RST]", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x1558, 0x6480, "Clevo V6xxAW", ALC245_FIXUP_CLEVO_NOISY_MIC), SND_PCI_QUIRK(0x1558, 0x70a1, "Clevo NB70T[HJK]", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x70b3, "Clevo NK70SB", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x70f2, "Clevo NH79EPY", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE), @@ -7722,6 +7731,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { HDA_CODEC_QUIRK(0x17aa, 0x386e, "Legion Y9000X 2022 IAH7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x386e, "Yoga Pro 7 14ARP8", ALC285_FIXUP_SPEAKER2_TO_DAC1), HDA_CODEC_QUIRK(0x17aa, 0x38a8, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */ + HDA_CODEC_QUIRK(0x17aa, 0x38a7, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */ SND_PCI_QUIRK(0x17aa, 0x386f, "Legion Pro 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3870, "Lenovo Yoga 7 14ARB7", ALC287_FIXUP_YOGA7_14ARB7_I2C), SND_PCI_QUIRK(0x17aa, 0x3877, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2), @@ -7769,7 +7779,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38df, "Y990 YG DUAL", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38f9, "Thinkbook 16P Gen5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), SND_PCI_QUIRK(0x17aa, 0x38fa, "Thinkbook 16P Gen5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), - SND_PCI_QUIRK(0x17aa, 0x38fc, "Lenovo Yoga Pro 7 15ASH11", ALC245_FIXUP_BASS_HP_DAC), + SND_PCI_QUIRK(0x17aa, 0x38fc, "Lenovo Yoga Pro 7 15ASH11", ALC287_FIXUP_LENOVO_YOGA_PRO7), SND_PCI_QUIRK(0x17aa, 0x38fd, "ThinkBook plus Gen5 Hybrid", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI), SND_PCI_QUIRK(0x17aa, 0x390d, "Lenovo Yoga Pro 7 14ASP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), @@ -7839,6 +7849,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1d05, 0x300f, "TongFang X6AR5xxY", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x3019, "TongFang X6FR5xxY", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x3031, "TongFang X6AR55xU", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x3034, "TongFang X6xx45xU", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d17, 0x3288, "Haier Boyue G42", ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS), SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE), diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.c b/sound/hda/codecs/side-codecs/cs35l41_hda.c index acfccc848f82d..64a5bd895fd1e 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l41_hda.c @@ -1325,6 +1325,43 @@ static int cs35l41_fw_type_ctl_info(struct snd_kcontrol *kcontrol, struct snd_ct return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(cs35l41_hda_fw_ids), cs35l41_hda_fw_ids); } +static void cs35l41_remove_controls(struct cs35l41_hda *cs35l41) +{ + if (!cs35l41->codec) + return; + + snd_ctl_remove(cs35l41->codec->card, cs35l41->mute_override_ctl); + cs35l41->mute_override_ctl = NULL; + + snd_ctl_remove(cs35l41->codec->card, cs35l41->fw_load_ctl); + cs35l41->fw_load_ctl = NULL; + + snd_ctl_remove(cs35l41->codec->card, cs35l41->fw_type_ctl); + cs35l41->fw_type_ctl = NULL; +} + +static int cs35l41_add_control(struct cs35l41_hda *cs35l41, + struct snd_kcontrol_new *ctl, + struct snd_kcontrol **kctl) +{ + int ret; + + *kctl = snd_ctl_new1(ctl, cs35l41); + if (!*kctl) + return -ENOMEM; + + ret = snd_ctl_add(cs35l41->codec->card, *kctl); + if (ret) { + dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", ctl->name, ret); + *kctl = NULL; + return ret; + } + + dev_dbg(cs35l41->dev, "Added Control %s\n", ctl->name); + + return 0; +} + static int cs35l41_create_controls(struct cs35l41_hda *cs35l41) { char fw_type_ctl_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; @@ -1360,32 +1397,23 @@ static int cs35l41_create_controls(struct cs35l41_hda *cs35l41) scnprintf(mute_override_ctl_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s Forced Mute Status", cs35l41->amp_name); - ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&fw_type_ctl, cs35l41)); - if (ret) { - dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", fw_type_ctl.name, ret); - return ret; - } - - dev_dbg(cs35l41->dev, "Added Control %s\n", fw_type_ctl.name); - - ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&fw_load_ctl, cs35l41)); - if (ret) { - dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", fw_load_ctl.name, ret); - return ret; - } - - dev_dbg(cs35l41->dev, "Added Control %s\n", fw_load_ctl.name); + ret = cs35l41_add_control(cs35l41, &fw_type_ctl, &cs35l41->fw_type_ctl); + if (ret) + goto err; - ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&mute_override_ctl, cs35l41)); - if (ret) { - dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", mute_override_ctl.name, - ret); - return ret; - } + ret = cs35l41_add_control(cs35l41, &fw_load_ctl, &cs35l41->fw_load_ctl); + if (ret) + goto err; - dev_dbg(cs35l41->dev, "Added Control %s\n", mute_override_ctl.name); + ret = cs35l41_add_control(cs35l41, &mute_override_ctl, &cs35l41->mute_override_ctl); + if (ret) + goto err; return 0; + +err: + cs35l41_remove_controls(cs35l41); + return ret; } static bool cs35l41_dsm_supported(acpi_handle handle, unsigned int commands) @@ -1522,6 +1550,10 @@ static void cs35l41_hda_unbind(struct device *dev, struct device *master, void * device_link_remove(&cs35l41->codec->core.dev, cs35l41->dev); unlock_system_sleep(sleep_flags); memset(comp, 0, sizeof(*comp)); + + cs35l41_remove_controls(cs35l41); + cancel_work_sync(&cs35l41->fw_load_work); + cs35l41->codec = NULL; } } @@ -2060,6 +2092,7 @@ void cs35l41_hda_remove(struct device *dev) struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev); component_del(cs35l41->dev, &cs35l41_hda_comp_ops); + cancel_work_sync(&cs35l41->fw_load_work); pm_runtime_get_sync(cs35l41->dev); pm_runtime_dont_use_autosuspend(cs35l41->dev); diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.h b/sound/hda/codecs/side-codecs/cs35l41_hda.h index 7d003c598e93e..56ec07c0bb745 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda.h +++ b/sound/hda/codecs/side-codecs/cs35l41_hda.h @@ -57,6 +57,8 @@ enum control_bus { SPI }; +struct snd_kcontrol; + struct cs35l41_hda { struct device *dev; struct regmap *regmap; @@ -75,6 +77,9 @@ struct cs35l41_hda { int speaker_id; struct mutex fw_mutex; struct work_struct fw_load_work; + struct snd_kcontrol *fw_type_ctl; + struct snd_kcontrol *fw_load_ctl; + struct snd_kcontrol *mute_override_ctl; struct regmap_irq_chip_data *irq_data; bool firmware_running; diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c index 67240ce184e1a..dd1b0cc63ad6c 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c @@ -588,6 +588,9 @@ static void tas2781_hda_unbind(struct device *dev, comp->playback_hook = NULL; } + request_firmware_nowait_cancel(tas_hda->priv->dev, tas_hda->priv, + tasdev_fw_ready); + tas2781_hda_remove_controls(tas_hda); tasdevice_config_info_remove(tas_hda->priv); diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c index 0e4f3553f2738..d243baff95a72 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c @@ -750,6 +750,9 @@ static void tas2781_hda_unbind(struct device *dev, struct device *master, comp->playback_hook = NULL; } + request_firmware_nowait_cancel(tas_priv->dev, tas_priv, + tasdev_fw_ready); + tas2781_hda_remove_controls(tas_hda); tasdevice_config_info_remove(tas_priv); diff --git a/sound/hda/common/jack.c b/sound/hda/common/jack.c index 98ba1c4d5ba4f..e0a5cc38540ba 100644 --- a/sound/hda/common/jack.c +++ b/sound/hda/common/jack.c @@ -58,6 +58,12 @@ static u32 read_pin_sense(struct hda_codec *codec, hda_nid_t nid, int dev_id) AC_VERB_GET_PIN_SENSE, dev_id); if (codec->inv_jack_detect) val ^= AC_PINSENSE_PRESENCE; + if (codec->eld_jack_detect) { + if (val & AC_PINSENSE_ELDV) + val |= AC_PINSENSE_PRESENCE; + else + val &= ~AC_PINSENSE_PRESENCE; + } return val; } diff --git a/sound/hda/controllers/Kconfig b/sound/hda/controllers/Kconfig index 72855f2df4514..5d6a77e68588e 100644 --- a/sound/hda/controllers/Kconfig +++ b/sound/hda/controllers/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config SND_HDA_INTEL tristate "HD Audio PCI" - depends on SND_PCI + depends on PCI select SND_HDA select SND_INTEL_DSP_CONFIG help diff --git a/sound/hda/core/regmap.c b/sound/hda/core/regmap.c index e7b866fc52c15..d6eb17aa9e086 100644 --- a/sound/hda/core/regmap.c +++ b/sound/hda/core/regmap.c @@ -214,7 +214,7 @@ static int hda_reg_read_coef(struct hdac_device *codec, unsigned int reg, err = snd_hdac_exec_verb(codec, verb, 0, NULL); if (err < 0) return err; - verb = (reg & ~0xfffff) | (AC_VERB_GET_COEF_INDEX << 8); + verb = (reg & ~0xfffff) | (AC_VERB_GET_PROC_COEF << 8); return snd_hdac_exec_verb(codec, verb, 0, val); } @@ -232,7 +232,7 @@ static int hda_reg_write_coef(struct hdac_device *codec, unsigned int reg, err = snd_hdac_exec_verb(codec, verb, 0, NULL); if (err < 0) return err; - verb = (reg & ~0xfffff) | (AC_VERB_GET_COEF_INDEX << 8) | + verb = (reg & ~0xfffff) | (AC_VERB_SET_PROC_COEF << 8) | (val & 0xffff); return snd_hdac_exec_verb(codec, verb, 0, NULL); } diff --git a/sound/isa/gus/gus_pcm.c b/sound/isa/gus/gus_pcm.c index a0757e1ede465..08ccb4d80adeb 100644 --- a/sound/isa/gus/gus_pcm.c +++ b/sound/isa/gus/gus_pcm.c @@ -851,6 +851,8 @@ int snd_gf1_pcm_new(struct snd_gus_card *gus, int pcm_dev, int control_index) kctl = snd_ctl_new1(&snd_gf1_pcm_volume_control1, gus); else kctl = snd_ctl_new1(&snd_gf1_pcm_volume_control, gus); + if (!kctl) + return -ENOMEM; kctl->id.index = control_index; err = snd_ctl_add(card, kctl); if (err < 0) diff --git a/sound/pci/cmipci.c b/sound/pci/cmipci.c index cd73b68336394..ff4bfbf94b81b 100644 --- a/sound/pci/cmipci.c +++ b/sound/pci/cmipci.c @@ -2637,16 +2637,22 @@ static int snd_cmipci_mixer_new(struct cmipci *cm, int pcm_spdif_device) } if (cm->can_ac3_hw) { kctl = snd_ctl_new1(&snd_cmipci_spdif_default, cm); + if (!kctl) + return -ENOMEM; kctl->id.device = pcm_spdif_device; err = snd_ctl_add(card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_cmipci_spdif_mask, cm); + if (!kctl) + return -ENOMEM; kctl->id.device = pcm_spdif_device; err = snd_ctl_add(card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_cmipci_spdif_stream, cm); + if (!kctl) + return -ENOMEM; kctl->id.device = pcm_spdif_device; err = snd_ctl_add(card, kctl); if (err < 0) diff --git a/sound/pci/es1938.c b/sound/pci/es1938.c index 280125eff3624..b6dcb721fffce 100644 --- a/sound/pci/es1938.c +++ b/sound/pci/es1938.c @@ -1655,6 +1655,8 @@ static int snd_es1938_mixer(struct es1938 *chip) for (idx = 0; idx < ARRAY_SIZE(snd_es1938_controls); idx++) { struct snd_kcontrol *kctl; kctl = snd_ctl_new1(&snd_es1938_controls[idx], chip); + if (!kctl) + return -ENOMEM; switch (idx) { case 0: chip->master_volume = kctl; diff --git a/sound/pci/ice1712/aureon.c b/sound/pci/ice1712/aureon.c index 1191a2686dfd2..d6abff2978f3d 100644 --- a/sound/pci/ice1712/aureon.c +++ b/sound/pci/ice1712/aureon.c @@ -1891,6 +1891,8 @@ static int aureon_add_controls(struct snd_ice1712 *ice) for (i = 0; i < ARRAY_SIZE(cs8415_controls); i++) { struct snd_kcontrol *kctl; kctl = snd_ctl_new1(&cs8415_controls[i], ice); + if (!kctl) + return -ENOMEM; if (i > 1) kctl->id.device = ice->pcm->device; err = snd_ctl_add(ice->card, kctl); diff --git a/sound/pci/ice1712/ice1712.c b/sound/pci/ice1712/ice1712.c index 1e39b985bef26..4cec56769c0f6 100644 --- a/sound/pci/ice1712/ice1712.c +++ b/sound/pci/ice1712/ice1712.c @@ -2346,21 +2346,29 @@ int snd_ice1712_spdif_build_controls(struct snd_ice1712 *ice) if (snd_BUG_ON(!ice->pcm_pro)) return -EIO; kctl = snd_ctl_new1(&snd_ice1712_spdif_default, ice); + if (!kctl) + return -ENOMEM; kctl->id.device = ice->pcm_pro->device; err = snd_ctl_add(ice->card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_ice1712_spdif_maskc, ice); + if (!kctl) + return -ENOMEM; kctl->id.device = ice->pcm_pro->device; err = snd_ctl_add(ice->card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_ice1712_spdif_maskp, ice); + if (!kctl) + return -ENOMEM; kctl->id.device = ice->pcm_pro->device; err = snd_ctl_add(ice->card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_ice1712_spdif_stream, ice); + if (!kctl) + return -ENOMEM; kctl->id.device = ice->pcm_pro->device; err = snd_ctl_add(ice->card, kctl); if (err < 0) diff --git a/sound/pci/ice1712/ice1724.c b/sound/pci/ice1712/ice1724.c index 65bf48647d089..b16c84983b81a 100644 --- a/sound/pci/ice1712/ice1724.c +++ b/sound/pci/ice1712/ice1724.c @@ -2379,16 +2379,22 @@ static int snd_vt1724_spdif_build_controls(struct snd_ice1712 *ice) return err; kctl = snd_ctl_new1(&snd_vt1724_spdif_default, ice); + if (!kctl) + return -ENOMEM; kctl->id.device = ice->pcm->device; err = snd_ctl_add(ice->card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_vt1724_spdif_maskc, ice); + if (!kctl) + return -ENOMEM; kctl->id.device = ice->pcm->device; err = snd_ctl_add(ice->card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_vt1724_spdif_maskp, ice); + if (!kctl) + return -ENOMEM; kctl->id.device = ice->pcm->device; err = snd_ctl_add(ice->card, kctl); if (err < 0) diff --git a/sound/pci/ymfpci/ymfpci_main.c b/sound/pci/ymfpci/ymfpci_main.c index b9a09568afc9e..2ccb976e68e0b 100644 --- a/sound/pci/ymfpci/ymfpci_main.c +++ b/sound/pci/ymfpci/ymfpci_main.c @@ -1781,16 +1781,22 @@ int snd_ymfpci_mixer(struct snd_ymfpci *chip, int rear_switch) if (snd_BUG_ON(!chip->pcm_spdif)) return -ENXIO; kctl = snd_ctl_new1(&snd_ymfpci_spdif_default, chip); + if (!kctl) + return -ENOMEM; kctl->id.device = chip->pcm_spdif->device; err = snd_ctl_add(chip->card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_ymfpci_spdif_mask, chip); + if (!kctl) + return -ENOMEM; kctl->id.device = chip->pcm_spdif->device; err = snd_ctl_add(chip->card, kctl); if (err < 0) return err; kctl = snd_ctl_new1(&snd_ymfpci_spdif_stream, chip); + if (!kctl) + return -ENOMEM; kctl->id.device = chip->pcm_spdif->device; err = snd_ctl_add(chip->card, kctl); if (err < 0) diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index 09b475c83c496..e8b6819cc4b45 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -303,13 +303,14 @@ static int create_sdw_dailink(struct snd_soc_card *card, static int create_sdw_dailinks(struct snd_soc_card *card, struct snd_soc_dai_link **dai_links, int *be_id, - struct asoc_sdw_dailink *soc_dais, + struct asoc_sdw_dailink *soc_dais, int num_dais, struct snd_soc_codec_conf **codec_conf) { struct device *dev = card->dev; struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card); struct amd_mc_ctx *amd_ctx = (struct amd_mc_ctx *)ctx->private; struct snd_soc_dai_link_component *sdw_platform_component; + int i; int ret; sdw_platform_component = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component), @@ -329,7 +330,7 @@ static int create_sdw_dailinks(struct snd_soc_card *card, } /* generate DAI links by each sdw link */ - while (soc_dais->initialised) { + for (i = 0; i < num_dais && soc_dais->initialised; i++) { int current_be_id = 0; ret = create_sdw_dailink(card, soc_dais, dai_links, @@ -463,7 +464,7 @@ static int soc_card_dai_links_create(struct snd_soc_card *card) /* SDW */ if (sdw_be_num) { ret = create_sdw_dailinks(card, &dai_links, &be_id, - soc_dais, &codec_conf); + soc_dais, num_ends, &codec_conf); if (ret) return ret; } diff --git a/sound/soc/amd/acp/acp-sdw-sof-mach.c b/sound/soc/amd/acp/acp-sdw-sof-mach.c index a0fd8a6f99708..a423853f3a97d 100644 --- a/sound/soc/amd/acp/acp-sdw-sof-mach.c +++ b/sound/soc/amd/acp/acp-sdw-sof-mach.c @@ -220,13 +220,14 @@ static int create_sdw_dailink(struct snd_soc_card *card, static int create_sdw_dailinks(struct snd_soc_card *card, struct snd_soc_dai_link **dai_links, int *be_id, - struct asoc_sdw_dailink *sof_dais, + struct asoc_sdw_dailink *sof_dais, int num_dais, struct snd_soc_codec_conf **codec_conf) { + int i; int ret; /* generate DAI links by each sdw link */ - while (sof_dais->initialised) { + for (i = 0; i < num_dais && sof_dais->initialised; i++) { int current_be_id = 0; ret = create_sdw_dailink(card, sof_dais, dai_links, @@ -334,7 +335,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) /* SDW */ if (sdw_be_num) { ret = create_sdw_dailinks(card, &dai_links, &be_id, - sof_dais, &codec_conf); + sof_dais, num_ends, &codec_conf); if (ret) return ret; } diff --git a/sound/soc/codecs/adau1372.c b/sound/soc/codecs/adau1372.c index d7363f9d53bb3..879afeb81c422 100644 --- a/sound/soc/codecs/adau1372.c +++ b/sound/soc/codecs/adau1372.c @@ -813,6 +813,11 @@ static int adau1372_set_power(struct adau1372 *adau1372, bool enable) if (adau1372->use_pll) { ret = adau1372_enable_pll(adau1372); if (ret) { + if (!adau1372->pd_gpio) + regmap_update_bits(adau1372->regmap, + ADAU1372_REG_CLK_CTRL, + ADAU1372_CLK_CTRL_PLL_EN, + 0); regcache_cache_only(adau1372->regmap, true); if (adau1372->pd_gpio) gpiod_set_value(adau1372->pd_gpio, 1); diff --git a/sound/soc/codecs/aw88261.c b/sound/soc/codecs/aw88261.c index a6805d5405cd1..a6545f5a8ed0d 100644 --- a/sound/soc/codecs/aw88261.c +++ b/sound/soc/codecs/aw88261.c @@ -284,22 +284,22 @@ static void aw88261_reg_force_set(struct aw88261 *aw88261) if (aw88261->frcset_en == AW88261_FRCSET_ENABLE) { /* set FORCE_PWM */ regmap_update_bits(aw88261->regmap, AW88261_BSTCTRL3_REG, - AW88261_FORCE_PWM_MASK, AW88261_FORCE_PWM_FORCEMINUS_PWM_VALUE); + ~AW88261_FORCE_PWM_MASK, AW88261_FORCE_PWM_FORCEMINUS_PWM_VALUE); /* set BOOST_OS_WIDTH */ regmap_update_bits(aw88261->regmap, AW88261_BSTCTRL5_REG, - AW88261_BST_OS_WIDTH_MASK, AW88261_BST_OS_WIDTH_50NS_VALUE); + ~AW88261_BST_OS_WIDTH_MASK, AW88261_BST_OS_WIDTH_50NS_VALUE); /* set BURST_LOOPR */ regmap_update_bits(aw88261->regmap, AW88261_BSTCTRL6_REG, - AW88261_BST_LOOPR_MASK, AW88261_BST_LOOPR_340K_VALUE); + ~AW88261_BST_LOOPR_MASK, AW88261_BST_LOOPR_340K_VALUE); /* set RSQN_DLY */ regmap_update_bits(aw88261->regmap, AW88261_BSTCTRL7_REG, - AW88261_RSQN_DLY_MASK, AW88261_RSQN_DLY_35NS_VALUE); + ~AW88261_RSQN_DLY_MASK, AW88261_RSQN_DLY_35NS_VALUE); /* set BURST_SSMODE */ regmap_update_bits(aw88261->regmap, AW88261_BSTCTRL8_REG, - AW88261_BURST_SSMODE_MASK, AW88261_BURST_SSMODE_FAST_VALUE); + ~AW88261_BURST_SSMODE_MASK, AW88261_BURST_SSMODE_FAST_VALUE); /* set BST_BURST */ regmap_update_bits(aw88261->regmap, AW88261_BSTCTRL9_REG, - AW88261_BST_BURST_MASK, AW88261_BST_BURST_30MA_VALUE); + ~AW88261_BST_BURST_MASK, AW88261_BST_BURST_30MA_VALUE); } else { dev_dbg(aw88261->aw_pa->dev, "needn't set reg value"); } diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c index 795e2764d67ec..9928608296d88 100644 --- a/sound/soc/codecs/cs35l56-shared.c +++ b/sound/soc/codecs/cs35l56-shared.c @@ -534,6 +534,7 @@ static void cs35l56_spi_system_reset(struct cs35l56_base *cs35l56_base) * The regmap must remain in cache-only until the chip has * booted, so use a bypassed read. */ + val = 0; ret = read_poll_timeout(regmap_read_bypassed, read_ret, (val > 0) && (val < 0xffffffff), CS35L56_HALO_STATE_POLL_US, @@ -1259,7 +1260,7 @@ ssize_t cs35l56_cal_data_debugfs_write(struct cs35l56_base *cs35l56_base, return -EMSGSIZE; ret = simple_write_to_buffer(&cal_data, sizeof(cal_data), ppos, from, count); - if (ret) + if (ret < 0) return ret; ret = cs35l56_stash_calibration(cs35l56_base, &cal_data); @@ -1293,6 +1294,7 @@ EXPORT_SYMBOL_NS_GPL(cs35l56_create_cal_debugfs, "SND_SOC_CS35L56_SHARED"); void cs35l56_remove_cal_debugfs(struct cs35l56_base *cs35l56_base) { debugfs_remove_recursive(cs35l56_base->debugfs); + cs35l56_base->debugfs = ERR_PTR(-ENOENT); } EXPORT_SYMBOL_NS_GPL(cs35l56_remove_cal_debugfs, "SND_SOC_CS35L56_SHARED"); diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c index 4fbbdcc871516..033028a4734b5 100644 --- a/sound/soc/codecs/cs35l56.c +++ b/sound/soc/codecs/cs35l56.c @@ -1326,7 +1326,7 @@ VISIBLE_IF_KUNIT int cs35l56_set_fw_name(struct snd_soc_component *component) } EXPORT_SYMBOL_IF_KUNIT(cs35l56_set_fw_name); -static int cs35l56_component_probe(struct snd_soc_component *component) +static int _cs35l56_component_probe(struct snd_soc_component *component) { struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component); struct cs35l56_private *cs35l56 = snd_soc_component_get_drvdata(component); @@ -1426,6 +1426,17 @@ static void cs35l56_component_remove(struct snd_soc_component *component) cs35l56->component = NULL; } +static int cs35l56_component_probe(struct snd_soc_component *component) +{ + int ret; + + ret = _cs35l56_component_probe(component); + if (ret < 0) + cs35l56_component_remove(component); + + return ret; +} + static int cs35l56_set_bias_level(struct snd_soc_component *component, enum snd_soc_bias_level level) { @@ -1480,6 +1491,7 @@ static int __maybe_unused cs35l56_runtime_resume_i2c_spi(struct device *dev) int cs35l56_system_suspend(struct device *dev) { struct cs35l56_private *cs35l56 = dev_get_drvdata(dev); + int ret; dev_dbg(dev, "system_suspend\n"); @@ -1495,7 +1507,11 @@ int cs35l56_system_suspend(struct device *dev) if (cs35l56->base.irq) disable_irq(cs35l56->base.irq); - return pm_runtime_force_suspend(dev); + ret = pm_runtime_force_suspend(dev); + if ((ret < 0) && cs35l56->base.irq) + enable_irq(cs35l56->base.irq); + + return ret; } EXPORT_SYMBOL_GPL(cs35l56_system_suspend); @@ -1964,11 +1980,14 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56) cs35l56_dai, ARRAY_SIZE(cs35l56_dai)); if (ret < 0) { dev_err_probe(cs35l56->base.dev, ret, "Register codec failed\n"); - goto err; + goto err_remove_wm_adsp; } return 0; +err_remove_wm_adsp: + wm_adsp2_remove(&cs35l56->dsp); + err: gpiod_set_value_cansleep(cs35l56->base.reset_gpio, 0); regulator_bulk_disable(ARRAY_SIZE(cs35l56->supplies), cs35l56->supplies); @@ -2076,6 +2095,8 @@ void cs35l56_remove(struct cs35l56_private *cs35l56) destroy_workqueue(cs35l56->dsp_wq); + wm_adsp2_remove(&cs35l56->dsp); + pm_runtime_dont_use_autosuspend(cs35l56->base.dev); pm_runtime_suspend(cs35l56->base.dev); pm_runtime_disable(cs35l56->base.dev); diff --git a/sound/soc/codecs/cs530x.c b/sound/soc/codecs/cs530x.c index 18b5ff75feec4..2c7e331359117 100644 --- a/sound/soc/codecs/cs530x.c +++ b/sound/soc/codecs/cs530x.c @@ -1093,6 +1093,29 @@ static int cs530x_component_probe(struct snd_soc_component *component) return 0; } +static bool cs530x_mclk_freq_is_valid(struct cs530x_priv *cs530x, + unsigned int freq) +{ + /* + * All these chips support 48 kHz- and 44.1 kHz-related sample rates, + * but they differ in what MCLK frequency is required for achieving + * the sample rate. + */ + switch (cs530x->devtype) { + case CS4282: + case CS4302: + case CS4304: + case CS4308: + return freq == 49152000 || freq == 45158400; + case CS5302: + case CS5304: + case CS5308: + return freq == 24576000 || freq == 22579200; + } + + return false; +} + static int cs530x_set_sysclk(struct snd_soc_component *component, int clk_id, int source, unsigned int freq, int dir) { @@ -1101,11 +1124,7 @@ static int cs530x_set_sysclk(struct snd_soc_component *component, int clk_id, switch (source) { case CS530X_SYSCLK_SRC_MCLK: - switch (freq) { - case CS530X_SYSCLK_REF_45_1MHZ: - case CS530X_SYSCLK_REF_49_1MHZ: - break; - default: + if (!cs530x_mclk_freq_is_valid(cs530x, freq)) { dev_err(component->dev, "Invalid MCLK source rate %d\n", freq); return -EINVAL; } diff --git a/sound/soc/codecs/cs530x.h b/sound/soc/codecs/cs530x.h index 1e2f6a7a589c1..18aa4dfd0c860 100644 --- a/sound/soc/codecs/cs530x.h +++ b/sound/soc/codecs/cs530x.h @@ -200,12 +200,6 @@ /* IN_VOL_CTL5 and OUT_VOL_CTL5 */ #define CS530X_INOUT_VU BIT(0) -/* MCLK Reference Source Frequency */ -/* 41KHz related */ -#define CS530X_SYSCLK_REF_45_1MHZ 45158400 -/* 48KHz related */ -#define CS530X_SYSCLK_REF_49_1MHZ 49152000 - /* System Clock Source */ #define CS530X_SYSCLK_SRC_MCLK 0 #define CS530X_SYSCLK_SRC_PLL 1 diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c index 2652fcf2a3a34..3220f9226e0b2 100644 --- a/sound/soc/codecs/hdac_hdmi.c +++ b/sound/soc/codecs/hdac_hdmi.c @@ -911,12 +911,14 @@ static int hdac_hdmi_set_pin_port_mux(struct snd_kcontrol *kcontrol, struct hdac_device *hdev = dev_to_hdac_dev(dev); struct hdac_hdmi_priv *hdmi = hdev_to_hdmi_priv(hdev); struct hdac_hdmi_pcm *pcm; - const char *cvt_name = e->texts[ucontrol->value.enumerated.item[0]]; + const char *cvt_name; ret = snd_soc_dapm_put_enum_double(kcontrol, ucontrol); if (ret < 0) return ret; + cvt_name = e->texts[ucontrol->value.enumerated.item[0]]; + if (port == NULL) return -EINVAL; diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c index 528d5b167ecff..58a5798823d7f 100644 --- a/sound/soc/codecs/lpass-va-macro.c +++ b/sound/soc/codecs/lpass-va-macro.c @@ -244,6 +244,11 @@ static const struct va_macro_data sm8250_va_data = { .version = LPASS_CODEC_VERSION_1_0, }; +static const struct va_macro_data sc7280_va_data = { + .has_swr_master = false, + .has_npl_clk = false, +}; + static const struct va_macro_data sm8450_va_data = { .has_swr_master = true, .has_npl_clk = true, @@ -1755,7 +1760,7 @@ static const struct dev_pm_ops va_macro_pm_ops = { }; static const struct of_device_id va_macro_dt_match[] = { - { .compatible = "qcom,sc7280-lpass-va-macro", .data = &sm8250_va_data }, + { .compatible = "qcom,sc7280-lpass-va-macro", .data = &sc7280_va_data }, { .compatible = "qcom,sm6115-lpass-va-macro", .data = &sm8450_va_data }, { .compatible = "qcom,sm8250-lpass-va-macro", .data = &sm8250_va_data }, { .compatible = "qcom,sm8450-lpass-va-macro", .data = &sm8450_va_data }, diff --git a/sound/soc/codecs/rt5575-spi.c b/sound/soc/codecs/rt5575-spi.c index 9a349965435b6..d5b3a57c88666 100644 --- a/sound/soc/codecs/rt5575-spi.c +++ b/sound/soc/codecs/rt5575-spi.c @@ -17,7 +17,7 @@ struct rt5575_spi_burst_write { u8 cmd; - u32 addr; + __le32 addr; u8 data[RT5575_SPI_BUF_LEN]; u8 dummy; } __packed; diff --git a/sound/soc/codecs/sma1307.c b/sound/soc/codecs/sma1307.c index 5850bf6e71cad..5cc0834873f00 100644 --- a/sound/soc/codecs/sma1307.c +++ b/sound/soc/codecs/sma1307.c @@ -1592,6 +1592,7 @@ static void sma1307_check_fault_worker(struct work_struct *work) struct sma1307_priv *sma1307 = container_of(work, struct sma1307_priv, check_fault_work.work); unsigned int status1_val, status2_val; + char volume[sizeof("VOLUME=0x12345678")]; char *envp[3] = { NULL, NULL, NULL }; if (sma1307->tsdw_cnt) @@ -1607,7 +1608,7 @@ static void sma1307_check_fault_worker(struct work_struct *work) if (~status1_val & SMA1307_OT1_OK_STATUS) { dev_crit(sma1307->dev, "%s: OT1(Over Temperature Level 1)\n", __func__); - envp[0] = kasprintf(GFP_KERNEL, "STATUS=OT1"); + envp[0] = "STATUS=OT1"; if (sma1307->sw_ot1_prot) { /* Volume control (Current Volume -3dB) */ if ((sma1307->cur_vol + 6) <= 0xFA) { @@ -1615,8 +1616,9 @@ static void sma1307_check_fault_worker(struct work_struct *work) regmap_write(sma1307->regmap, SMA1307_0A_SPK_VOL, sma1307->cur_vol); - envp[1] = kasprintf(GFP_KERNEL, - "VOLUME=0x%02X", sma1307->cur_vol); + snprintf(volume, sizeof(volume), + "VOLUME=0x%02X", sma1307->cur_vol); + envp[1] = volume; } } sma1307->tsdw_cnt++; @@ -1625,48 +1627,53 @@ static void sma1307_check_fault_worker(struct work_struct *work) SMA1307_0A_SPK_VOL, sma1307->init_vol); sma1307->tsdw_cnt = 0; sma1307->cur_vol = sma1307->init_vol; - envp[0] = kasprintf(GFP_KERNEL, "STATUS=OT1_CLEAR"); - envp[1] = kasprintf(GFP_KERNEL, - "VOLUME=0x%02X", sma1307->cur_vol); + envp[0] = "STATUS=OT1_CLEAR"; + snprintf(volume, sizeof(volume), "VOLUME=0x%02X", + sma1307->cur_vol); + envp[1] = volume; } if (~status1_val & SMA1307_OT2_OK_STATUS) { dev_crit(sma1307->dev, "%s: OT2(Over Temperature Level 2)\n", __func__); - envp[0] = kasprintf(GFP_KERNEL, "STATUS=OT2"); + envp[0] = "STATUS=OT2"; + envp[1] = NULL; } if (status1_val & SMA1307_UVLO_STATUS) { dev_crit(sma1307->dev, "%s: UVLO(Under Voltage Lock Out)\n", __func__); - envp[0] = kasprintf(GFP_KERNEL, "STATUS=UVLO"); + envp[0] = "STATUS=UVLO"; + envp[1] = NULL; } if (status1_val & SMA1307_OVP_BST_STATUS) { dev_crit(sma1307->dev, "%s: OVP_BST(Over Voltage Protection)\n", __func__); - envp[0] = kasprintf(GFP_KERNEL, "STATUS=OVP_BST"); + envp[0] = "STATUS=OVP_BST"; + envp[1] = NULL; } if (status2_val & SMA1307_OCP_SPK_STATUS) { dev_crit(sma1307->dev, "%s: OCP_SPK(Over Current Protect SPK)\n", __func__); - envp[0] = kasprintf(GFP_KERNEL, "STATUS=OCP_SPK"); + envp[0] = "STATUS=OCP_SPK"; + envp[1] = NULL; } if (status2_val & SMA1307_OCP_BST_STATUS) { dev_crit(sma1307->dev, "%s: OCP_BST(Over Current Protect Boost)\n", __func__); - envp[0] = kasprintf(GFP_KERNEL, "STATUS=OCP_BST"); + envp[0] = "STATUS=OCP_BST"; + envp[1] = NULL; } if (status2_val & SMA1307_CLK_MON_STATUS) { dev_crit(sma1307->dev, "%s: CLK_FAULT(No clock input)\n", __func__); - envp[0] = kasprintf(GFP_KERNEL, "STATUS=CLK_FAULT"); + envp[0] = "STATUS=CLK_FAULT"; + envp[1] = NULL; } if (envp[0] != NULL) { if (kobject_uevent_env(sma1307->kobj, KOBJ_CHANGE, envp)) dev_err(sma1307->dev, "%s: Error sending uevent\n", __func__); - kfree(envp[0]); - kfree(envp[1]); } if (sma1307->check_fault_status) { diff --git a/sound/soc/codecs/tlv320aic3x.c b/sound/soc/codecs/tlv320aic3x.c index ce22298b43efc..f8e847d28258b 100644 --- a/sound/soc/codecs/tlv320aic3x.c +++ b/sound/soc/codecs/tlv320aic3x.c @@ -1049,11 +1049,13 @@ static int aic3x_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) { + static const u8 dual_rate_q[] = {4, 8, 9, 12, 16}; struct snd_soc_component *component = dai->component; struct aic3x_priv *aic3x = snd_soc_component_get_drvdata(component); int codec_clk = 0, bypass_pll = 0, fsref, last_clk = 0; u8 data, j, r, p, pll_q, pll_p = 1, pll_r = 1, pll_j = 1; u16 d, pll_d = 1; + bool dual_rate; int clk; int width = aic3x->slot_width; @@ -1079,14 +1081,25 @@ static int aic3x_hw_params(struct snd_pcm_substream *substream, /* Fsref can be 44100 or 48000 */ fsref = (params_rate(params) % 11025 == 0) ? 44100 : 48000; + dual_rate = params_rate(params) >= 64000; /* Try to find a value for Q which allows us to bypass the PLL and * generate CODEC_CLK directly. */ - for (pll_q = 2; pll_q < 18; pll_q++) - if (aic3x->sysclk / (128 * pll_q) == fsref) { - bypass_pll = 1; - break; + if (dual_rate) { + for (int i = 0; i < ARRAY_SIZE(dual_rate_q); i++) { + pll_q = dual_rate_q[i]; + if (aic3x->sysclk / (128 * pll_q) == fsref) { + bypass_pll = 1; + break; + } } + } else { + for (pll_q = 2; pll_q < 18; pll_q++) + if (aic3x->sysclk / (128 * pll_q) == fsref) { + bypass_pll = 1; + break; + } + } if (bypass_pll) { pll_q &= 0xf; @@ -1106,13 +1119,13 @@ static int aic3x_hw_params(struct snd_pcm_substream *substream, * right DAC to right channel input */ data = (LDAC2LCH | RDAC2RCH); data |= (fsref == 44100) ? FSREF_44100 : FSREF_48000; - if (params_rate(params) >= 64000) + if (dual_rate) data |= DUAL_RATE_MODE; snd_soc_component_write(component, AIC3X_CODEC_DATAPATH_REG, data); /* codec sample rate select */ data = (fsref * 20) / params_rate(params); - if (params_rate(params) < 64000) + if (!dual_rate) data /= 2; data /= 5; data -= 2; diff --git a/sound/soc/codecs/wm_adsp.c b/sound/soc/codecs/wm_adsp.c index ca630c9948e49..baa75e7ff53ba 100644 --- a/sound/soc/codecs/wm_adsp.c +++ b/sound/soc/codecs/wm_adsp.c @@ -1170,7 +1170,14 @@ EXPORT_SYMBOL_GPL(wm_adsp2_component_probe); int wm_adsp2_component_remove(struct wm_adsp *dsp, struct snd_soc_component *component) { + if (!dsp) + return 0; + + if (!dsp->component) + return 0; + cs_dsp_cleanup_debugfs(&dsp->cs_dsp); + dsp->component = NULL; return 0; } diff --git a/sound/soc/fsl/fsl_asrc_dma.c b/sound/soc/fsl/fsl_asrc_dma.c index 5aa96af994c41..38f2b7c63133a 100644 --- a/sound/soc/fsl/fsl_asrc_dma.c +++ b/sound/soc/fsl/fsl_asrc_dma.c @@ -288,6 +288,26 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component, config_be.dst_addr_width = buswidth; config_be.dst_maxburst = dma_params_be->maxburst; + /* + * For eDMA, the back-end may report a maxburst size that is not evenly + * divisible by the channel count. This causes the DMA transfer length + * to misalign with the FIFO boundary, resulting in wrong data and + * audible noise. Align maxburst to the nearest valid boundary: + * - If maxburst >= channel count, override to the channel count so + * each transfer equals exactly one audio frame. + * - If maxburst < channel count, override to 1 to avoid partial-frame + * transfers. + */ + if (asrc->use_edma && (dma_params_be->maxburst % params_channels(params))) { + if (dma_params_be->maxburst >= params_channels(params)) { + config_be.src_maxburst = params_channels(params); + config_be.dst_maxburst = params_channels(params); + } else { + config_be.src_maxburst = 1; + config_be.dst_maxburst = 1; + } + } + memset(&audio_config, 0, sizeof(audio_config)); config_be.peripheral_config = &audio_config; config_be.peripheral_size = sizeof(audio_config); diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c index 40a3b74321744..f819f33ec46b8 100644 --- a/sound/soc/fsl/fsl_audmix.c +++ b/sound/soc/fsl/fsl_audmix.c @@ -117,6 +117,9 @@ static int fsl_audmix_put_mix_clk_src(struct snd_kcontrol *kcontrol, unsigned int *item = ucontrol->value.enumerated.item; unsigned int reg_val, val, mix_clk; + if (item[0] >= e->items) + return -EINVAL; + /* Get current state */ reg_val = snd_soc_component_read(comp, FSL_AUDMIX_CTR); mix_clk = ((reg_val & FSL_AUDMIX_CTR_MIXCLK_MASK) @@ -157,6 +160,9 @@ static int fsl_audmix_put_out_src(struct snd_kcontrol *kcontrol, unsigned int reg_val, val, mask = 0, ctr = 0; int ret; + if (item[0] >= e->items) + return -EINVAL; + /* Get current state */ reg_val = snd_soc_component_read(comp, FSL_AUDMIX_CTR); diff --git a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c index a7fef772760a6..2634699534db7 100644 --- a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c +++ b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c @@ -766,6 +766,11 @@ static const dai_register_cb dai_register_cbs[] = { mt8183_dai_memif_register, }; +static void mt8183_afe_release_reserved_mem(void *data) +{ + of_reserved_mem_device_release(data); +} + static int mt8183_afe_pcm_dev_probe(struct platform_device *pdev) { struct mtk_base_afe *afe; @@ -794,6 +799,12 @@ static int mt8183_afe_pcm_dev_probe(struct platform_device *pdev) if (ret) { dev_info(dev, "no reserved memory found, pre-allocating buffers instead\n"); afe->preallocate_buffers = true; + } else { + ret = devm_add_action_or_reset(dev, + mt8183_afe_release_reserved_mem, + dev); + if (ret) + return ret; } /* initial audio related clock */ @@ -833,17 +844,21 @@ static int mt8183_afe_pcm_dev_probe(struct platform_device *pdev) /* enable clock for regcache get default value from hw */ afe_priv->pm_runtime_bypass_reg_ctl = true; - pm_runtime_get_sync(dev); - - ret = regmap_reinit_cache(afe->regmap, &mt8183_afe_regmap_config); + ret = pm_runtime_resume_and_get(dev); if (ret) { - dev_err(dev, "regmap_reinit_cache fail, ret %d\n", ret); + afe_priv->pm_runtime_bypass_reg_ctl = false; goto err_pm_disable; } + ret = regmap_reinit_cache(afe->regmap, &mt8183_afe_regmap_config); pm_runtime_put_sync(dev); afe_priv->pm_runtime_bypass_reg_ctl = false; + if (ret) { + dev_err(dev, "regmap_reinit_cache fail, ret %d\n", ret); + goto err_pm_disable; + } + regcache_cache_only(afe->regmap, true); regcache_mark_dirty(afe->regmap); diff --git a/sound/soc/mediatek/mt8189/mt8189-afe-pcm.c b/sound/soc/mediatek/mt8189/mt8189-afe-pcm.c index 24b0c78815f61..77cf2b604f6ce 100644 --- a/sound/soc/mediatek/mt8189/mt8189-afe-pcm.c +++ b/sound/soc/mediatek/mt8189/mt8189-afe-pcm.c @@ -2351,9 +2351,13 @@ static int mt8189_afe_runtime_resume(struct device *dev) static int mt8189_afe_component_probe(struct snd_soc_component *component) { struct mtk_base_afe *afe = snd_soc_component_get_drvdata(component); + int ret; /* enable clock for regcache get default value from hw */ - pm_runtime_get_sync(afe->dev); + ret = pm_runtime_resume_and_get(afe->dev); + if (ret) + return dev_err_probe(afe->dev, ret, "failed to resume device\n"); + mtk_afe_add_sub_dai_control(component); pm_runtime_put_sync(afe->dev); @@ -2417,6 +2421,11 @@ static const struct reg_sequence mt8189_cg_patch[] = { { AUDIO_TOP_CON4, 0x361c }, }; +static void mt8189_afe_release_reserved_mem(void *data) +{ + of_reserved_mem_device_release(data); +} + static int mt8189_afe_pcm_dev_probe(struct platform_device *pdev) { int ret, i; @@ -2431,8 +2440,15 @@ static int mt8189_afe_pcm_dev_probe(struct platform_device *pdev) return ret; ret = of_reserved_mem_device_init(dev); - if (ret) + if (ret) { dev_warn(dev, "failed to assign memory region: %d\n", ret); + } else { + ret = devm_add_action_or_reset(dev, + mt8189_afe_release_reserved_mem, + dev); + if (ret) + return ret; + } afe = devm_kzalloc(dev, sizeof(*afe), GFP_KERNEL); if (!afe) @@ -2533,18 +2549,22 @@ static int mt8189_afe_pcm_dev_probe(struct platform_device *pdev) dev_pm_syscore_device(dev, true); /* enable clock for regcache get default value from hw */ - pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return dev_err_probe(dev, ret, "failed to resume device\n"); afe->regmap = devm_regmap_init_mmio(dev, afe->base_addr, &mt8189_afe_regmap_config); - if (IS_ERR(afe->regmap)) - return PTR_ERR(afe->regmap); + if (IS_ERR(afe->regmap)) { + ret = PTR_ERR(afe->regmap); + goto err_pm_put; + } ret = regmap_register_patch(afe->regmap, mt8189_cg_patch, ARRAY_SIZE(mt8189_cg_patch)); if (ret < 0) { dev_err(dev, "Failed to apply cg patch\n"); - goto err_pm_disable; + goto err_pm_put; } regmap_read(afe->regmap, AFE_IRQ_MCU_EN, &tmp_reg); @@ -2563,12 +2583,12 @@ static int mt8189_afe_pcm_dev_probe(struct platform_device *pdev) afe->num_dai_drivers); if (ret) { dev_err(dev, "afe component err: %d\n", ret); - goto err_pm_disable; + return ret; } return 0; -err_pm_disable: +err_pm_put: pm_runtime_put_sync(dev); return ret; } @@ -2578,14 +2598,12 @@ static void mt8189_afe_pcm_dev_remove(struct platform_device *pdev) struct mtk_base_afe *afe = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; - pm_runtime_put_sync(dev); if (!pm_runtime_status_suspended(dev)) mt8189_afe_runtime_suspend(dev); mt8189_afe_disable_main_clock(afe); /* disable afe clock */ mt8189_afe_disable_reg_rw_clk(afe); - of_reserved_mem_device_release(dev); } static const struct of_device_id mt8189_afe_pcm_dt_match[] = { diff --git a/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c b/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c index 3d32fe46118ec..db0ae44a86afe 100644 --- a/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c +++ b/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c @@ -2155,6 +2155,11 @@ static const dai_register_cb dai_register_cbs[] = { mt8192_dai_memif_register, }; +static void mt8192_afe_release_reserved_mem(void *data) +{ + of_reserved_mem_device_release(data); +} + static int mt8192_afe_pcm_dev_probe(struct platform_device *pdev) { struct mtk_base_afe *afe; @@ -2184,6 +2189,10 @@ static int mt8192_afe_pcm_dev_probe(struct platform_device *pdev) if (ret) { dev_info(dev, "no reserved memory found, pre-allocating buffers instead\n"); afe->preallocate_buffers = true; + } else { + ret = devm_add_action_or_reset(dev, mt8192_afe_release_reserved_mem, dev); + if (ret) + return ret; } /* init audio related clock */ @@ -2218,15 +2227,19 @@ static int mt8192_afe_pcm_dev_probe(struct platform_device *pdev) /* enable clock for regcache get default value from hw */ afe_priv->pm_runtime_bypass_reg_ctl = true; - pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret) { + afe_priv->pm_runtime_bypass_reg_ctl = false; + return dev_err_probe(dev, ret, "failed to resume device\n"); + } ret = regmap_reinit_cache(afe->regmap, &mt8192_afe_regmap_config); - if (ret) - return dev_err_probe(dev, ret, "regmap_reinit_cache fail\n"); - pm_runtime_put_sync(dev); afe_priv->pm_runtime_bypass_reg_ctl = false; + if (ret) + return dev_err_probe(dev, ret, "regmap_reinit_cache fail\n"); + regcache_cache_only(afe->regmap, true); regcache_mark_dirty(afe->regmap); diff --git a/sound/soc/meson/aiu-acodec-ctrl.c b/sound/soc/meson/aiu-acodec-ctrl.c index 483772ba69cd7..94c5d65335233 100644 --- a/sound/soc/meson/aiu-acodec-ctrl.c +++ b/sound/soc/meson/aiu-acodec-ctrl.c @@ -36,6 +36,9 @@ static int aiu_acodec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol, struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; unsigned int mux, changed; + if (ucontrol->value.enumerated.item[0] >= e->items) + return -EINVAL; + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); changed = snd_soc_component_test_bits(component, e->reg, CTRL_DIN_LRCLK_SRC, diff --git a/sound/soc/meson/aiu-codec-ctrl.c b/sound/soc/meson/aiu-codec-ctrl.c index 396f815077e29..60bb4cdfee520 100644 --- a/sound/soc/meson/aiu-codec-ctrl.c +++ b/sound/soc/meson/aiu-codec-ctrl.c @@ -28,6 +28,9 @@ static int aiu_codec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol, struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; unsigned int mux, changed; + if (ucontrol->value.enumerated.item[0] >= e->items) + return -EINVAL; + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); changed = snd_soc_component_test_bits(component, e->reg, CTRL_DATA_SEL, diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 2ab378fb50326..2873e831a8deb 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -549,6 +549,10 @@ static int graph_callback(const struct gpr_resp_pkt *data, void *priv, int op) token = hdr->token & APM_WRITE_TOKEN_MASK; done = data->payload; + if (!graph->rx_data.buf) { + mutex_unlock(&graph->lock); + break; + } phys = graph->rx_data.buf[token].phys; mutex_unlock(&graph->lock); /* token numbering starts at 0 */ @@ -571,6 +575,10 @@ static int graph_callback(const struct gpr_resp_pkt *data, void *priv, int op) client_event = APM_CLIENT_EVENT_DATA_READ_DONE; mutex_lock(&graph->lock); rd_done = data->payload; + if (!graph->tx_data.buf) { + mutex_unlock(&graph->lock); + break; + } phys = graph->tx_data.buf[hdr->token].phys; mutex_unlock(&graph->lock); /* token numbering starts at 0 */ diff --git a/sound/soc/renesas/rcar/rsnd.h b/sound/soc/renesas/rcar/rsnd.h index 04c70690f7a25..3e666125959b4 100644 --- a/sound/soc/renesas/rcar/rsnd.h +++ b/sound/soc/renesas/rcar/rsnd.h @@ -624,7 +624,7 @@ struct rsnd_priv { #define RSND_GEN2 (2 << 0) #define RSND_GEN3 (3 << 0) #define RSND_GEN4 (4 << 0) -#define RSND_SOC_MASK (0xFF << 4) +#define RSND_SOC_MASK (0xF << 4) #define RSND_SOC_E (1 << 4) /* E1/E2/E3 */ /* diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c index 2bfc8e5aee31d..243b83bb2bcf3 100644 --- a/sound/soc/sdca/sdca_asoc.c +++ b/sound/soc/sdca/sdca_asoc.c @@ -160,6 +160,9 @@ static int ge_put_enum_double(struct snd_kcontrol *kcontrol, unsigned int reg = e->reg; int ret; + if (item[0] >= e->items) + return -EINVAL; + reg &= ~SDW_SDCA_CTL_CSEL(0x3F); reg |= SDW_SDCA_CTL_CSEL(SDCA_CTL_GE_DETECTED_MODE); diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c index e440c23271001..4e8646d231c4f 100644 --- a/sound/soc/sdw_utils/soc_sdw_utils.c +++ b/sound/soc/sdw_utils/soc_sdw_utils.c @@ -818,6 +818,7 @@ struct asoc_sdw_codec_info codec_info_list[] = { { .direction = {true, false}, .codec_name = "cs42l43-codec", + .component_name = "cs42l43-spk", .dai_name = "cs42l43-dp6", .dai_type = SOC_SDW_DAI_TYPE_AMP, .dailink = {SOC_SDW_AMP_OUT_DAI_ID, SOC_SDW_UNUSED_DAI_ID}, diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index d6192204e613d..a26771c8e6ee6 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -2906,20 +2906,18 @@ static struct snd_soc_dapm_widget *dapm_find_widget( { struct snd_soc_dapm_widget *w; struct snd_soc_dapm_widget *fallback = NULL; - char prefixed_pin[80]; - const char *pin_name; - const char *prefix = dapm_prefix(dapm); - - if (prefix) { - snprintf(prefixed_pin, sizeof(prefixed_pin), "%s %s", - prefix, pin); - pin_name = prefixed_pin; - } else { - pin_name = pin; - } + bool pin_has_prefix = snd_soc_dapm_pin_has_prefix(dapm->card, pin); + bool match; for_each_card_widgets(dapm->card, w) { - if (!strcmp(w->name, pin_name)) { + match = false; + + if (!strcmp(pin, w->name)) + match = true; + else if (!pin_has_prefix && !snd_soc_dapm_widget_name_cmp(w, pin)) + match = true; + + if (match) { if (w->dapm == dapm) return w; else @@ -4872,6 +4870,33 @@ int snd_soc_dapm_ignore_suspend(struct snd_soc_dapm_context *dapm, } EXPORT_SYMBOL_GPL(snd_soc_dapm_ignore_suspend); +/** + * snd_soc_dapm_pin_has_prefix - check if given pin has a known prefix + * @card: card to be checked + * @pin: pin name + * + * Returns true if given pin has a known prefix + */ +bool snd_soc_dapm_pin_has_prefix(struct snd_soc_card *card, const char *pin) +{ + struct snd_soc_component *component; + const char *prefix; + size_t prefix_len; + + for_each_card_components(card, component) { + prefix = component->name_prefix; + if (!prefix) + continue; + + prefix_len = strlen(prefix); + if (!strncmp(pin, prefix, prefix_len) && pin[prefix_len] == ' ') + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(snd_soc_dapm_pin_has_prefix); + /** * snd_soc_dapm_free - free dapm resources * @dapm: DAPM context diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c index 85679c8e02299..35cbe29d22758 100644 --- a/sound/soc/soc-topology.c +++ b/sound/soc/soc-topology.c @@ -1326,9 +1326,24 @@ static int soc_tplg_dapm_complete(struct soc_tplg *tplg) return ret; } +static int soc_tplg_check_name(const char *name) +{ + if (strnlen(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) == + SNDRV_CTL_ELEM_ID_NAME_MAXLEN) + return -EINVAL; + + return 0; +} + static int set_stream_info(struct soc_tplg *tplg, struct snd_soc_pcm_stream *stream, struct snd_soc_tplg_stream_caps *caps) { + int ret; + + ret = soc_tplg_check_name(caps->name); + if (ret) + return ret; + stream->stream_name = devm_kstrdup(tplg->dev, caps->name, GFP_KERNEL); if (!stream->stream_name) return -ENOMEM; @@ -1380,7 +1395,11 @@ static int soc_tplg_dai_create(struct soc_tplg *tplg, if (dai_drv == NULL) return -ENOMEM; - if (strlen(pcm->dai_name)) { + ret = soc_tplg_check_name(pcm->dai_name); + if (ret) + goto err; + + if (pcm->dai_name[0]) { dai_drv->name = devm_kstrdup(tplg->dev, pcm->dai_name, GFP_KERNEL); if (!dai_drv->name) { ret = -ENOMEM; @@ -1486,7 +1505,11 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg, if (tplg->ops) link->dobj.unload = tplg->ops->link_unload; - if (strlen(pcm->pcm_name)) { + ret = soc_tplg_check_name(pcm->pcm_name); + if (ret) + goto err; + + if (pcm->pcm_name[0]) { link->name = devm_kstrdup(tplg->dev, pcm->pcm_name, GFP_KERNEL); link->stream_name = devm_kstrdup(tplg->dev, pcm->pcm_name, GFP_KERNEL); if (!link->name || !link->stream_name) { @@ -1496,7 +1519,11 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg, } link->id = le32_to_cpu(pcm->pcm_id); - if (strlen(pcm->dai_name)) { + ret = soc_tplg_check_name(pcm->dai_name); + if (ret) + goto err; + + if (pcm->dai_name[0]) { link->cpus->dai_name = devm_kstrdup(tplg->dev, pcm->dai_name, GFP_KERNEL); if (!link->cpus->dai_name) { ret = -ENOMEM; @@ -1848,6 +1875,10 @@ static int soc_tplg_dai_config(struct soc_tplg *tplg, memset(&dai_component, 0, sizeof(dai_component)); + ret = soc_tplg_check_name(d->dai_name); + if (ret) + return ret; + dai_component.dai_name = d->dai_name; dai = snd_soc_find_dai(&dai_component); if (!dai) { diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig index e31f4c4061d80..915abbef398d5 100644 --- a/sound/soc/sof/intel/Kconfig +++ b/sound/soc/sof/intel/Kconfig @@ -266,10 +266,8 @@ config SND_SOC_SOF_METEORLAKE config SND_SOC_SOF_INTEL_LNL tristate - select SOUNDWIRE_INTEL if SND_SOC_SOF_INTEL_SOUNDWIRE != n select SND_SOC_SOF_HDA_GENERIC select SND_SOC_SOF_INTEL_SOUNDWIRE_LINK_BASELINE - select SND_SOF_SOF_HDA_SDW_BPT if SND_SOC_SOF_INTEL_SOUNDWIRE != n select SND_SOC_SOF_IPC4 select SND_SOC_SOF_INTEL_MTL @@ -329,8 +327,10 @@ config SND_SOC_SOF_HDA_GENERIC select SND_INTEL_DSP_CONFIG select SND_SOC_SOF_HDA_LINK_BASELINE select SND_SOC_SOF_HDA_PROBES - select SND_SOC_SDW_UTILS if SND_SOC_SOF_INTEL_SOUNDWIRE + select SND_SOC_SDW_UTILS if SND_SOC_SOF_INTEL_SOUNDWIRE !=n select SND_SOC_SOF_HDA_MLINK if SND_SOC_SOF_HDA_LINK + select SND_SOF_SOF_HDA_SDW_BPT if SND_SOC_SOF_INTEL_LNL != n && \ + SND_SOC_SOF_INTEL_SOUNDWIRE !=n help This option is not user-selectable but automagically handled by 'select' statements at a higher level. diff --git a/sound/soc/sof/ipc3-control.c b/sound/soc/sof/ipc3-control.c index 2b1befad6d5c0..d1697401b1da0 100644 --- a/sound/soc/sof/ipc3-control.c +++ b/sound/soc/sof/ipc3-control.c @@ -315,10 +315,13 @@ static int sof_ipc3_bytes_get(struct snd_sof_control *scontrol, } /* be->max has been verified to be >= sizeof(struct sof_abi_hdr) */ - if (data->size > scontrol->max_size - sizeof(*data)) { + if (data->size > scontrol->max_size - sizeof(*cdata) - + sizeof(*data)) { dev_err_ratelimited(scomp->dev, "%u bytes of control data is invalid, max is %zu\n", - data->size, scontrol->max_size - sizeof(*data)); + data->size, + scontrol->max_size - sizeof(*cdata) - + sizeof(*data)); return -EINVAL; } @@ -336,6 +339,8 @@ static int sof_ipc3_bytes_put(struct snd_sof_control *scontrol, struct sof_ipc_ctrl_data *cdata = scontrol->ipc_control_data; struct snd_soc_component *scomp = scontrol->scomp; struct sof_abi_hdr *data = cdata->data; + const struct sof_abi_hdr *new_hdr = + (const struct sof_abi_hdr *)ucontrol->value.bytes.data; size_t size; if (scontrol->max_size > sizeof(ucontrol->value.bytes.data)) { @@ -344,14 +349,18 @@ static int sof_ipc3_bytes_put(struct snd_sof_control *scontrol, return -EINVAL; } - /* scontrol->max_size has been verified to be >= sizeof(struct sof_abi_hdr) */ - if (data->size > scontrol->max_size - sizeof(*data)) { - dev_err_ratelimited(scomp->dev, "data size too big %u bytes max is %zu\n", - data->size, scontrol->max_size - sizeof(*data)); + /* Validate the new data's size, not the old one */ + if (new_hdr->size > scontrol->max_size - sizeof(*cdata) - + sizeof(*new_hdr)) { + dev_err_ratelimited(scomp->dev, + "data size too big %u bytes max is %zu\n", + new_hdr->size, + scontrol->max_size - sizeof(*cdata) - + sizeof(*new_hdr)); return -EINVAL; } - size = data->size + sizeof(*data); + size = new_hdr->size + sizeof(*new_hdr); /* copy from kcontrol */ memcpy(data, ucontrol->value.bytes.data, size); @@ -389,9 +398,17 @@ static int sof_ipc3_bytes_ext_put(struct snd_sof_control *scontrol, } /* be->max is coming from topology */ - if (header.length > scontrol->max_size) { - dev_err_ratelimited(scomp->dev, "Bytes data size %d exceeds max %zu\n", - header.length, scontrol->max_size); + if (header.length > scontrol->max_size - sizeof(*cdata)) { + dev_err_ratelimited(scomp->dev, "Bytes data size %u exceeds max %zu\n", + header.length, scontrol->max_size - sizeof(*cdata)); + return -EINVAL; + } + + /* Ensure the data is large enough to contain the ABI header */ + if (header.length < sizeof(struct sof_abi_hdr)) { + dev_err_ratelimited(scomp->dev, + "Bytes data size %u less than ABI header %zu\n", + header.length, sizeof(struct sof_abi_hdr)); return -EINVAL; } @@ -427,7 +444,7 @@ static int sof_ipc3_bytes_ext_put(struct snd_sof_control *scontrol, } /* be->max has been verified to be >= sizeof(struct sof_abi_hdr) */ - if (cdata->data->size > scontrol->max_size - sizeof(struct sof_abi_hdr)) { + if (cdata->data->size > scontrol->max_size - sizeof(*cdata) - sizeof(struct sof_abi_hdr)) { dev_err_ratelimited(scomp->dev, "Mismatch in ABI data size (truncated?)\n"); goto err_restore; } @@ -443,7 +460,7 @@ static int sof_ipc3_bytes_ext_put(struct snd_sof_control *scontrol, err_restore: /* If we have an issue, we restore the old, valid bytes control data */ if (scontrol->old_ipc_control_data) { - memcpy(cdata->data, scontrol->old_ipc_control_data, scontrol->max_size); + memcpy(cdata, scontrol->old_ipc_control_data, scontrol->max_size); kfree(scontrol->old_ipc_control_data); scontrol->old_ipc_control_data = NULL; } @@ -482,10 +499,13 @@ static int _sof_ipc3_bytes_ext_get(struct snd_sof_control *scontrol, } /* check data size doesn't exceed max coming from topology */ - if (cdata->data->size > scontrol->max_size - sizeof(struct sof_abi_hdr)) { - dev_err_ratelimited(scomp->dev, "User data size %d exceeds max size %zu\n", + if (cdata->data->size > scontrol->max_size - sizeof(*cdata) - + sizeof(struct sof_abi_hdr)) { + dev_err_ratelimited(scomp->dev, + "User data size %u exceeds max size %zu\n", cdata->data->size, - scontrol->max_size - sizeof(struct sof_abi_hdr)); + scontrol->max_size - sizeof(*cdata) - + sizeof(struct sof_abi_hdr)); return -EINVAL; } @@ -535,6 +555,15 @@ static void snd_sof_update_control(struct snd_sof_control *scontrol, return; } + /* Verify the size fits within the allocation */ + if (cdata->num_elems > scontrol->max_size - sizeof(*local_cdata) - + sizeof(*local_cdata->data)) { + dev_err(scomp->dev, + "cdata binary size %u exceeds buffer\n", + cdata->num_elems); + return; + } + /* copy the new binary data */ memcpy(local_cdata->data, cdata->data, cdata->num_elems); } else if (cdata->num_elems != scontrol->num_channels) { @@ -626,16 +655,28 @@ static void sof_ipc3_control_update(struct snd_sof_dev *sdev, void *ipc_control_ return; } - expected_size = sizeof(struct sof_ipc_ctrl_data); switch (cdata->type) { case SOF_CTRL_TYPE_VALUE_CHAN_GET: case SOF_CTRL_TYPE_VALUE_CHAN_SET: - expected_size += cdata->num_elems * - sizeof(struct sof_ipc_ctrl_value_chan); + if (check_mul_overflow((size_t)cdata->num_elems, + sizeof(struct sof_ipc_ctrl_value_chan), + &expected_size)) + return; + if (check_add_overflow(expected_size, + sizeof(struct sof_ipc_ctrl_data), + &expected_size)) + return; break; case SOF_CTRL_TYPE_DATA_GET: case SOF_CTRL_TYPE_DATA_SET: - expected_size += cdata->num_elems + sizeof(struct sof_abi_hdr); + if (check_add_overflow((size_t)cdata->num_elems, + sizeof(struct sof_abi_hdr), + &expected_size)) + return; + if (check_add_overflow(expected_size, + sizeof(struct sof_ipc_ctrl_data), + &expected_size)) + return; break; default: return; diff --git a/sound/soc/sof/ipc4-control.c b/sound/soc/sof/ipc4-control.c index 596c3d77a34e1..b603a54492210 100644 --- a/sound/soc/sof/ipc4-control.c +++ b/sound/soc/sof/ipc4-control.c @@ -562,6 +562,8 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, struct snd_soc_component *scomp = scontrol->scomp; struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); struct sof_abi_hdr *data = cdata->data; + const struct sof_abi_hdr *new_hdr = + (const struct sof_abi_hdr *)ucontrol->value.bytes.data; size_t size; int ret; @@ -572,15 +574,16 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, return -EINVAL; } - /* scontrol->max_size has been verified to be >= sizeof(struct sof_abi_hdr) */ - if (data->size > scontrol->max_size - sizeof(*data)) { + /* Validate the new data's size, not the old one */ + if (new_hdr->size > scontrol->max_size - sizeof(*new_hdr)) { dev_err_ratelimited(scomp->dev, "data size too big %u bytes max is %zu\n", - data->size, scontrol->max_size - sizeof(*data)); + new_hdr->size, + scontrol->max_size - sizeof(*new_hdr)); return -EINVAL; } - size = data->size + sizeof(*data); + size = new_hdr->size + sizeof(*new_hdr); /* copy from kcontrol */ memcpy(data, ucontrol->value.bytes.data, size); @@ -880,6 +883,16 @@ static void sof_ipc4_control_update(struct snd_sof_dev *sdev, void *ipc_message) */ if (type == SND_SOC_TPLG_TYPE_BYTES) { struct sof_abi_hdr *data = cdata->data; + size_t source_size = struct_size(msg_data, data, msg_data->num_elems); + + if (source_size > ndata->event_data_size) { + dev_warn(sdev->dev, + "%s: invalid bytes notification size for %s (%zu, %u)\n", + __func__, scontrol->name, source_size, + ndata->event_data_size); + scontrol->comp_data_dirty = true; + goto notify; + } if (msg_data->num_elems > scontrol->max_size - sizeof(*data)) { dev_warn(sdev->dev, @@ -892,6 +905,17 @@ static void sof_ipc4_control_update(struct snd_sof_dev *sdev, void *ipc_message) scontrol->size = sizeof(*cdata) + sizeof(*data) + data->size; } } else { + size_t source_size = struct_size(msg_data, chanv, msg_data->num_elems); + + if (source_size > ndata->event_data_size) { + dev_warn(sdev->dev, + "%s: invalid channel notification size for %s (%zu, %u)\n", + __func__, scontrol->name, source_size, + ndata->event_data_size); + scontrol->comp_data_dirty = true; + goto notify; + } + for (i = 0; i < msg_data->num_elems; i++) { u32 channel = msg_data->chanv[i].channel; @@ -919,6 +943,8 @@ static void sof_ipc4_control_update(struct snd_sof_dev *sdev, void *ipc_message) scontrol->comp_data_dirty = true; } +notify: + /* * Look up the ALSA kcontrol of the scontrol to be able to send a * notification to user space diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 63d582c658915..8f6f89b983eb0 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -733,10 +733,13 @@ static int sof_parse_token_sets(struct snd_soc_component *scomp, int ret; while (array_size > 0 && total < count * token_instance_num) { + if (array_size < (int)sizeof(*array)) + return -EINVAL; + asize = le32_to_cpu(array->size); /* validate asize */ - if (asize < sizeof(*array)) { + if (asize < (int)sizeof(*array)) { dev_err(scomp->dev, "error: invalid array size 0x%x\n", asize); return -EINVAL; @@ -2534,6 +2537,8 @@ int snd_sof_load_topology(struct snd_soc_component *scomp, const char *file) if (strstr(file, "dummy")) { dev_err(scomp->dev, "Function topology is required, please upgrade sof-firmware\n"); + + kfree(tplg_files); return -EINVAL; } tplg_files[0] = file; diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c index ece33b7ff190a..efc8f33886682 100644 --- a/sound/soc/tegra/tegra210_ahub.c +++ b/sound/soc/tegra/tegra210_ahub.c @@ -62,13 +62,15 @@ static int tegra_ahub_put_value_enum(struct snd_kcontrol *kctl, struct snd_soc_dapm_update update[TEGRA_XBAR_UPDATE_MAX_REG] = { }; int val_bytes = snd_soc_component_regmap_val_bytes(cmpnt); unsigned int *item = uctl->value.enumerated.item; - unsigned int value = e->values[item[0]]; + unsigned int value; unsigned int i, bit_pos, reg_idx = 0, reg_val = 0; int change = 0; if (item[0] >= e->items) return -EINVAL; + value = e->values[item[0]]; + if (value) { /* Get the register index and value to set */ reg_idx = (value - 1) / (8 * val_bytes); diff --git a/sound/usb/caiaq/input.c b/sound/usb/caiaq/input.c index 5c70fdf61cc13..2db4d1332df1c 100644 --- a/sound/usb/caiaq/input.c +++ b/sound/usb/caiaq/input.c @@ -330,7 +330,7 @@ static void snd_usb_caiaq_tks4_dispatch(struct snd_usb_caiaqdev *cdev, { struct device *dev = caiaqdev_to_dev(cdev); - while (len) { + while (len >= TKS4_MSGBLOCK_SIZE) { unsigned int i, block_id = (buf[0] << 8) | buf[1]; switch (block_id) { diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index 6fbcb117555c6..24cd7692bd01c 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -1780,8 +1780,16 @@ static void snd_usb_handle_sync_urb(struct snd_usb_endpoint *ep, /* * skip empty packets. At least M-Audio's Fast Track Ultra stops * streaming once it received a 0-byte OUT URB + * + * However, on devices where bytes==0 means every sync-source + * packet errored (e.g. Behringer Flow 8 returning -EXDEV bursts + * for entire capture URBs), an unconditional return starves the + * IFB-fed OUT ring permanently. Such devices set + * QUIRK_FLAG_IFB_SILENCE_ON_EMPTY to fall through and enqueue a + * packet_info with size 0 packets, so playback emits silence + * and the OUT ring keeps moving. */ - if (bytes == 0) + if (bytes == 0 && !(ep->chip->quirk_flags & QUIRK_FLAG_IFB_SILENCE_ON_EMPTY)) return; spin_lock_irqsave(&ep->lock, flags); diff --git a/sound/usb/fcp.c b/sound/usb/fcp.c index 0fc4d063c48a5..c45dbe4d45328 100644 --- a/sound/usb/fcp.c +++ b/sound/usb/fcp.c @@ -1083,6 +1083,8 @@ static int fcp_find_fc_interface(struct usb_mixer_interface *mixer) if (desc->bInterfaceClass != 255) continue; + if (desc->bNumEndpoints < 1) + continue; epd = get_endpoint(intf->altsetting, 0); private->bInterfaceNumber = desc->bInterfaceNumber; diff --git a/sound/usb/midi2.c b/sound/usb/midi2.c index 04aeb9052f139..3ec6332917726 100644 --- a/sound/usb/midi2.c +++ b/sound/usb/midi2.c @@ -470,6 +470,11 @@ static int create_midi2_endpoint(struct snd_usb_midi2_interface *umidi, static void free_midi2_endpoint(struct snd_usb_midi2_endpoint *ep) { list_del(&ep->list); + if (!ep->disconnected) { + ep->disconnected = 1; + kill_midi_urbs(ep, false); + drain_urb_queue(ep); + } free_midi_urbs(ep); kfree(ep); } diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index 1bdaa46d4fe18..06d111e687ebb 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -333,6 +333,7 @@ static int snd_audigy2nx_led_put(struct snd_kcontrol *kcontrol, int index = kcontrol->private_value & 0xff; unsigned int value = ucontrol->value.integer.value[0]; int old_value = kcontrol->private_value >> 8; + unsigned long old_pval = kcontrol->private_value; int err; if (value > 1) @@ -341,7 +342,11 @@ static int snd_audigy2nx_led_put(struct snd_kcontrol *kcontrol, return 0; kcontrol->private_value = (value << 8) | index; err = snd_audigy2nx_led_update(mixer, value, index); - return err < 0 ? err : 1; + if (err < 0) { + kcontrol->private_value = old_pval; + return err; + } + return 1; } static int snd_audigy2nx_led_resume(struct usb_mixer_elem_list *list) @@ -487,6 +492,7 @@ static int snd_emu0204_ch_switch_put(struct snd_kcontrol *kcontrol, struct usb_mixer_elem_list *list = snd_kcontrol_chip(kcontrol); struct usb_mixer_interface *mixer = list->mixer; unsigned int value = ucontrol->value.enumerated.item[0]; + unsigned long old_pval = kcontrol->private_value; int err; if (value > 1) @@ -497,7 +503,11 @@ static int snd_emu0204_ch_switch_put(struct snd_kcontrol *kcontrol, kcontrol->private_value = value; err = snd_emu0204_ch_switch_update(mixer, value); - return err < 0 ? err : 1; + if (err < 0) { + kcontrol->private_value = old_pval; + return err; + } + return 1; } static int snd_emu0204_ch_switch_resume(struct usb_mixer_elem_list *list) @@ -567,46 +577,30 @@ static bool snd_dualsense_ih_match(struct input_handler *handler, { struct dualsense_mixer_elem_info *mei; struct usb_device *snd_dev; - char *input_dev_path, *usb_dev_path; - size_t usb_dev_path_len; - bool match = false; + struct device *parent; mei = container_of(handler, struct dualsense_mixer_elem_info, ih); snd_dev = mei->info.head.mixer->chip->dev; - input_dev_path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL); - if (!input_dev_path) { - dev_warn(&snd_dev->dev, "Failed to get input dev path\n"); - return false; - } - - usb_dev_path = kobject_get_path(&snd_dev->dev.kobj, GFP_KERNEL); - if (!usb_dev_path) { - dev_warn(&snd_dev->dev, "Failed to get USB dev path\n"); - goto free_paths; - } - /* * Ensure the VID:PID matched input device supposedly owned by the * hid-playstation driver belongs to the actual hardware handled by - * the current USB audio device, which implies input_dev_path being - * a subpath of usb_dev_path. + * the current USB audio device. * * This verification is necessary when there is more than one identical * controller attached to the host system. + * + * The input device is registered below the HID device, USB interface and + * USB device, so compare the parent chain directly instead of building + * kobject path strings. This avoids dereferencing kobject names while the + * USB device hierarchy is being torn down during disconnect. */ - usb_dev_path_len = strlen(usb_dev_path); - if (usb_dev_path_len >= strlen(input_dev_path)) - goto free_paths; - - usb_dev_path[usb_dev_path_len] = '/'; - match = !memcmp(input_dev_path, usb_dev_path, usb_dev_path_len + 1); - -free_paths: - kfree(input_dev_path); - kfree(usb_dev_path); + for (parent = dev->dev.parent; parent; parent = parent->parent) { + if (parent == &snd_dev->dev) + return true; + } - return match; + return false; } static int snd_dualsense_ih_connect(struct input_handler *handler, @@ -821,7 +815,11 @@ static int snd_xonar_u1_switch_put(struct snd_kcontrol *kcontrol, kcontrol->private_value = new_status; err = snd_xonar_u1_switch_update(list->mixer, new_status); - return err < 0 ? err : 1; + if (err < 0) { + kcontrol->private_value = old_status; + return err; + } + return 1; } static int snd_xonar_u1_switch_resume(struct usb_mixer_elem_list *list) @@ -1159,7 +1157,8 @@ static int snd_nativeinstruments_control_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct usb_mixer_elem_list *list = snd_kcontrol_chip(kcontrol); - u8 oldval = (kcontrol->private_value >> 24) & 0xff; + unsigned long old_pval = kcontrol->private_value; + u8 oldval = (old_pval >> 24) & 0xff; u8 newval = ucontrol->value.integer.value[0]; int err; @@ -1169,7 +1168,11 @@ static int snd_nativeinstruments_control_put(struct snd_kcontrol *kcontrol, kcontrol->private_value &= ~(0xff << 24); kcontrol->private_value |= (unsigned int)newval << 24; err = snd_ni_update_cur_val(list); - return err < 0 ? err : 1; + if (err < 0) { + kcontrol->private_value = old_pval; + return err; + } + return 1; } static const struct snd_kcontrol_new snd_nativeinstruments_ta6_mixers[] = { @@ -1324,7 +1327,8 @@ static int snd_ftu_eff_switch_put(struct snd_kcontrol *kctl, struct snd_ctl_elem_value *ucontrol) { struct usb_mixer_elem_list *list = snd_kcontrol_chip(kctl); - unsigned int pval = list->kctl->private_value; + unsigned long old_pval = list->kctl->private_value; + unsigned int pval = old_pval; int cur_val, err, new_val; cur_val = pval >> 24; @@ -1335,7 +1339,11 @@ static int snd_ftu_eff_switch_put(struct snd_kcontrol *kctl, kctl->private_value &= ~(0xff << 24); kctl->private_value |= new_val << 24; err = snd_ftu_eff_switch_update(list); - return err < 0 ? err : 1; + if (err < 0) { + kctl->private_value = old_pval; + return err; + } + return 1; } static int snd_ftu_create_effect_switch(struct usb_mixer_interface *mixer, @@ -2114,13 +2122,18 @@ static int snd_soundblaster_e1_switch_put(struct snd_kcontrol *kcontrol, { struct usb_mixer_elem_list *list = snd_kcontrol_chip(kcontrol); unsigned char value = !!ucontrol->value.integer.value[0]; + unsigned long old_pval = kcontrol->private_value; int err; if (kcontrol->private_value == value) return 0; kcontrol->private_value = value; err = snd_soundblaster_e1_switch_update(list->mixer, value); - return err < 0 ? err : 1; + if (err < 0) { + kcontrol->private_value = old_pval; + return err; + } + return 1; } static int snd_soundblaster_e1_switch_resume(struct usb_mixer_elem_list *list) @@ -2998,12 +3011,14 @@ static int snd_bbfpro_ctl_put(struct snd_kcontrol *kcontrol, if (val == old_value) return 0; + err = snd_bbfpro_ctl_update(mixer, reg, idx, val); + if (err < 0) + return err; + kcontrol->private_value = reg | ((idx & SND_BBFPRO_CTL_IDX_MASK) << SND_BBFPRO_CTL_IDX_SHIFT) | ((val & SND_BBFPRO_CTL_VAL_MASK) << SND_BBFPRO_CTL_VAL_SHIFT); - - err = snd_bbfpro_ctl_update(mixer, reg, idx, val); - return err < 0 ? err : 1; + return 1; } static int snd_bbfpro_ctl_resume(struct usb_mixer_elem_list *list) @@ -3188,11 +3203,13 @@ static int snd_bbfpro_vol_put(struct snd_kcontrol *kcontrol, new_val = uvalue & SND_BBFPRO_MIXER_VAL_MASK; + err = snd_bbfpro_vol_update(mixer, idx, new_val); + if (err < 0) + return err; + kcontrol->private_value = idx | (new_val << SND_BBFPRO_MIXER_VAL_SHIFT); - - err = snd_bbfpro_vol_update(mixer, idx, new_val); - return err < 0 ? err : 1; + return 1; } static int snd_bbfpro_vol_resume(struct usb_mixer_elem_list *list) @@ -4451,6 +4468,7 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer) case USB_ID(0x1235, 0x821b): /* Focusrite Scarlett 16i16 4th Gen */ case USB_ID(0x1235, 0x821c): /* Focusrite Scarlett 18i16 4th Gen */ case USB_ID(0x1235, 0x821d): /* Focusrite Scarlett 18i20 4th Gen */ + case USB_ID(0x1235, 0x821e): /* Focusrite ISA C8X */ err = snd_fcp_init(mixer); break; diff --git a/sound/usb/mixer_scarlett.c b/sound/usb/mixer_scarlett.c index 1bb01e8276549..673eb8d8724db 100644 --- a/sound/usb/mixer_scarlett.c +++ b/sound/usb/mixer_scarlett.c @@ -680,7 +680,9 @@ static int scarlett_ctl_enum_put(struct snd_kcontrol *kctl, val = ucontrol->value.integer.value[0]; val = val + opt->start; if (val != oval) { - snd_usb_set_cur_mix_value(elem, 0, 0, val); + err = snd_usb_set_cur_mix_value(elem, 0, 0, val); + if (err < 0) + return err; return 1; } return 0; diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index a4fac46522013..d7b8c204110c3 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -612,6 +612,20 @@ struct scarlett2_config_set { const struct scarlett2_config items[SCARLETT2_CONFIG_COUNT]; }; +/* Map firmware versions to config sets per-device. + * + * Each device lists one or more entries, sorted in ascending order of + * from_firmware_version. At probe time the running firmware version + * is looked up against this list and the last entry whose + * from_firmware_version is <= the running version is selected. + * + * The list is terminated by a sentinel entry with config_set == NULL. + */ +struct scarlett2_config_set_entry { + u16 from_firmware_version; + const struct scarlett2_config_set *config_set; +}; + /* Input gain TLV dB ranges */ static const DECLARE_TLV_DB_MINMAX( @@ -923,6 +937,63 @@ static const struct scarlett2_config_set scarlett2_config_set_gen4_2i2 = { } }; +/* 2i2 Gen 4, firmware version 2417 and above + * + * Firmware 2417 shifted DIRECT_MONITOR_GAIN by 4 bytes; all other + * offsets are unchanged from scarlett2_config_set_gen4_2i2. + */ +static const struct scarlett2_config_set scarlett2_config_set_gen4_2i2_2417 = { + .notifications = scarlett4_2i2_notifications, + .param_buf_addr = 0xfc, + .input_gain_tlv = db_scale_gen4_gain, + .autogain_status_texts = scarlett2_autogain_status_gen4, + .items = { + [SCARLETT2_CONFIG_MSD_SWITCH] = { + .offset = 0x49, .size = 8, .activate = 4 }, + + [SCARLETT2_CONFIG_DIRECT_MONITOR] = { + .offset = 0x14a, .size = 8, .activate = 16, .pbuf = 1 }, + + [SCARLETT2_CONFIG_AUTOGAIN_SWITCH] = { + .offset = 0x135, .size = 8, .activate = 10, .pbuf = 1 }, + + [SCARLETT2_CONFIG_AUTOGAIN_STATUS] = { + .offset = 0x137, .size = 8 }, + + [SCARLETT2_CONFIG_AG_MEAN_TARGET] = { + .offset = 0x131, .size = 8, .activate = 29, .pbuf = 1 }, + + [SCARLETT2_CONFIG_AG_PEAK_TARGET] = { + .offset = 0x132, .size = 8, .activate = 30, .pbuf = 1 }, + + [SCARLETT2_CONFIG_PHANTOM_SWITCH] = { + .offset = 0x48, .size = 8, .activate = 11, .pbuf = 1, + .mute = 1 }, + + [SCARLETT2_CONFIG_INPUT_GAIN] = { + .offset = 0x4b, .size = 8, .activate = 12, .pbuf = 1 }, + + [SCARLETT2_CONFIG_LEVEL_SWITCH] = { + .offset = 0x3c, .size = 8, .activate = 13, .pbuf = 1, + .mute = 1 }, + + [SCARLETT2_CONFIG_SAFE_SWITCH] = { + .offset = 0x147, .size = 8, .activate = 14, .pbuf = 1 }, + + [SCARLETT2_CONFIG_AIR_SWITCH] = { + .offset = 0x3e, .size = 8, .activate = 15, .pbuf = 1 }, + + [SCARLETT2_CONFIG_INPUT_SELECT_SWITCH] = { + .offset = 0x14b, .size = 8, .activate = 17, .pbuf = 1 }, + + [SCARLETT2_CONFIG_INPUT_LINK_SWITCH] = { + .offset = 0x14e, .size = 8, .activate = 18, .pbuf = 1 }, + + [SCARLETT2_CONFIG_DIRECT_MONITOR_GAIN] = { + .offset = 0x2a4, .size = 16, .activate = 36 } + } +}; + /* 4i4 Gen 4 */ static const struct scarlett2_config_set scarlett2_config_set_gen4_4i4 = { .notifications = scarlett4_4i4_notifications, @@ -1100,8 +1171,8 @@ struct scarlett2_meter_entry { }; struct scarlett2_device_info { - /* which set of configuration parameters the device uses */ - const struct scarlett2_config_set *config_set; + /* which sets of configuration parameters the device uses */ + const struct scarlett2_config_set_entry *config_sets; /* minimum firmware version required */ u16 min_firmware_version; @@ -1343,7 +1414,10 @@ struct scarlett2_data { /*** Model-specific data ***/ static const struct scarlett2_device_info s6i6_gen2_info = { - .config_set = &scarlett2_config_set_gen2a, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen2a }, + { } + }, .level_input_count = 2, .pad_input_count = 2, @@ -1393,7 +1467,10 @@ static const struct scarlett2_device_info s6i6_gen2_info = { }; static const struct scarlett2_device_info s18i8_gen2_info = { - .config_set = &scarlett2_config_set_gen2a, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen2a }, + { } + }, .level_input_count = 2, .pad_input_count = 4, @@ -1446,7 +1523,10 @@ static const struct scarlett2_device_info s18i8_gen2_info = { }; static const struct scarlett2_device_info s18i20_gen2_info = { - .config_set = &scarlett2_config_set_gen2b, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen2b }, + { } + }, .line_out_descrs = { "Monitor L", @@ -1503,7 +1583,10 @@ static const struct scarlett2_device_info s18i20_gen2_info = { }; static const struct scarlett2_device_info solo_gen3_info = { - .config_set = &scarlett2_config_set_gen3a, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen3a }, + { } + }, .level_input_count = 1, .level_input_first = 1, .air_input_count = 1, @@ -1513,7 +1596,10 @@ static const struct scarlett2_device_info solo_gen3_info = { }; static const struct scarlett2_device_info s2i2_gen3_info = { - .config_set = &scarlett2_config_set_gen3a, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen3a }, + { } + }, .level_input_count = 2, .air_input_count = 2, .phantom_count = 1, @@ -1522,7 +1608,10 @@ static const struct scarlett2_device_info s2i2_gen3_info = { }; static const struct scarlett2_device_info s4i4_gen3_info = { - .config_set = &scarlett2_config_set_gen3b, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen3b }, + { } + }, .level_input_count = 2, .pad_input_count = 2, .air_input_count = 2, @@ -1571,7 +1660,10 @@ static const struct scarlett2_device_info s4i4_gen3_info = { }; static const struct scarlett2_device_info s8i6_gen3_info = { - .config_set = &scarlett2_config_set_gen3b, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen3b }, + { } + }, .level_input_count = 2, .pad_input_count = 2, .air_input_count = 2, @@ -1637,7 +1729,10 @@ static const char * const scarlett2_spdif_s18i8_gen3_texts[] = { }; static const struct scarlett2_device_info s18i8_gen3_info = { - .config_set = &scarlett2_config_set_gen3c, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen3c }, + { } + }, .has_speaker_switching = 1, .level_input_count = 2, .pad_input_count = 4, @@ -1729,7 +1824,10 @@ static const char * const scarlett2_spdif_s18i20_gen3_texts[] = { }; static const struct scarlett2_device_info s18i20_gen3_info = { - .config_set = &scarlett2_config_set_gen3c, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_gen3c }, + { } + }, .has_speaker_switching = 1, .has_talkback = 1, .level_input_count = 2, @@ -1803,7 +1901,10 @@ static const struct scarlett2_device_info s18i20_gen3_info = { }; static const struct scarlett2_device_info vocaster_one_info = { - .config_set = &scarlett2_config_set_vocaster, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 1769, &scarlett2_config_set_vocaster }, + { } + }, .min_firmware_version = 1769, .has_devmap = 1, @@ -1847,7 +1948,10 @@ static const struct scarlett2_device_info vocaster_one_info = { }; static const struct scarlett2_device_info vocaster_two_info = { - .config_set = &scarlett2_config_set_vocaster, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 1769, &scarlett2_config_set_vocaster }, + { } + }, .min_firmware_version = 1769, .has_devmap = 1, @@ -1892,7 +1996,10 @@ static const struct scarlett2_device_info vocaster_two_info = { }; static const struct scarlett2_device_info solo_gen4_info = { - .config_set = &scarlett2_config_set_gen4_solo, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 2115, &scarlett2_config_set_gen4_solo }, + { } + }, .min_firmware_version = 2115, .has_devmap = 1, @@ -1947,7 +2054,11 @@ static const struct scarlett2_device_info solo_gen4_info = { }; static const struct scarlett2_device_info s2i2_gen4_info = { - .config_set = &scarlett2_config_set_gen4_2i2, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 2115, &scarlett2_config_set_gen4_2i2 }, + { 2417, &scarlett2_config_set_gen4_2i2_2417 }, + { } + }, .min_firmware_version = 2115, .has_devmap = 1, @@ -2002,7 +2113,10 @@ static const struct scarlett2_device_info s2i2_gen4_info = { }; static const struct scarlett2_device_info s4i4_gen4_info = { - .config_set = &scarlett2_config_set_gen4_4i4, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 2089, &scarlett2_config_set_gen4_4i4 }, + { } + }, .min_firmware_version = 2089, .has_devmap = 1, @@ -2051,7 +2165,10 @@ static const struct scarlett2_device_info s4i4_gen4_info = { }; static const struct scarlett2_device_info clarett_2pre_info = { - .config_set = &scarlett2_config_set_clarett, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_clarett }, + { } + }, .level_input_count = 2, .air_input_count = 2, @@ -2107,7 +2224,10 @@ static const char * const scarlett2_spdif_clarett_texts[] = { }; static const struct scarlett2_device_info clarett_4pre_info = { - .config_set = &scarlett2_config_set_clarett, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_clarett }, + { } + }, .level_input_count = 2, .air_input_count = 4, @@ -2163,7 +2283,10 @@ static const struct scarlett2_device_info clarett_4pre_info = { }; static const struct scarlett2_device_info clarett_8pre_info = { - .config_set = &scarlett2_config_set_clarett, + .config_sets = (const struct scarlett2_config_set_entry[]) { + { 0, &scarlett2_config_set_clarett }, + { } + }, .level_input_count = 2, .air_input_count = 8, @@ -8211,10 +8334,32 @@ static void scarlett2_private_suspend(struct usb_mixer_interface *mixer) /*** Initialisation ***/ +/* Select the config_set matching the running firmware version. + * + * The device info's config_sets array is ordered by ascending + * from_firmware_version; pick the last entry whose version is <= the + * running firmware version. If the running firmware is older than the + * first entry's from_firmware_version (i.e. older than the driver's + * minimum supported version for this device), the first entry's + * config_set is selected anyway so firmware updates can still be done + * (requires only the ACK handler), but the usual mixer controls + * aren't created. + */ +static void scarlett2_resolve_config_set(struct scarlett2_data *private) +{ + const struct scarlett2_config_set_entry *entry = + private->info->config_sets; + + private->config_set = entry->config_set; + for (entry++; entry->config_set; entry++) + if (entry->from_firmware_version <= private->firmware_version) + private->config_set = entry->config_set; +} + static void scarlett2_count_io(struct scarlett2_data *private) { const struct scarlett2_device_info *info = private->info; - const struct scarlett2_config_set *config_set = info->config_set; + const struct scarlett2_config_set *config_set = private->config_set; const int (*port_count)[SCARLETT2_PORT_DIRNS] = info->port_count; int port_type, srcs = 0, dsts = 0, i; @@ -8311,9 +8456,14 @@ static int scarlett2_init_private(struct usb_mixer_interface *mixer, mixer->private_suspend = scarlett2_private_suspend; private->info = entry->info; - private->config_set = entry->info->config_set; + + /* Set config_set to the first entry's config_set so the + * notify handler has a valid pointer while USB init runs; it + * is re-resolved once the firmware version has been read. + */ + private->config_set = entry->info->config_sets[0].config_set; + private->series_name = entry->series_name; - scarlett2_count_io(private); private->scarlett2_seq = 0; private->mixer = mixer; @@ -8717,6 +8867,13 @@ static int snd_scarlett2_controls_create( if (err < 0) return err; + /* Now that the firmware version is known, pick the matching + * config_set + */ + scarlett2_resolve_config_set(private); + + scarlett2_count_io(private); + /* Get the upgrade & settings flash segment numbers */ err = scarlett2_get_flash_segment_nums(mixer); if (err < 0) diff --git a/sound/usb/mixer_us16x08.c b/sound/usb/mixer_us16x08.c index 8a02964e5d7b6..ebff185cbd2c0 100644 --- a/sound/usb/mixer_us16x08.c +++ b/sound/usb/mixer_us16x08.c @@ -224,14 +224,14 @@ static int snd_us16x08_route_put(struct snd_kcontrol *kcontrol, err = snd_us16x08_send_urb(chip, buf, sizeof(route_msg)); - if (err > 0) { - elem->cached |= 1 << index; - elem->cache_val[index] = val; - } else { + if (err < 0) { usb_audio_dbg(chip, "Failed to set routing, err:%d\n", err); + return err; } - return err > 0 ? 1 : 0; + elem->cached |= 1 << index; + elem->cache_val[index] = val; + return 1; } static int snd_us16x08_master_info(struct snd_kcontrol *kcontrol, @@ -283,14 +283,14 @@ static int snd_us16x08_master_put(struct snd_kcontrol *kcontrol, buf[5] = index + 1; err = snd_us16x08_send_urb(chip, buf, sizeof(mix_msg_out)); - if (err > 0) { - elem->cached |= 1 << index; - elem->cache_val[index] = val; - } else { + if (err < 0) { usb_audio_dbg(chip, "Failed to set master, err:%d\n", err); + return err; } - return err > 0 ? 1 : 0; + elem->cached |= 1 << index; + elem->cache_val[index] = val; + return 1; } static int snd_us16x08_bus_put(struct snd_kcontrol *kcontrol, @@ -324,14 +324,14 @@ static int snd_us16x08_bus_put(struct snd_kcontrol *kcontrol, break; } - if (err > 0) { - elem->cached |= 1; - elem->cache_val[0] = val; - } else { + if (err < 0) { usb_audio_dbg(chip, "Failed to set bus parameter, err:%d\n", err); + return err; } - return err > 0 ? 1 : 0; + elem->cached |= 1; + elem->cache_val[0] = val; + return 1; } static int snd_us16x08_bus_get(struct snd_kcontrol *kcontrol, @@ -392,14 +392,14 @@ static int snd_us16x08_channel_put(struct snd_kcontrol *kcontrol, err = snd_us16x08_send_urb(chip, buf, sizeof(mix_msg_in)); - if (err > 0) { - elem->cached |= 1 << index; - elem->cache_val[index] = val; - } else { + if (err < 0) { usb_audio_dbg(chip, "Failed to set channel, err:%d\n", err); + return err; } - return err > 0 ? 1 : 0; + elem->cached |= 1 << index; + elem->cache_val[index] = val; + return 1; } static int snd_us16x08_mix_info(struct snd_kcontrol *kcontrol, @@ -435,6 +435,7 @@ static int snd_us16x08_comp_put(struct snd_kcontrol *kcontrol, int index = ucontrol->id.index; char buf[sizeof(comp_msg)]; int val_idx, val; + int threshold, ratio, attack, release, gain, switch_on; int err; val = ucontrol->value.integer.value[0]; @@ -447,36 +448,61 @@ static int snd_us16x08_comp_put(struct snd_kcontrol *kcontrol, /* new control value incl. bias*/ val_idx = elem->head.id - SND_US16X08_ID_COMP_BASE; - store->val[val_idx][index] = ucontrol->value.integer.value[0]; + threshold = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_THRESHOLD)] + [index]; + ratio = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_RATIO)][index]; + attack = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_ATTACK)][index]; + release = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_RELEASE)] + [index]; + gain = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_GAIN)][index]; + switch_on = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_SWITCH)] + [index]; + + switch (val_idx) { + case COMP_STORE_IDX(SND_US16X08_ID_COMP_THRESHOLD): + threshold = val; + break; + case COMP_STORE_IDX(SND_US16X08_ID_COMP_RATIO): + ratio = val; + break; + case COMP_STORE_IDX(SND_US16X08_ID_COMP_ATTACK): + attack = val; + break; + case COMP_STORE_IDX(SND_US16X08_ID_COMP_RELEASE): + release = val; + break; + case COMP_STORE_IDX(SND_US16X08_ID_COMP_GAIN): + gain = val; + break; + case COMP_STORE_IDX(SND_US16X08_ID_COMP_SWITCH): + switch_on = val; + break; + } /* prepare compressor URB message from template */ memcpy(buf, comp_msg, sizeof(comp_msg)); /* place comp values in message buffer watch bias! */ - buf[8] = store->val[ - COMP_STORE_IDX(SND_US16X08_ID_COMP_THRESHOLD)][index] - - SND_US16X08_COMP_THRESHOLD_BIAS; - buf[11] = ratio_map[store->val[ - COMP_STORE_IDX(SND_US16X08_ID_COMP_RATIO)][index]]; - buf[14] = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_ATTACK)][index] - + SND_US16X08_COMP_ATTACK_BIAS; - buf[17] = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_RELEASE)][index] - + SND_US16X08_COMP_RELEASE_BIAS; - buf[20] = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_GAIN)][index]; - buf[26] = store->val[COMP_STORE_IDX(SND_US16X08_ID_COMP_SWITCH)][index]; + buf[8] = threshold - SND_US16X08_COMP_THRESHOLD_BIAS; + buf[11] = ratio_map[ratio]; + buf[14] = attack + SND_US16X08_COMP_ATTACK_BIAS; + buf[17] = release + SND_US16X08_COMP_RELEASE_BIAS; + buf[20] = gain; + buf[26] = switch_on; /* place channel selector in message buffer */ buf[5] = index + 1; err = snd_us16x08_send_urb(chip, buf, sizeof(comp_msg)); - if (err > 0) { - elem->cached |= 1 << index; - elem->cache_val[index] = val; - } else { + if (err < 0) { usb_audio_dbg(chip, "Failed to set compressor, err:%d\n", err); + return err; } + store->val[val_idx][index] = val; + elem->cached |= 1 << index; + elem->cache_val[index] = val; return 1; } @@ -529,13 +555,13 @@ static int snd_us16x08_eqswitch_put(struct snd_kcontrol *kcontrol, msleep(15); } - if (err > 0) { - elem->cached |= 1 << index; - elem->cache_val[index] = val; - } else { + if (err < 0) { usb_audio_dbg(chip, "Failed to set eq switch, err:%d\n", err); + return err; } + elem->cached |= 1 << index; + elem->cache_val[index] = val; return 1; } @@ -578,11 +604,10 @@ static int snd_us16x08_eq_put(struct snd_kcontrol *kcontrol, /* copy URB buffer from EQ template */ memcpy(buf, eqs_msq, sizeof(eqs_msq)); - store->val[b_idx][p_idx][index] = val; - buf[20] = store->val[b_idx][3][index]; - buf[17] = store->val[b_idx][2][index]; - buf[14] = store->val[b_idx][1][index]; - buf[11] = store->val[b_idx][0][index]; + buf[20] = p_idx == 3 ? val : store->val[b_idx][3][index]; + buf[17] = p_idx == 2 ? val : store->val[b_idx][2][index]; + buf[14] = p_idx == 1 ? val : store->val[b_idx][1][index]; + buf[11] = p_idx == 0 ? val : store->val[b_idx][0][index]; /* place channel index in URB buffer */ buf[5] = index + 1; @@ -592,14 +617,15 @@ static int snd_us16x08_eq_put(struct snd_kcontrol *kcontrol, err = snd_us16x08_send_urb(chip, buf, sizeof(eqs_msq)); - if (err > 0) { - /* store new value in EQ band cache */ - elem->cached |= 1 << index; - elem->cache_val[index] = val; - } else { + if (err < 0) { usb_audio_dbg(chip, "Failed to set eq param, err:%d\n", err); + return err; } + store->val[b_idx][p_idx][index] = val; + /* store new value in EQ band cache */ + elem->cached |= 1 << index; + elem->cache_val[index] = val; return 1; } @@ -1418,4 +1444,3 @@ int snd_us16x08_controls_create(struct usb_mixer_interface *mixer) return 0; } - diff --git a/sound/usb/qcom/mixer_usb_offload.c b/sound/usb/qcom/mixer_usb_offload.c index 2adeb64f4d33f..005138714f724 100644 --- a/sound/usb/qcom/mixer_usb_offload.c +++ b/sound/usb/qcom/mixer_usb_offload.c @@ -113,7 +113,7 @@ int snd_usb_offload_create_ctl(struct snd_usb_audio *chip, struct device *bedev) struct snd_usb_substream *subs; struct snd_usb_stream *as; char ctl_name[48]; - int ret; + int ret = 0; list_for_each_entry(as, &chip->pcm_list, list) { subs = &as->substream[SNDRV_PCM_STREAM_PLAYBACK]; diff --git a/sound/usb/qcom/qc_audio_offload.c b/sound/usb/qcom/qc_audio_offload.c index a0009503b2c59..ed833a43db635 100644 --- a/sound/usb/qcom/qc_audio_offload.c +++ b/sound/usb/qcom/qc_audio_offload.c @@ -794,15 +794,23 @@ static void qmi_stop_session(void) continue; } /* Release XHCI endpoints */ - if (info->data_ep_pipe) + if (info->data_ep_pipe) { ep = usb_pipe_endpoint(uadev[pcm_card_num].udev, info->data_ep_pipe); - xhci_sideband_remove_endpoint(uadev[pcm_card_num].sb, ep); + if (ep) + xhci_sideband_remove_endpoint(uadev[pcm_card_num].sb, + ep); + info->data_ep_pipe = 0; + } - if (info->sync_ep_pipe) + if (info->sync_ep_pipe) { ep = usb_pipe_endpoint(uadev[pcm_card_num].udev, info->sync_ep_pipe); - xhci_sideband_remove_endpoint(uadev[pcm_card_num].sb, ep); + if (ep) + xhci_sideband_remove_endpoint(uadev[pcm_card_num].sb, + ep); + info->sync_ep_pipe = 0; + } disable_audio_stream(subs); } @@ -1157,6 +1165,7 @@ uaudio_endpoint_setup(struct snd_usb_substream *subs, tr_pa = page_to_phys(pg); mem_info->dma = sg_dma_address(sgt->sgl); sg_free_table(sgt); + kfree(sgt); /* data transfer ring */ iova = uaudio_iommu_map_pa(MEM_XFER_RING, dma_coherent, tr_pa, @@ -1226,6 +1235,7 @@ static int uaudio_event_ring_setup(struct snd_usb_substream *subs, er_pa = page_to_phys(pg); mem_info->dma = sg_dma_address(sgt->sgl); sg_free_table(sgt); + kfree(sgt); iova = uaudio_iommu_map_pa(MEM_EVENT_RING, dma_coherent, er_pa, PAGE_SIZE); @@ -1615,8 +1625,13 @@ static void handle_uaudio_stream_req(struct qmi_handle *handle, if (req_msg->service_interval_valid) { ret = get_data_interval_from_si(subs, req_msg->service_interval); - if (ret == -EINVAL) + if (ret == -EINVAL) { + if (req_msg->enable) { + guard(mutex)(&chip->mutex); + subs->opened = 0; + } goto response; + } datainterval = ret; } @@ -1637,6 +1652,11 @@ static void handle_uaudio_stream_req(struct qmi_handle *handle, subs->opened = 0; } } else { + if (info_idx < 0) { + ret = -EINVAL; + goto response; + } + info = &uadev[pcm_card_num].info[info_idx]; if (info->data_ep_pipe) { ep = usb_pipe_endpoint(uadev[pcm_card_num].udev, diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index f4e885f96e838..ed66e2af95143 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -2365,6 +2365,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x1397, 0x0509, /* Behringer UMC404HD */ QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB), + DEVICE_FLG(0x1397, 0x050c, /* Behringer Flow 8 */ + QUIRK_FLAG_IFB_SILENCE_ON_EMPTY), DEVICE_FLG(0x13e5, 0x0001, /* Serato Phono */ QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x152a, 0x880a, /* NeuralDSP Quad Cortex */ @@ -2606,6 +2608,7 @@ static const char *const snd_usb_audio_quirk_flag_names[] = { QUIRK_STRING_ENTRY(SKIP_IFACE_SETUP), QUIRK_STRING_ENTRY(MIXER_PLAYBACK_LINEAR_VOL), QUIRK_STRING_ENTRY(MIXER_CAPTURE_LINEAR_VOL), + QUIRK_STRING_ENTRY(IFB_SILENCE_ON_EMPTY), NULL }; diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 58fd07f8c3c97..9afcad8f143a0 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -236,6 +236,12 @@ extern bool snd_usb_skip_validation; * QUIRK_FLAG_MIXER_CAPTURE_LINEAR_VOL * Similar to QUIRK_FLAG_MIXER_PLAYBACK_LINEAR_VOL, but for capture streams. * Overrides QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE + * QUIRK_FLAG_IFB_SILENCE_ON_EMPTY + * In implicit feedback mode, when an entire capture URB returns with + * all iso_frame_desc[i].status != 0 (bytes==0), do not silently return + * from snd_usb_handle_sync_urb. Instead fall through and enqueue a + * packet_info containing only size-0 packets, so the OUT ring keeps + * moving (emits silence). Needed by Behringer Flow 8 (1397:050c). */ enum { @@ -268,6 +274,7 @@ enum { QUIRK_TYPE_SKIP_IFACE_SETUP = 26, QUIRK_TYPE_MIXER_PLAYBACK_LINEAR_VOL = 27, QUIRK_TYPE_MIXER_CAPTURE_LINEAR_VOL = 28, + QUIRK_TYPE_IFB_SILENCE_ON_EMPTY = 29, /* Please also edit snd_usb_audio_quirk_flag_names */ }; @@ -302,5 +309,6 @@ enum { #define QUIRK_FLAG_SKIP_IFACE_SETUP QUIRK_FLAG(SKIP_IFACE_SETUP) #define QUIRK_FLAG_MIXER_PLAYBACK_LINEAR_VOL QUIRK_FLAG(MIXER_PLAYBACK_LINEAR_VOL) #define QUIRK_FLAG_MIXER_CAPTURE_LINEAR_VOL QUIRK_FLAG(MIXER_CAPTURE_LINEAR_VOL) +#define QUIRK_FLAG_IFB_SILENCE_ON_EMPTY QUIRK_FLAG(IFB_SILENCE_ON_EMPTY) #endif /* __USBAUDIO_H */ diff --git a/sound/usb/usx2y/us144mkii.c b/sound/usb/usx2y/us144mkii.c index 94553b61013c5..58ef23146f209 100644 --- a/sound/usb/usx2y/us144mkii.c +++ b/sound/usb/usx2y/us144mkii.c @@ -585,19 +585,24 @@ static void tascam_disconnect(struct usb_interface *intf) return; if (intf->cur_altsetting->desc.bInterfaceNumber == 0) { - /* Ensure all deferred work is complete before freeing resources */ snd_card_disconnect(tascam->card); - cancel_work_sync(&tascam->stop_work); - cancel_work_sync(&tascam->capture_work); - cancel_work_sync(&tascam->midi_in_work); - cancel_work_sync(&tascam->midi_out_work); - cancel_work_sync(&tascam->stop_pcm_work); + /* + * Kill the URBs before cancelling the work, so a late URB + * completion cannot re-arm a work that then runs after + * snd_card_free(). + */ usb_kill_anchored_urbs(&tascam->playback_anchor); usb_kill_anchored_urbs(&tascam->capture_anchor); usb_kill_anchored_urbs(&tascam->feedback_anchor); usb_kill_anchored_urbs(&tascam->midi_in_anchor); usb_kill_anchored_urbs(&tascam->midi_out_anchor); + + cancel_work_sync(&tascam->stop_work); + cancel_work_sync(&tascam->capture_work); + cancel_work_sync(&tascam->midi_in_work); + cancel_work_sync(&tascam->midi_out_work); + cancel_work_sync(&tascam->stop_pcm_work); timer_delete_sync(&tascam->error_timer); tascam_free_urbs(tascam); snd_card_free(tascam->card); diff --git a/sound/usb/usx2y/us144mkii_capture.c b/sound/usb/usx2y/us144mkii_capture.c index af120bf621733..fa01da98151aa 100644 --- a/sound/usb/usx2y/us144mkii_capture.c +++ b/sound/usb/usx2y/us144mkii_capture.c @@ -302,7 +302,6 @@ void capture_urb_complete(struct urb *urb) } usb_get_urb(urb); - usb_anchor_urb(urb, &tascam->capture_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); if (ret < 0) { dev_err_ratelimited(tascam->card->dev, @@ -312,6 +311,7 @@ void capture_urb_complete(struct urb *urb) usb_put_urb(urb); atomic_dec( &tascam->active_urbs); /* Decrement on failed resubmission */ + return; } out: usb_put_urb(urb); diff --git a/sound/virtio/virtio_kctl.c b/sound/virtio/virtio_kctl.c index ffb903d56297a..45f7b6a5b3080 100644 --- a/sound/virtio/virtio_kctl.c +++ b/sound/virtio/virtio_kctl.c @@ -18,6 +18,21 @@ static const snd_ctl_elem_type_t g_v2a_type_map[] = { [VIRTIO_SND_CTL_TYPE_IEC958] = SNDRV_CTL_ELEM_TYPE_IEC958 }; +/* Map for converting VirtIO types to maximum value counts. */ +static const unsigned int g_v2a_count_map[] = { + [VIRTIO_SND_CTL_TYPE_BOOLEAN] = + ARRAY_SIZE(((struct virtio_snd_ctl_value *)0)->value.integer), + [VIRTIO_SND_CTL_TYPE_INTEGER] = + ARRAY_SIZE(((struct virtio_snd_ctl_value *)0)->value.integer), + [VIRTIO_SND_CTL_TYPE_INTEGER64] = + ARRAY_SIZE(((struct virtio_snd_ctl_value *)0)->value.integer64), + [VIRTIO_SND_CTL_TYPE_ENUMERATED] = + ARRAY_SIZE(((struct virtio_snd_ctl_value *)0)->value.enumerated), + [VIRTIO_SND_CTL_TYPE_BYTES] = + ARRAY_SIZE(((struct virtio_snd_ctl_value *)0)->value.bytes), + [VIRTIO_SND_CTL_TYPE_IEC958] = 1 +}; + /* Map for converting VirtIO access rights to ALSA access rights. */ static const unsigned int g_v2a_access_map[] = { [VIRTIO_SND_CTL_ACCESS_READ] = SNDRV_CTL_ELEM_ACCESS_READ, @@ -36,6 +51,37 @@ static const unsigned int g_v2a_mask_map[] = { [VIRTIO_SND_CTL_EVT_MASK_TLV] = SNDRV_CTL_EVENT_MASK_TLV }; +static int virtsnd_kctl_validate_info(struct virtio_snd *snd, u32 cid, + struct virtio_snd_ctl_info *kinfo) +{ + struct virtio_device *vdev = snd->vdev; + unsigned int type = le32_to_cpu(kinfo->type); + unsigned int count = le32_to_cpu(kinfo->count); + + if (type >= ARRAY_SIZE(g_v2a_type_map)) { + dev_err(&vdev->dev, "control #%u: unknown type %u\n", + cid, type); + return -EINVAL; + } + + if (count > g_v2a_count_map[type] || + (type == VIRTIO_SND_CTL_TYPE_IEC958 && count != 1)) { + dev_err(&vdev->dev, "control #%u: invalid count %u for type %u\n", + cid, count, type); + return -EINVAL; + } + + if (type == VIRTIO_SND_CTL_TYPE_ENUMERATED && + !le32_to_cpu(kinfo->value.enumerated.items)) { + dev_err(&vdev->dev, + "control #%u: no items for enumerated control\n", + cid); + return -EINVAL; + } + + return 0; +} + /** * virtsnd_kctl_info() - Returns information about the control. * @kcontrol: ALSA control element. @@ -385,6 +431,10 @@ int virtsnd_kctl_parse_cfg(struct virtio_snd *snd) struct virtio_snd_ctl_info *kinfo = &snd->kctl_infos[i]; unsigned int type = le32_to_cpu(kinfo->type); + rc = virtsnd_kctl_validate_info(snd, i, kinfo); + if (rc) + return rc; + if (type == VIRTIO_SND_CTL_TYPE_ENUMERATED) { rc = virtsnd_kctl_get_enum_items(snd, i); if (rc) diff --git a/sound/virtio/virtio_pcm.c b/sound/virtio/virtio_pcm.c index eb9cc8131905d..be3893de40a55 100644 --- a/sound/virtio/virtio_pcm.c +++ b/sound/virtio/virtio_pcm.c @@ -77,7 +77,8 @@ static const struct virtsnd_v2a_rate g_v2a_rate_map[] = { [VIRTIO_SND_PCM_RATE_88200] = { SNDRV_PCM_RATE_88200, 88200 }, [VIRTIO_SND_PCM_RATE_96000] = { SNDRV_PCM_RATE_96000, 96000 }, [VIRTIO_SND_PCM_RATE_176400] = { SNDRV_PCM_RATE_176400, 176400 }, - [VIRTIO_SND_PCM_RATE_192000] = { SNDRV_PCM_RATE_192000, 192000 } + [VIRTIO_SND_PCM_RATE_192000] = { SNDRV_PCM_RATE_192000, 192000 }, + [VIRTIO_SND_PCM_RATE_384000] = { SNDRV_PCM_RATE_384000, 384000 } }; /** diff --git a/sound/virtio/virtio_pcm_ops.c b/sound/virtio/virtio_pcm_ops.c index 6297a9c61e70b..1105e7ff3523a 100644 --- a/sound/virtio/virtio_pcm_ops.c +++ b/sound/virtio/virtio_pcm_ops.c @@ -90,7 +90,8 @@ static const struct virtsnd_a2v_rate g_a2v_rate_map[] = { { 88200, VIRTIO_SND_PCM_RATE_88200 }, { 96000, VIRTIO_SND_PCM_RATE_96000 }, { 176400, VIRTIO_SND_PCM_RATE_176400 }, - { 192000, VIRTIO_SND_PCM_RATE_192000 } + { 192000, VIRTIO_SND_PCM_RATE_192000 }, + { 384000, VIRTIO_SND_PCM_RATE_384000 } }; static int virtsnd_pcm_sync_stop(struct snd_pcm_substream *substream); diff --git a/sound/xen/xen_snd_front_alsa.c b/sound/xen/xen_snd_front_alsa.c index dc626480123ac..a6dd196f73d66 100644 --- a/sound/xen/xen_snd_front_alsa.c +++ b/sound/xen/xen_snd_front_alsa.c @@ -378,7 +378,7 @@ static int alsa_open(struct snd_pcm_substream *substream) stream_clear(stream); - xen_snd_front_evtchnl_pair_set_connected(stream->evt_pair, true); + xen_snd_front_evtchnl_set_connected(&stream->evt_pair->req, true); ret = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_FORMAT, alsa_hw_rule, stream, @@ -498,6 +498,8 @@ static int alsa_hw_free(struct snd_pcm_substream *substream) struct xen_snd_front_pcm_stream_info *stream = stream_get(substream); int ret; + xen_snd_front_evtchnl_set_connected(&stream->evt_pair->evt, false); + ret = xen_snd_front_stream_close(&stream->evt_pair->req); stream_free(stream); return ret; @@ -532,6 +534,7 @@ static int alsa_prepare(struct snd_pcm_substream *substream) return ret; stream->is_open = true; + xen_snd_front_evtchnl_set_connected(&stream->evt_pair->evt, true); } return 0; @@ -571,20 +574,24 @@ void xen_snd_front_alsa_handle_cur_pos(struct xen_snd_front_evtchnl *evtchnl, { struct snd_pcm_substream *substream = evtchnl->u.evt.substream; struct xen_snd_front_pcm_stream_info *stream = stream_get(substream); + struct snd_pcm_runtime *runtime = substream->runtime; snd_pcm_uframes_t delta, new_hw_ptr, cur_frame; - cur_frame = bytes_to_frames(substream->runtime, pos_bytes); + if (!runtime->buffer_size || !runtime->period_size) + return; + + cur_frame = bytes_to_frames(runtime, pos_bytes); delta = cur_frame - stream->be_cur_frame; stream->be_cur_frame = cur_frame; new_hw_ptr = (snd_pcm_uframes_t)atomic_read(&stream->hw_ptr); - new_hw_ptr = (new_hw_ptr + delta) % substream->runtime->buffer_size; + new_hw_ptr = (new_hw_ptr + delta) % runtime->buffer_size; atomic_set(&stream->hw_ptr, (int)new_hw_ptr); stream->out_frames += delta; - if (stream->out_frames > substream->runtime->period_size) { - stream->out_frames %= substream->runtime->period_size; + if (stream->out_frames > runtime->period_size) { + stream->out_frames %= runtime->period_size; snd_pcm_period_elapsed(substream); } } diff --git a/sound/xen/xen_snd_front_evtchnl.c b/sound/xen/xen_snd_front_evtchnl.c index bc03f71bf16e3..17a30452c0cca 100644 --- a/sound/xen/xen_snd_front_evtchnl.c +++ b/sound/xen/xen_snd_front_evtchnl.c @@ -94,6 +94,9 @@ static irqreturn_t evtchnl_interrupt_evt(int irq, void *dev_id) guard(mutex)(&channel->ring_io_lock); + if (unlikely(channel->state != EVTCHNL_STATE_CONNECTED)) + return IRQ_HANDLED; + prod = page->in_prod; /* Ensure we see ring contents up to prod. */ virt_rmb(); @@ -430,8 +433,8 @@ int xen_snd_front_evtchnl_publish_all(struct xen_snd_front_info *front_info) return ret; } -void xen_snd_front_evtchnl_pair_set_connected(struct xen_snd_front_evtchnl_pair *evt_pair, - bool is_connected) +void xen_snd_front_evtchnl_set_connected(struct xen_snd_front_evtchnl *channel, + bool is_connected) { enum xen_snd_front_evtchnl_state state; @@ -440,13 +443,16 @@ void xen_snd_front_evtchnl_pair_set_connected(struct xen_snd_front_evtchnl_pair else state = EVTCHNL_STATE_DISCONNECTED; - scoped_guard(mutex, &evt_pair->req.ring_io_lock) { - evt_pair->req.state = state; + scoped_guard(mutex, &channel->ring_io_lock) { + channel->state = state; } +} - scoped_guard(mutex, &evt_pair->evt.ring_io_lock) { - evt_pair->evt.state = state; - } +void xen_snd_front_evtchnl_pair_set_connected(struct xen_snd_front_evtchnl_pair *evt_pair, + bool is_connected) +{ + xen_snd_front_evtchnl_set_connected(&evt_pair->req, is_connected); + xen_snd_front_evtchnl_set_connected(&evt_pair->evt, is_connected); } void xen_snd_front_evtchnl_pair_clear(struct xen_snd_front_evtchnl_pair *evt_pair) @@ -456,7 +462,11 @@ void xen_snd_front_evtchnl_pair_clear(struct xen_snd_front_evtchnl_pair *evt_pai } scoped_guard(mutex, &evt_pair->evt.ring_io_lock) { - evt_pair->evt.evt_next_id = 0; + evt_pair->evt.evt_id = 0; + /* Drop obsolete events queued for the previous stream instance. */ + evt_pair->evt.u.evt.page->in_cons = + evt_pair->evt.u.evt.page->in_prod; + /* Ensure the consumer index is visible before stream reuse. */ + virt_wmb(); } } - diff --git a/sound/xen/xen_snd_front_evtchnl.h b/sound/xen/xen_snd_front_evtchnl.h index 3675fba705647..f6ebdb09c0298 100644 --- a/sound/xen/xen_snd_front_evtchnl.h +++ b/sound/xen/xen_snd_front_evtchnl.h @@ -37,9 +37,9 @@ struct xen_snd_front_evtchnl { /* State of the event channel. */ enum xen_snd_front_evtchnl_state state; enum xen_snd_front_evtchnl_type type; - /* Either response id or incoming event id. */ + /* Current response id or next expected incoming event id. */ u16 evt_id; - /* Next request id or next expected event id. */ + /* Next request id. */ u16 evt_next_id; /* Shared ring access lock. */ struct mutex ring_io_lock; @@ -77,6 +77,8 @@ void xen_snd_front_evtchnl_free_all(struct xen_snd_front_info *front_info); int xen_snd_front_evtchnl_publish_all(struct xen_snd_front_info *front_info); void xen_snd_front_evtchnl_flush(struct xen_snd_front_evtchnl *evtchnl); +void xen_snd_front_evtchnl_set_connected(struct xen_snd_front_evtchnl *channel, + bool is_connected); void xen_snd_front_evtchnl_pair_set_connected(struct xen_snd_front_evtchnl_pair *evt_pair, bool is_connected); diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c index ec356deb27c9e..ce69d1e5468e3 100644 --- a/tools/bpf/bpftool/cgroup.c +++ b/tools/bpf/bpftool/cgroup.c @@ -78,6 +78,13 @@ static unsigned int query_flags; static struct btf *btf_vmlinux; static __u32 btf_vmlinux_id; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; + btf_vmlinux_id = 0; +} + static enum bpf_attach_type parse_attach_type(const char *str) { const char *attach_type_str; @@ -388,6 +395,8 @@ static int do_show(int argc, char **argv) if (json_output) jsonw_end_array(json_wtr); + free_btf_vmlinux(); + exit_cgroup: close(cgroup_fd); exit: @@ -437,7 +446,9 @@ static int do_show_tree_fn(const char *fpath, const struct stat *sb, printf("%s\n", fpath); } - btf_vmlinux = libbpf_find_kernel_btf(); + if (!btf_vmlinux) + btf_vmlinux = libbpf_find_kernel_btf(); + for (i = 0; i < ARRAY_SIZE(cgroup_attach_types); i++) show_bpf_progs(cgroup_fd, cgroup_attach_types[i], ftw->level); @@ -540,6 +551,7 @@ static int do_show_tree(int argc, char **argv) if (json_output) jsonw_end_array(json_wtr); + free_btf_vmlinux(); free(cgroup_alloced); return ret; diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 2f9e10752e288..d6040b52d4e1b 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -1399,7 +1399,7 @@ static int do_skeleton(int argc, char **argv) continue; if (use_loader) - printf("t\tint %s_fd;\n", ident); + printf("\t\tint %s_fd;\n", ident); else printf("\t\tstruct bpf_link *%s;\n", ident); } diff --git a/tools/bpf/bpftool/net.c b/tools/bpf/bpftool/net.c index 974189da8a916..dba28755d2849 100644 --- a/tools/bpf/bpftool/net.c +++ b/tools/bpf/bpftool/net.c @@ -603,14 +603,14 @@ static int query_flow_dissector(struct bpf_attach_info *attach_info) &attach_flags, prog_ids, &prog_cnt); close(fd); if (err) { - if (errno == EINVAL) { + if (err == -EINVAL) { /* Older kernel's don't support querying * flow dissector programs. */ errno = 0; return 0; } - p_err("can't query prog: %s", strerror(errno)); + p_err("can't query prog: %s", strerror(-err)); return -1; } diff --git a/tools/include/nolibc/getopt.h b/tools/include/nolibc/getopt.h index 87565e3b6a339..3ad140f692dfe 100644 --- a/tools/include/nolibc/getopt.h +++ b/tools/include/nolibc/getopt.h @@ -71,7 +71,7 @@ int getopt(int argc, char * const argv[], const char *optstring) d = optstring[i++]; } while (d && d != c); - if (d != c || c == ':') { + if (!d || d != c || c == ':') { optopt = c; if (optstring[0] != ':' && opterr) fprintf(stderr, "%s: unrecognized option: %c\n", argv[0], *optchar); diff --git a/tools/include/nolibc/stackprotector.h b/tools/include/nolibc/stackprotector.h index ae8b1d3a374dc..4d1aa5371c77c 100644 --- a/tools/include/nolibc/stackprotector.h +++ b/tools/include/nolibc/stackprotector.h @@ -42,7 +42,8 @@ uintptr_t __stack_chk_guard; static __no_stack_protector void __stack_chk_init(void) { - __nolibc_syscall3(__NR_getrandom, &__stack_chk_guard, sizeof(__stack_chk_guard), 0); + __nolibc_syscall3(__NR_getrandom, &__stack_chk_guard, sizeof(__stack_chk_guard), + GRND_INSECURE | GRND_NONBLOCK); /* a bit more randomness in case getrandom() fails, ensure the guard is never 0 */ if (__stack_chk_guard != (uintptr_t) &__stack_chk_guard) __stack_chk_guard ^= (uintptr_t) &__stack_chk_guard; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 677be9a473476..9bbf35d5f141f 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -6698,6 +6698,7 @@ struct bpf_prog_info { __u32 verified_insns; __u32 attach_btf_obj_id; __u32 attach_btf_id; + __u32 :32; } __attribute__((aligned(8))); struct bpf_map_info { @@ -6719,6 +6720,7 @@ struct bpf_map_info { __u64 map_extra; __aligned_u64 hash; __u32 hash_size; + __u32 :32; } __attribute__((aligned(8))); struct bpf_btf_info { diff --git a/tools/include/uapi/linux/if_link.h b/tools/include/uapi/linux/if_link.h index 7e46ca4cd31bb..757ce5e9426e9 100644 --- a/tools/include/uapi/linux/if_link.h +++ b/tools/include/uapi/linux/if_link.h @@ -1526,6 +1526,8 @@ enum { IFLA_BOND_MISSED_MAX, IFLA_BOND_NS_IP6_TARGET, IFLA_BOND_COUPLED_CONTROL, + IFLA_BOND_BROADCAST_NEIGH, + IFLA_BOND_LACP_STRICT, __IFLA_BOND_MAX, }; diff --git a/tools/lib/api/fs/fs.c b/tools/lib/api/fs/fs.c index edec23406dbc6..cbd8eab0d1df0 100644 --- a/tools/lib/api/fs/fs.c +++ b/tools/lib/api/fs/fs.c @@ -261,8 +261,8 @@ static const char *mount_overload(struct fs *fs) /* "PERF_" + name + "_ENVIRONMENT" + '\0' */ char upper_name[5 + name_len + 12 + 1]; - snprintf(upper_name, name_len, "PERF_%s_ENVIRONMENT", fs->name); - mem_toupper(upper_name, name_len); + snprintf(upper_name, sizeof(upper_name), "PERF_%s_ENVIRONMENT", fs->name); + mem_toupper(upper_name, strlen(upper_name)); return getenv(upper_name) ?: *fs->mounts; } @@ -294,11 +294,14 @@ int filename__read_int(const char *filename, int *value) { char line[64]; int fd = open(filename, O_RDONLY), err = -1; + ssize_t n; if (fd < 0) return -errno; - if (read(fd, line, sizeof(line)) > 0) { + n = read(fd, line, sizeof(line) - 1); + if (n > 0) { + line[n] = '\0'; *value = atoi(line); err = 0; } @@ -312,11 +315,14 @@ static int filename__read_ull_base(const char *filename, { char line[64]; int fd = open(filename, O_RDONLY), err = -1; + ssize_t n; if (fd < 0) return -errno; - if (read(fd, line, sizeof(line)) > 0) { + n = read(fd, line, sizeof(line) - 1); + if (n > 0) { + line[n] = '\0'; *value = strtoull(line, NULL, base); if (*value != ULLONG_MAX) err = 0; @@ -370,12 +376,13 @@ int filename__write_int(const char *filename, int value) { int fd = open(filename, O_WRONLY), err = -1; char buf[64]; + int len; if (fd < 0) return -errno; - sprintf(buf, "%d", value); - if (write(fd, buf, sizeof(buf)) == sizeof(buf)) + len = sprintf(buf, "%d", value); + if (write(fd, buf, len) == len) err = 0; close(fd); diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index ceb57b46a8782..771aeaa0262b2 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -4578,12 +4578,14 @@ static int btf_dedup_prep(struct btf_dedup *d) case BTF_KIND_RESTRICT: case BTF_KIND_PTR: case BTF_KIND_FWD: - case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: case BTF_KIND_TYPE_TAG: h = btf_hash_common(t); break; + case BTF_KIND_TYPEDEF: + h = btf_hash_typedef(t); + break; case BTF_KIND_INT: case BTF_KIND_DECL_TAG: h = btf_hash_int_decl_tag(t); diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index 9478b8f78f260..40004aa3af71b 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -398,13 +398,12 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) blob_fd_array_off(gen, i)); emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); emit(gen, BPF_EXIT_INSN()); - if (OPTS_GET(gen->opts, gen_hash, false)) - compute_sha_update_offsets(gen); - - pr_debug("gen: finish %s\n", errstr(gen->error)); if (!gen->error) { struct gen_loader_opts *opts = gen->opts; + if (OPTS_GET(opts, gen_hash, false)) + compute_sha_update_offsets(gen); + opts->insns = gen->insn_start; opts->insns_sz = gen->insn_cur - gen->insn_start; opts->data = gen->data_start; @@ -419,6 +418,7 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) bpf_insn_bswap(insn++); } } + pr_debug("gen: finish %s\n", errstr(gen->error)); return gen->error; } @@ -545,13 +545,22 @@ void bpf_gen__map_create(struct bpf_gen *gen, default: break; } - /* conditionally update max_entries */ - if (map_idx >= 0) + + /* + * Conditionally update max_entries from the host-supplied loader + * ctx. This sizes the map at runtime, but for a signed loader + * (gen_hash) it would let an untrusted host re-dimension the + * program's maps after emit_signature_match(), outside what the + * signature attests to. Keep the signer-provided max_entries + * baked into the blob in that case. + */ + if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false)) move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, sizeof(struct bpf_loader_ctx) + sizeof(struct bpf_map_desc) * map_idx + offsetof(struct bpf_map_desc, max_entries), true /* check that max_entries != 0 */); + /* emit MAP_CREATE command */ emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", @@ -585,6 +594,23 @@ static void emit_signature_match(struct bpf_gen *gen) __s64 off; int i; + /* + * Reject if the metadata map is not exclusive. Without exclusivity + * the cached map->sha[] verified above can be stale: another BPF + * program with map access could have mutated the contents between + * BPF_OBJ_GET_INFO_BY_FD and loader execution. + */ + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, + 0, 0, 0, 0)); + emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, SHA256_DIGEST_LENGTH)); + off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; + if (is_simm16(off)) { + emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); + emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, off)); + } else { + gen->error = -ERANGE; + } + for (i = 0; i < SHA256_DWORD_SIZE; i++) { emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 0, 0, 0, 0)); @@ -1053,7 +1079,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, prog_idx, prog_type, insns_off, insn_cnt, license_off); /* convert blob insns to target endianness */ - if (gen->swapped_endian) { + if (gen->swapped_endian && !gen->error) { struct bpf_insn *insn = gen->data_start + insns_off; int i; @@ -1091,7 +1117,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, sizeof(struct bpf_core_relo)); /* convert all info blobs to target endianness */ - if (gen->swapped_endian) + if (gen->swapped_endian && !gen->error) info_blob_bswap(gen, func_info, line_info, core_relos, load_attr); libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); @@ -1169,27 +1195,36 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, value = add_data(gen, pvalue, value_size); key = add_data(gen, &zero, sizeof(zero)); - /* if (map_desc[map_idx].initial_value) { + /* + * if (map_desc[map_idx].initial_value) { * if (ctx->flags & BPF_SKEL_KERNEL) * bpf_probe_read_kernel(value, value_size, initial_value); * else * bpf_copy_from_user(value, value_size, initial_value); * } + * + * The runtime initial_value comes from the host-supplied loader + * ctx and would overwrite the blob value after emit_signature_match() + * has already validated map->sha[]. For a signed loader (gen_hash) + * the attested blob value must be authoritative, so skip the override + * and leave the hashed value in place. */ - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, initial_value))); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, value)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, - offsetof(struct bpf_loader_ctx, flags))); - emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); + if (!OPTS_GET(gen->opts, gen_hash, false)) { + emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, + sizeof(struct bpf_loader_ctx) + + sizeof(struct bpf_map_desc) * map_idx + + offsetof(struct bpf_map_desc, initial_value))); + emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value)); + emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); + emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, + offsetof(struct bpf_loader_ctx, flags))); + emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); + } map_update_attr = add_data(gen, &attr, attr_size); pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 3a80a018fc7d5..83aae7a39d36d 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -12280,7 +12280,7 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) { DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); - unsigned long offset = 0; + long offset = 0; const char *func_name; char *func; int n; @@ -12302,6 +12302,13 @@ static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf pr_warn("kprobe name is invalid: %s\n", func_name); return -EINVAL; } + + if (offset < 0) { + free(func); + pr_warn("kprobe offset must be a non-negative integer: %li\n", offset); + return -EINVAL; + } + if (opts.retprobe && offset != 0) { free(func); pr_warn("kretprobes do not support offset specification\n"); diff --git a/tools/lib/bpf/strset.c b/tools/lib/bpf/strset.c index 2464bcbd04e03..ace73c6b3d62b 100644 --- a/tools/lib/bpf/strset.c +++ b/tools/lib/bpf/strset.c @@ -107,6 +107,41 @@ static void *strset_add_str_mem(struct strset *set, size_t add_sz) set->strs_data_len, set->strs_data_max_len, add_sz); } +static long strset_str_append(struct strset *set, const char *s) +{ + uintptr_t old_data = (uintptr_t)set->strs_data; + size_t old_data_len = set->strs_data_len; + uintptr_t old_s = (uintptr_t)s; + long len = strlen(s) + 1; + void *p; + + /* + * Hashmap keys are always offsets within set->strs_data, so to even + * look up some string from the "outside", we need to first append it + * at the end, so that it can be addressed with an offset. Luckily, + * until set->strs_data_len is incremented, that string is just a piece + * of garbage for the rest of the code, so no harm, no foul. On the + * other hand, if the string is unique, it's already appended and + * ready to be used, only a simple set->strs_data_len increment away. + */ + p = strset_add_str_mem(set, len); + if (!p) + return -ENOMEM; + + /* + * The set->strs_data might have reallocated and if 's' pointed + * to an internal string within the old buffer, then it became + * dangling and needs to be reconstructed before the copy. + */ + if (old_data && old_data != (uintptr_t)set->strs_data && + old_s >= old_data && old_s < old_data + old_data_len) + s = set->strs_data + (old_s - old_data); + + memcpy(p, s, len); + + return len; +} + /* Find string offset that corresponds to a given string *s*. * Returns: * - >0 offset into string data, if string is found; @@ -116,16 +151,12 @@ static void *strset_add_str_mem(struct strset *set, size_t add_sz) int strset__find_str(struct strset *set, const char *s) { long old_off, new_off, len; - void *p; - /* see strset__add_str() for why we do this */ - len = strlen(s) + 1; - p = strset_add_str_mem(set, len); - if (!p) - return -ENOMEM; + len = strset_str_append(set, s); + if (len < 0) + return len; new_off = set->strs_data_len; - memcpy(p, s, len); if (hashmap__find(set->strs_hash, new_off, &old_off)) return old_off; @@ -142,24 +173,13 @@ int strset__find_str(struct strset *set, const char *s) int strset__add_str(struct strset *set, const char *s) { long old_off, new_off, len; - void *p; int err; - /* Hashmap keys are always offsets within set->strs_data, so to even - * look up some string from the "outside", we need to first append it - * at the end, so that it can be addressed with an offset. Luckily, - * until set->strs_data_len is incremented, that string is just a piece - * of garbage for the rest of the code, so no harm, no foul. On the - * other hand, if the string is unique, it's already appended and - * ready to be used, only a simple set->strs_data_len increment away. - */ - len = strlen(s) + 1; - p = strset_add_str_mem(set, len); - if (!p) - return -ENOMEM; + len = strset_str_append(set, s); + if (len < 0) + return len; new_off = set->strs_data_len; - memcpy(p, s, len); /* Now attempt to add the string, but only if the string with the same * contents doesn't exist already (HASHMAP_ADD strategy). If such diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index e3710933fd52a..57fb82bb81b58 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -468,10 +468,10 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt) { - char path[PATH_MAX], line[PATH_MAX], mode[16]; + char path[PATH_MAX], line[4096], mode[16]; size_t seg_start, seg_end, seg_off; struct elf_seg *seg; - int tmp_pid, i, err; + int tmp_pid, n, i, err; FILE *f; *seg_cnt = 0; @@ -480,8 +480,13 @@ static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, * /proc//root/. They will be reported as just / in * /proc//maps. */ - if (sscanf(lib_path, "/proc/%d/root%s", &tmp_pid, path) == 2 && pid == tmp_pid) + /* %n is not counted in sscanf() return value, so initialize it. */ + n = 0; + if (sscanf(lib_path, "/proc/%d/root%n", &tmp_pid, &n) == 1 && + n > 0 && pid == tmp_pid && lib_path[n] == '/') { + libbpf_strlcpy(path, lib_path + n, sizeof(path)); goto proceed; + } if (!realpath(lib_path, path)) { pr_warn("usdt: failed to get absolute path of '%s' (err %s), using path as is...\n", @@ -504,8 +509,11 @@ static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, * 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so * 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0 * 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so + * + * Some VMA names can be longer than the local buffer. Bound the + * writes, but still consume the rest of the line. */ - while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n", + while (fscanf(f, "%zx-%zx %15s %zx %*s %*d%4095[^\n]%*[^\n]\n", &seg_start, &seg_end, mode, &seg_off, line) == 5) { void *tmp; diff --git a/tools/lib/symbol/kallsyms.c b/tools/lib/symbol/kallsyms.c index e335ac2b9e197..d64bd9cc82a90 100644 --- a/tools/lib/symbol/kallsyms.c +++ b/tools/lib/symbol/kallsyms.c @@ -60,7 +60,7 @@ int kallsyms__parse(const char *filename, void *arg, read_to_eol(&io); continue; } - for (i = 0; i < sizeof(symbol_name); i++) { + for (i = 0; i < KSYM_NAME_LEN; i++) { ch = io__get_char(&io); if (ch < 0 || ch == '\n') break; @@ -68,6 +68,9 @@ int kallsyms__parse(const char *filename, void *arg, } symbol_name[i] = '\0'; + if (i == KSYM_NAME_LEN) + read_to_eol(&io); + err = process_symbol(arg, symbol_name, symbol_type, start); if (err) break; diff --git a/tools/mm/slabinfo.c b/tools/mm/slabinfo.c index 54c7265ab52d9..583bb699404ed 100644 --- a/tools/mm/slabinfo.c +++ b/tools/mm/slabinfo.c @@ -33,7 +33,7 @@ struct slabinfo { unsigned int hwcache_align, object_size, objs_per_slab; unsigned int sanity_checks, slab_size, store_user, trace; int order, poison, reclaim_account, red_zone; - unsigned long partial, objects, slabs, objects_partial, objects_total; + unsigned long partial, objects, slabs, objects_partial, total_objects; unsigned long alloc_fastpath, alloc_slowpath; unsigned long free_fastpath, free_slowpath; unsigned long free_frozen, free_add_partial, free_remove_partial; @@ -798,7 +798,7 @@ static void slab_debug(struct slabinfo *s) fprintf(stderr, "%s can only enable trace for one slab at a time\n", s->name); } if (!tracing && s->trace) - set_obj(s, "trace", 1); + set_obj(s, "trace", 0); } static void totals(void) @@ -1263,7 +1263,7 @@ static void read_slab_dir(void) slab->object_size = get_obj("object_size"); slab->objects = get_obj("objects"); slab->objects_partial = get_obj("objects_partial"); - slab->objects_total = get_obj("objects_total"); + slab->total_objects = get_obj("total_objects"); slab->objs_per_slab = get_obj("objs_per_slab"); slab->order = get_obj("order"); slab->partial = get_obj("partial"); diff --git a/tools/net/ynl/Makefile b/tools/net/ynl/Makefile index d514a48dae270..3cefe4ed96cb9 100644 --- a/tools/net/ynl/Makefile +++ b/tools/net/ynl/Makefile @@ -22,7 +22,7 @@ tests: | lib generated libynl.a ynltool: | lib generated libynl.a libynl.a: | lib generated @echo -e "\tAR $@" - @ar rcs $@ lib/ynl.o generated/*-user.o + @$(AR) rcs $@ lib/ynl.o generated/*-user.o $(SUBDIRS): @if [ -f "$@/Makefile" ] ; then \ diff --git a/tools/net/ynl/generated/Makefile b/tools/net/ynl/generated/Makefile index 86e1e4a959a75..ea4128f612d6c 100644 --- a/tools/net/ynl/generated/Makefile +++ b/tools/net/ynl/generated/Makefile @@ -37,7 +37,7 @@ all: protos.a $(HDRS) $(SRCS) $(KHDRS) $(KSRCS) $(UAPI) $(RSTS) protos.a: $(OBJS) @echo -e "\tAR $@" - @ar rcs $@ $(OBJS) + @$(AR) rcs $@ $(OBJS) %-user.h: $(SPECS_DIR)/%.yaml $(TOOL) @echo -e "\tGEN $@" diff --git a/tools/net/ynl/lib/Makefile b/tools/net/ynl/lib/Makefile index 4b2b98704ff9b..9b98c05996000 100644 --- a/tools/net/ynl/lib/Makefile +++ b/tools/net/ynl/lib/Makefile @@ -15,7 +15,7 @@ all: ynl.a ynl.a: $(OBJS) @echo -e "\tAR $@" - @ar rcs $@ $(OBJS) + @$(AR) rcs $@ $(OBJS) clean: rm -f *.o *.d *~ diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index f3df2bde119fc..58631d62011d6 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -27,27 +27,16 @@ static ssize_t demangled_name_len(const char *name); -static inline u32 str_hash(const char *str) -{ - return jhash(str, strlen(str), 0); -} - -static inline u32 str_hash_demangled(const char *str) +u32 str_hash_demangled(const char *str) { return jhash(str, demangled_name_len(str), 0); } -#define __elf_table(name) (elf->name##_hash) -#define __elf_bits(name) (elf->name##_bits) - -#define __elf_table_entry(name, key) \ - __elf_table(name)[hash_min(key, __elf_bits(name))] - #define elf_hash_add(name, node, key) \ ({ \ struct elf_hash_node *__node = node; \ - __node->next = __elf_table_entry(name, key); \ - __elf_table_entry(name, key) = __node; \ + __node->next = __elf_table_entry(elf, name, key); \ + __elf_table_entry(elf, name, key) = __node; \ }) static inline void __elf_hash_del(struct elf_hash_node *node, @@ -69,30 +58,20 @@ static inline void __elf_hash_del(struct elf_hash_node *node, } #define elf_hash_del(name, node, key) \ - __elf_hash_del(node, &__elf_table_entry(name, key)) - -#define elf_list_entry(ptr, type, member) \ -({ \ - typeof(ptr) __ptr = (ptr); \ - __ptr ? container_of(__ptr, type, member) : NULL; \ -}) - -#define elf_hash_for_each_possible(name, obj, member, key) \ - for (obj = elf_list_entry(__elf_table_entry(name, key), typeof(*obj), member); \ - obj; \ - obj = elf_list_entry(obj->member.next, typeof(*(obj)), member)) + __elf_hash_del(node, &__elf_table_entry(elf, name, key)) #define elf_alloc_hash(name, size) \ ({ \ - __elf_bits(name) = max(10, ilog2(size)); \ - __elf_table(name) = mmap(NULL, sizeof(struct elf_hash_node *) << __elf_bits(name), \ + __elf_bits(elf, name) = max(10, ilog2(size)); \ + __elf_table(elf, name) = mmap(NULL, \ + sizeof(struct elf_hash_node *) << __elf_bits(elf, name), \ PROT_READ|PROT_WRITE, \ MAP_PRIVATE|MAP_ANON, -1, 0); \ - if (__elf_table(name) == (void *)-1L) { \ + if (__elf_table(elf, name) == (void *)-1L) { \ ERROR_GLIBC("mmap fail " #name); \ - __elf_table(name) = NULL; \ + __elf_table(elf, name) = NULL; \ } \ - __elf_table(name); \ + __elf_table(elf, name); \ }) static inline unsigned long __sym_start(struct symbol *s) @@ -141,7 +120,7 @@ struct section *find_section_by_name(const struct elf *elf, const char *name) { struct section *sec; - elf_hash_for_each_possible(section_name, sec, name_hash, str_hash(name)) { + elf_hash_for_each_possible(elf, section_name, sec, name_hash, str_hash(name)) { if (!strcmp(sec->name, name)) return sec; } @@ -154,7 +133,7 @@ static struct section *find_section_by_index(struct elf *elf, { struct section *sec; - elf_hash_for_each_possible(section, sec, hash, idx) { + elf_hash_for_each_possible(elf, section, sec, hash, idx) { if (sec->idx == idx) return sec; } @@ -166,7 +145,7 @@ static struct symbol *find_symbol_by_index(struct elf *elf, unsigned int idx) { struct symbol *sym; - elf_hash_for_each_possible(symbol, sym, hash, idx) { + elf_hash_for_each_possible(elf, symbol, sym, hash, idx) { if (sym->idx == idx) return sym; } @@ -285,7 +264,7 @@ struct symbol *find_symbol_by_name(const struct elf *elf, const char *name) { struct symbol *sym; - elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash(name)) { if (!strcmp(sym->name, name)) return sym; } @@ -300,7 +279,7 @@ static struct symbol *find_local_symbol_by_file_and_name(const struct elf *elf, { struct symbol *sym; - elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) { + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash_demangled(name)) { if (sym->bind == STB_LOCAL && sym->file == file && !strcmp(sym->name, name)) { return sym; @@ -314,7 +293,7 @@ struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *nam { struct symbol *sym; - elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) { + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash_demangled(name)) { if (!strcmp(sym->name, name) && !is_local_sym(sym)) return sym; } @@ -322,21 +301,9 @@ struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *nam return NULL; } -void iterate_global_symbol_by_demangled_name(const struct elf *elf, - const char *demangled_name, - void (*process)(struct symbol *sym, void *data), - void *data) -{ - struct symbol *sym; - - elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(demangled_name)) { - if (!strcmp(sym->demangled_name, demangled_name) && !is_local_sym(sym)) - process(sym, data); - } -} - +/* If there are multiple matches, return the first one in the range */ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec, - unsigned long offset, unsigned int len) + unsigned long offset, unsigned int len) { struct reloc *reloc, *r = NULL; struct section *rsec; @@ -347,7 +314,7 @@ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *se return NULL; for_offset_range(o, offset, offset + len) { - elf_hash_for_each_possible(reloc, reloc, hash, + elf_hash_for_each_possible(elf, reloc, reloc, hash, sec_offset_hash(rsec, o)) { if (reloc->sec != rsec) continue; @@ -358,11 +325,11 @@ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *se r = reloc; } } - if (r) + if (r && (reloc_offset(r) & OFFSET_STRIDE_MASK) == o) return r; } - return NULL; + return r; } struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset) diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index 25573e5af76ef..b142984eb9b5a 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -21,6 +21,13 @@ #define SEC_NAME_LEN 1024 #define SYM_NAME_LEN 512 +static inline u32 str_hash(const char *str) +{ + return jhash(str, strlen(str), 0); +} + +u32 str_hash_demangled(const char *str); + #define bswap_if_needed(elf, val) __bswap_if_needed(&elf->ehdr, val) #ifdef LIBELF_USE_DEPRECATED @@ -130,6 +137,23 @@ struct elf { struct symbol *symbol_data; }; +#define __elf_table(elf, name) ((elf)->name##_hash) +#define __elf_bits(elf, name) ((elf)->name##_bits) + +#define __elf_table_entry(elf, name, key) \ + __elf_table(elf, name)[hash_min(key, __elf_bits(elf, name))] + +#define elf_list_entry(ptr, type, member) \ +({ \ + typeof(ptr) __ptr = (ptr); \ + __ptr ? container_of(__ptr, type, member) : NULL; \ +}) + +#define elf_hash_for_each_possible(elf, name, obj, member, key) \ + for (obj = elf_list_entry(__elf_table_entry(elf, name, key), typeof(*obj), member); \ + obj; \ + obj = elf_list_entry(obj->member.next, typeof(*(obj)), member)) + struct elf *elf_open_read(const char *name, int flags); struct elf *elf_create_file(GElf_Ehdr *ehdr, const char *name); @@ -186,9 +210,6 @@ struct symbol *find_func_by_offset(struct section *sec, unsigned long offset); struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset); struct symbol *find_symbol_by_name(const struct elf *elf, const char *name); struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *name); -void iterate_global_symbol_by_demangled_name(const struct elf *elf, const char *demangled_name, - void (*process)(struct symbol *sym, void *data), - void *data); struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset); int find_symbol_hole_containing(const struct section *sec, unsigned long offset); struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset); @@ -468,6 +489,11 @@ static inline void set_sym_next_reloc(struct reloc *reloc, struct reloc *next) #define for_each_sym_continue(elf, sym) \ list_for_each_entry_continue(sym, &elf->symbols, global_list) +#define for_each_sym_by_demangled_name(elf, name, sym) \ + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, \ + str_hash(name)) \ + if (strcmp(sym->demangled_name, name)) {} else + #define rsec_next_reloc(rsec, reloc) \ reloc_idx(reloc) < sec_num_entries(rsec) - 1 ? reloc + 1 : NULL diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c index 0b0d1503851ff..d0b3d1eef0521 100644 --- a/tools/objtool/klp-diff.c +++ b/tools/objtool/klp-diff.c @@ -46,11 +46,6 @@ static const struct option klp_diff_options[] = { static DEFINE_HASHTABLE(exports, 15); -static inline u32 str_hash(const char *str) -{ - return jhash(str, strlen(str), 0); -} - static char *escape_str(const char *orig) { size_t len = 0; @@ -242,25 +237,39 @@ static struct symbol *next_file_symbol(struct elf *elf, struct symbol *sym) static bool is_uncorrelated_static_local(struct symbol *sym) { static const char * const vars[] = { - "__already_done.", - "__func__.", - "__key.", - "__warned.", - "_entry.", - "_entry_ptr.", - "_rs.", - "descriptor.", - "CSWTCH.", + "__already_done", + "__func__", + "__key", + "__warned", + "_entry", + "_entry_ptr", + "_rs", + "descriptor", + "CSWTCH", }; + const char *dot; if (!is_object_sym(sym) || !is_local_sym(sym)) return false; - if (!strcmp(sym->sec->name, ".data.once")) + /* WARN_ONCE, etc */ + if (!strcmp(sym->sec->name, ".data..once")) return true; + dot = strchr(sym->name, '.'); + if (!dot) + return false; + for (int i = 0; i < ARRAY_SIZE(vars); i++) { - if (strstarts(sym->name, vars[i])) + size_t len = strlen(vars[i]); + + /* GCC: . */ + if (strstarts(sym->name, vars[i]) && (sym->name[len] == '.')) + return true; + + /* Clang: .[.] */ + if (strstarts(dot + 1, vars[i]) && + (dot[1 + len] == '.' || dot[1 + len] == '\0')) return true; } @@ -356,22 +365,6 @@ static bool dont_correlate(struct symbol *sym) strstarts(sym->name, "__initcall__"); } -struct process_demangled_name_data { - struct symbol *ret; - int count; -}; - -static void process_demangled_name(struct symbol *sym, void *d) -{ - struct process_demangled_name_data *data = d; - - if (sym->twin) - return; - - data->count++; - data->ret = sym; -} - /* * When there is no full name match, try match demangled_name. This would * match original foo.llvm.123 to patched foo.llvm.456. @@ -383,16 +376,23 @@ static void process_demangled_name(struct symbol *sym, void *d) static int find_global_symbol_by_demangled_name(struct elf *elf, struct symbol *sym, struct symbol **out_sym) { - struct process_demangled_name_data data = {}; + struct symbol *sym2, *result = NULL; + int count = 0; - iterate_global_symbol_by_demangled_name(elf, sym->demangled_name, - process_demangled_name, - &data); - if (data.count > 1) { - ERROR("Multiple (%d) correlation candidates for %s", data.count, sym->name); + for_each_sym_by_demangled_name(elf, sym->demangled_name, sym2) { + if (is_local_sym(sym2) || sym2->twin) + continue; + + count++; + result = sym2; + } + + if (count > 1) { + ERROR("Multiple (%d) correlation candidates for %s", count, sym->name); return -1; } - *out_sym = data.ret; + + *out_sym = result; return 0; } @@ -655,7 +655,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym size_t size; /* bss doesn't have data */ - if (patched_sym->sec->data->d_buf) + if (patched_sym->sec->data && patched_sym->sec->data->d_buf) data = patched_sym->sec->data->d_buf + patched_sym->offset; if (is_sec_sym(patched_sym)) @@ -979,6 +979,13 @@ static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc) /* No dedicated section; find the symbol manually */ sym = find_symbol_containing(sec, arch_adjusted_addend(reloc)); if (!sym) { + /* + * This is presumably an .altinstr_replacement section which is + * empty due to it only having zero-length replacement(s). + */ + if (!sec_size(sec)) + return 1; + /* * This can happen for special section references to weak code * whose symbol has been stripped by the linker. @@ -999,6 +1006,9 @@ static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc) */ static int convert_reloc_sym(struct elf *elf, struct reloc *reloc) { + if (reloc_type(reloc) == R_NONE) + return 1; + if (is_reloc_allowed(reloc)) return 0; @@ -1239,6 +1249,7 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) for_each_reloc(patched_rsec, patched_reloc) { unsigned long offset; + int ret; if (reloc_offset(patched_reloc) < start || reloc_offset(patched_reloc) >= end) @@ -1252,12 +1263,15 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux")) continue; - if (convert_reloc_sym(e->patched, patched_reloc)) { + ret = convert_reloc_sym(e->patched, patched_reloc); + if (ret < 0) { ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc), "failed to convert reloc sym '%s' to its proper format", patched_reloc->sym->name); return -1; } + if (ret > 0) + continue; offset = out_sym->offset + (reloc_offset(patched_reloc) - patched_sym->offset); @@ -1334,7 +1348,7 @@ static int create_fake_symbols(struct elf *elf) sec = find_section_by_name(elf, ".discard.annotate_data"); if (!sec || !sec->rsec) - return 0; + goto entsize; for_each_reloc(sec->rsec, reloc) { unsigned long offset, size; @@ -1366,7 +1380,7 @@ static int create_fake_symbols(struct elf *elf) /* * 2) Make symbols for sh_entsize, and simple arrays of pointers: */ - +entsize: for_each_sec(elf, sec) { unsigned int entry_size; unsigned long offset; @@ -1400,6 +1414,7 @@ static int create_fake_symbols(struct elf *elf) /* Keep a special section entry if it references an included function */ static bool should_keep_special_sym(struct elf *elf, struct symbol *sym) { + bool annotate_insn = !strcmp(sym->sec->name, ".discard.annotate_insn"); struct reloc *reloc; if (is_sec_sym(sym) || !sym->sec->rsec) @@ -1409,7 +1424,16 @@ static bool should_keep_special_sym(struct elf *elf, struct symbol *sym) if (convert_reloc_sym(elf, reloc)) continue; - if (is_func_sym(reloc->sym) && reloc->sym->included) + if (!reloc->sym->clone || is_undef_sym(reloc->sym->clone)) + continue; + + /* + * Keep special section references to cloned functions. + * In some cases annotate_insn can also reference cloned alt + * replacement fake symbols; keep those references as well. + */ + if (is_func_sym(reloc->sym) || + (annotate_insn && is_notype_sym(reloc->sym))) return true; } @@ -1553,15 +1577,28 @@ static int clone_special_section(struct elfs *e, struct section *patched_sec) /* Extract only the needed bits from special sections */ static int clone_special_sections(struct elfs *e) { - struct section *patched_sec; + struct section *sec, *annotate_insn = NULL; - for_each_sec(e->patched, patched_sec) { - if (is_special_section(patched_sec)) { - if (clone_special_section(e, patched_sec)) + for_each_sec(e->patched, sec) { + if (is_special_section(sec)) { + if (!strcmp(sec->name, ".discard.annotate_insn")) { + annotate_insn = sec; + continue; + } + if (clone_special_section(e, sec)) return -1; } } + /* + * Do .discard.annotate_insn last, it can reference other special + * sections (alt replacements) so they need to be cloned first. + */ + if (annotate_insn) { + if (clone_special_section(e, annotate_insn)) + return -1; + } + return 0; } diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 333ddd0e4bd81..33f90968c8a95 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -109,6 +109,7 @@ endif ifeq ($(ARCH),s390) CFLAGS += -fPIC + CXXFLAGS += -fPIC endif ifeq ($(ARCH),mips) diff --git a/tools/perf/arch/common.c b/tools/perf/arch/common.c index 21836f70f231e..ad0cab830a4da 100644 --- a/tools/perf/arch/common.c +++ b/tools/perf/arch/common.c @@ -237,5 +237,7 @@ int perf_env__lookup_objdump(struct perf_env *env, char **path) */ bool perf_env__single_address_space(struct perf_env *env) { - return strcmp(perf_env__arch(env), "sparc"); + const char *arch = perf_env__arch(env); + + return strcmp(arch, "s390") && strcmp(arch, "sparc"); } diff --git a/tools/perf/arch/powerpc/util/auxtrace.c b/tools/perf/arch/powerpc/util/auxtrace.c index e39deff6c857a..4600a1661b4fe 100644 --- a/tools/perf/arch/powerpc/util/auxtrace.c +++ b/tools/perf/arch/powerpc/util/auxtrace.c @@ -71,6 +71,12 @@ struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, struct evsel *pos; int found = 0; + /* + * Set err value to zero here. Any fail later + * will set appropriate return code to err. + */ + *err = 0; + evlist__for_each_entry(evlist, pos) { if (strstarts(pos->name, "vpa_dtl")) { found = 1; diff --git a/tools/perf/arch/x86/tests/amd-ibs-period.c b/tools/perf/arch/x86/tests/amd-ibs-period.c index cee9e11c05e08..6a92b3a23ed7a 100644 --- a/tools/perf/arch/x86/tests/amd-ibs-period.c +++ b/tools/perf/arch/x86/tests/amd-ibs-period.c @@ -932,7 +932,7 @@ static bool kernel_v6_15_or_newer(void) endptr++; minor = strtol(endptr, NULL, 10); - return major >= 6 && minor >= 15; + return major > 6 || (major == 6 && minor >= 15); } int test__amd_ibs_period(struct test_suite *test __maybe_unused, diff --git a/tools/perf/bench/inject-buildid.c b/tools/perf/bench/inject-buildid.c index aad572a78d7fc..bfd2c5ec9488e 100644 --- a/tools/perf/bench/inject-buildid.c +++ b/tools/perf/bench/inject-buildid.c @@ -228,9 +228,12 @@ static ssize_t synthesize_sample(struct bench_data *data, struct bench_dso *dso, event.header.type = PERF_RECORD_SAMPLE; event.header.misc = PERF_RECORD_MISC_USER; - event.header.size = perf_event__sample_event_size(&sample, bench_sample_type, 0); - - perf_event__synthesize_sample(&event, bench_sample_type, 0, &sample); + event.header.size = perf_event__sample_event_size(&sample, bench_sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0); + perf_event__synthesize_sample(&event, bench_sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0, &sample); return writen(data->input_pipe[1], &event, event.header.size); } diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c index 5e57b78548f42..bdc7adc83b471 100644 --- a/tools/perf/builtin-annotate.c +++ b/tools/perf/builtin-annotate.c @@ -298,7 +298,8 @@ static int process_sample_event(const struct perf_tool *tool, goto out_put; } - if (ann->cpu_list && !test_bit(sample->cpu, ann->cpu_bitmap)) + if (ann->cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, ann->cpu_bitmap))) goto out_put; if (!al.filtered && diff --git a/tools/perf/builtin-buildid-list.c b/tools/perf/builtin-buildid-list.c index a91bbb34ac946..e0881b0ac38ff 100644 --- a/tools/perf/builtin-buildid-list.c +++ b/tools/perf/builtin-buildid-list.c @@ -61,7 +61,7 @@ static int sysfs__fprintf_build_id(FILE *fp) int ret; ret = sysfs__snprintf_build_id("/", sbuild_id, sizeof(sbuild_id)); - if (ret != sizeof(sbuild_id)) + if (ret + 1 != sizeof(sbuild_id)) return ret < 0 ? ret : -EINVAL; return fprintf(fp, "%s\n", sbuild_id); @@ -73,7 +73,7 @@ static int filename__fprintf_build_id(const char *name, FILE *fp) int ret; ret = filename__snprintf_build_id(name, sbuild_id, sizeof(sbuild_id)); - if (ret != sizeof(sbuild_id)) + if (ret + 1 != sizeof(sbuild_id)) return ret < 0 ? ret : -EINVAL; return fprintf(fp, "%s\n", sbuild_id); diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index 72a7802775ee9..e7b9909e1ac24 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -180,7 +180,8 @@ static void c2c_he_free(void *he) c2c_he = container_of(he, struct c2c_hist_entry, he); if (c2c_he->hists) { - hists__delete_entries(&c2c_he->hists->hists); + hists__delete_all_entries(&c2c_he->hists->hists); + perf_hpp__reset_output_field(&c2c_he->hists->list); zfree(&c2c_he->hists); } @@ -221,6 +222,8 @@ he__get_c2c_hists(struct hist_entry *he, ret = c2c_hists__init(hists, sort, nr_header_lines, env); if (ret) { + perf_hpp__reset_output_field(&hists->list); + c2c_he->hists = NULL; free(hists); return NULL; } @@ -241,6 +244,10 @@ static void c2c_he__set_cpu(struct c2c_hist_entry *c2c_he, "WARNING: no sample cpu value")) return; + /* cpuset bitmap has c2c.cpus_cnt bits from env->nr_cpus_avail */ + if (sample->cpu >= (unsigned int)c2c.cpus_cnt) + return; + __set_bit(sample->cpu, c2c_he->cpuset); } @@ -258,6 +265,10 @@ static void c2c_he__set_node(struct c2c_hist_entry *c2c_he, if (WARN_ONCE(node < 0, "WARNING: failed to find node\n")) return; + /* nodeset bitmap has c2c.nodes_cnt bits from env->nr_numa_nodes */ + if (node >= c2c.nodes_cnt) + return; + __set_bit(node, c2c_he->nodeset); if (c2c_he->paddr != sample->phys_addr) { @@ -386,7 +397,12 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, * Doing node stats only for single callchain data. */ int cpu = sample->cpu == (unsigned int) -1 ? 0 : sample->cpu; - int node = c2c.cpu2node[cpu]; + int node; + + /* cpu2node[] has c2c.cpus_cnt entries; large u32 wraps signed negative */ + if (cpu < 0 || cpu >= c2c.cpus_cnt) + cpu = 0; + node = c2c.cpu2node[cpu]; c2c_hists = he__get_c2c_hists(he, c2c.cl_sort, 2, machine->env); if (!c2c_hists) { @@ -405,7 +421,9 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, c2c_he = container_of(he, struct c2c_hist_entry, he); c2c_add_stats(&c2c_he->stats, &stats); c2c_add_stats(&c2c_hists->stats, &stats); - c2c_add_stats(&c2c_he->node_stats[node], &stats); + /* node_stats[] has c2c.nodes_cnt entries */ + if (node >= 0 && node < c2c.nodes_cnt) + c2c_add_stats(&c2c_he->node_stats[node], &stats); compute_stats(c2c_he, &stats, sample->weight); @@ -2351,6 +2369,10 @@ static int setup_nodes(struct perf_session *session) nodes[node] = set; perf_cpu_map__for_each_cpu_skip_any(cpu, idx, map) { + /* topology CPU IDs from perf.data may exceed nr_cpus_avail */ + if (cpu.cpu < 0 || cpu.cpu >= c2c.cpus_cnt) + continue; + __set_bit(cpu.cpu, set); if (WARN_ONCE(cpu2node[cpu.cpu] != -1, "node/cpu topology bug")) diff --git a/tools/perf/builtin-diff.c b/tools/perf/builtin-diff.c index 1b3df868849a9..29db0afc53ca0 100644 --- a/tools/perf/builtin-diff.c +++ b/tools/perf/builtin-diff.c @@ -416,7 +416,8 @@ static int diff__process_sample_event(const struct perf_tool *tool, goto out; } - if (cpu_list && !test_bit(sample->cpu, cpu_bitmap)) { + if (cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, cpu_bitmap))) { ret = 0; goto out; } diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c index f174bc69cec45..7a64935b7e2b0 100644 --- a/tools/perf/builtin-inject.c +++ b/tools/perf/builtin-inject.c @@ -26,6 +26,7 @@ #include "util/synthetic-events.h" #include "util/thread.h" #include "util/namespaces.h" +#include "util/unwind.h" #include "util/util.h" #include "util/tsc.h" @@ -215,12 +216,23 @@ static int perf_event__repipe_op4_synth(const struct perf_tool *tool, return perf_event__repipe_synth(tool, event); } +static int perf_event__repipe_synth_cb(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + return perf_event__repipe_synth(tool, event); +} + static int perf_event__repipe_attr(const struct perf_tool *tool, union perf_event *event, struct evlist **pevlist) { struct perf_inject *inject = container_of(tool, struct perf_inject, tool); + struct perf_event_attr attr; + size_t n_ids; + u64 *ids; int ret; ret = perf_event__process_attr(tool, event, pevlist); @@ -231,7 +243,37 @@ static int perf_event__repipe_attr(const struct perf_tool *tool, if (!inject->output.is_pipe) return 0; - return perf_event__repipe_synth(tool, event); + if (!inject->itrace_synth_opts.set) + return perf_event__repipe_synth(tool, event); + + if (event->header.size < sizeof(struct perf_event_header) + sizeof(u64)) { + pr_err("Attribute event size %u is too small\n", event->header.size); + return -EINVAL; + } + + if (event->header.size - sizeof(event->header) < event->attr.attr.size) { + pr_err("Attribute event size %u is too small for attr.size %u\n", + event->header.size, event->attr.attr.size); + return -EINVAL; + } + + memset(&attr, 0, sizeof(attr)); + memcpy(&attr, &event->attr.attr, + min_t(size_t, sizeof(attr), (size_t)event->attr.attr.size)); + + n_ids = event->header.size - sizeof(event->header) - event->attr.attr.size; + n_ids /= sizeof(u64); + ids = perf_record_header_attr_id(event); + + attr.size = sizeof(struct perf_event_attr); + attr.sample_type &= ~PERF_SAMPLE_AUX; + + if (inject->itrace_synth_opts.add_last_branch) { + attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + return perf_event__synthesize_attr(tool, &attr, (u32)n_ids, ids, + perf_event__repipe_synth_cb); } static int perf_event__repipe_event_update(const struct perf_tool *tool, @@ -330,8 +372,8 @@ perf_inject__cut_auxtrace_sample(struct perf_inject *inject, union perf_event *event, struct perf_sample *sample) { - size_t sz1 = sample->aux_sample.data - (void *)event; - size_t sz2 = event->header.size - sample->aux_sample.size - sz1; + size_t sz1 = sample->aux_sample.data - (void *)event - sizeof(u64); + size_t sz2 = event->header.size - sample->aux_sample.size - (sz1 + sizeof(u64)); union perf_event *ev; if (inject->event_copy == NULL) { @@ -342,13 +384,12 @@ perf_inject__cut_auxtrace_sample(struct perf_inject *inject, ev = (union perf_event *)inject->event_copy; if (sz1 > event->header.size || sz2 > event->header.size || sz1 + sz2 > event->header.size || - sz1 < sizeof(struct perf_event_header) + sizeof(u64)) + sz1 < sizeof(struct perf_event_header)) return event; memcpy(ev, event, sz1); memcpy((void *)ev + sz1, (void *)event + event->header.size - sz2, sz2); ev->header.size = sz1 + sz2; - ((u64 *)((void *)ev + sz1))[-1] = 0; return ev; } @@ -368,14 +409,77 @@ static int perf_event__repipe_sample(const struct perf_tool *tool, struct perf_inject *inject = container_of(tool, struct perf_inject, tool); - if (evsel && evsel->handler) { + if (evsel == NULL) + return perf_event__repipe_synth(tool, event); + + if (evsel->handler) { inject_handler f = evsel->handler; return f(tool, event, sample, evsel, machine); } build_id__mark_dso_hit(tool, event, sample, evsel, machine); - if (inject->itrace_synth_opts.set && sample->aux_sample.size) { + if (inject->itrace_synth_opts.set && + (inject->itrace_synth_opts.last_branch || + inject->itrace_synth_opts.add_last_branch)) { + union perf_event *event_copy = (void *)inject->event_copy; + struct branch_stack dummy_bs = { .nr = 0, .hw_idx = 0 }; + int err; + size_t sz; + u64 orig_type = evsel->core.attr.sample_type; + u64 orig_branch_type = evsel->core.attr.branch_sample_type; + + struct branch_stack *orig_bs = sample->branch_stack; + + if (event_copy == NULL) { + inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE); + if (!inject->event_copy) + return -ENOMEM; + + event_copy = (void *)inject->event_copy; + } + + if (!sample->branch_stack) + sample->branch_stack = &dummy_bs; + + if (inject->itrace_synth_opts.add_last_branch) { + /* Temporarily add in type bits for synthesis. */ + evsel->core.attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + evsel->core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + evsel->core.attr.sample_type &= ~PERF_SAMPLE_AUX; + + sz = perf_event__sample_event_size(sample, evsel->core.attr.sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type); + + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + evsel->core.attr.sample_type = orig_type; + evsel->core.attr.branch_sample_type = orig_branch_type; + sample->branch_stack = orig_bs; + return -EFAULT; + } + + event_copy->header.type = PERF_RECORD_SAMPLE; + event_copy->header.misc = event->header.misc; + event_copy->header.size = sz; + + err = perf_event__synthesize_sample(event_copy, evsel->core.attr.sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type, sample); + + evsel->core.attr.sample_type = orig_type; + evsel->core.attr.branch_sample_type = orig_branch_type; + sample->branch_stack = orig_bs; + + if (err) { + pr_err("Failed to synthesize sample\n"); + return err; + } + event = event_copy; + } else if (inject->itrace_synth_opts.set && + (evsel->core.attr.sample_type & PERF_SAMPLE_AUX)) { event = perf_inject__cut_auxtrace_sample(inject, event, sample); if (IS_ERR(event)) return PTR_ERR(event); @@ -396,7 +500,7 @@ static int perf_event__convert_sample_callchain(const struct perf_tool *tool, struct callchain_cursor_node *node; struct thread *thread; u64 sample_type = evsel->core.attr.sample_type; - u32 sample_size = event->header.size; + size_t sz; u64 i, k; int ret; @@ -438,7 +542,8 @@ static int perf_event__convert_sample_callchain(const struct perf_tool *tool, node = cursor->first; for (k = 0; k < cursor->nr && i < PERF_MAX_STACK_DEPTH; k++) { - if (machine__kernel_ip(machine, node->ip)) + if (machine->single_address_space && + machine__kernel_ip(machine, node->ip)) /* kernel IPs were added already */; else if (node->ms.sym && node->ms.sym->inlined) /* we can't handle inlined callchains */; @@ -454,17 +559,25 @@ static int perf_event__convert_sample_callchain(const struct perf_tool *tool, out: memcpy(event_copy, event, sizeof(event->header)); - /* adjust sample size for stack and regs */ - sample_size -= sample->user_stack.size; - sample_size -= (hweight64(evsel->core.attr.sample_regs_user) + 1) * sizeof(u64); - sample_size += (sample->callchain->nr + 1) * sizeof(u64); - event_copy->header.size = sample_size; - /* remove sample_type {STACK,REGS}_USER for synthesize */ sample_type &= ~(PERF_SAMPLE_STACK_USER | PERF_SAMPLE_REGS_USER); - perf_event__synthesize_sample(event_copy, sample_type, - evsel->core.attr.read_format, sample); + sz = perf_event__sample_event_size(sample, sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event_copy->header.size = sz; + + ret = perf_event__synthesize_sample(event_copy, sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type, sample); + if (ret) { + pr_err("Failed to synthesize sample\n"); + return ret; + } return perf_event__repipe_synth(tool, event_copy); } @@ -1100,7 +1213,8 @@ static int perf_inject__sched_stat(const struct perf_tool *tool, sample_sw.period = sample->period; sample_sw.time = sample->time; perf_event__synthesize_sample(event_sw, evsel->core.attr.sample_type, - evsel->core.attr.read_format, &sample_sw); + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type, &sample_sw); build_id__mark_dso_hit(tool, event_sw, &sample_sw, evsel, machine); ret = perf_event__repipe(tool, event_sw, &sample_sw, machine); perf_sample__exit(&sample_sw); @@ -2434,12 +2548,27 @@ static int __cmd_inject(struct perf_inject *inject) * synthesized hardware events, so clear the feature flag. */ if (inject->itrace_synth_opts.set) { + struct evsel *evsel; + perf_header__clear_feat(&session->header, HEADER_AUXTRACE); - if (inject->itrace_synth_opts.last_branch || - inject->itrace_synth_opts.add_last_branch) + + evlist__for_each_entry(session->evlist, evsel) { + evsel->core.attr.sample_type &= ~PERF_SAMPLE_AUX; + } + + if (inject->itrace_synth_opts.add_last_branch) { perf_header__set_feat(&session->header, HEADER_BRANCH_STACK); + + evlist__for_each_entry(session->evlist, evsel) { + evsel->core.attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + if (evsel->core.attr.size < PERF_ATTR_SIZE_VER2) + evsel->core.attr.size = PERF_ATTR_SIZE_VER2; + evsel->core.attr.branch_sample_type |= + PERF_SAMPLE_BRANCH_HW_INDEX; + } + } } /* @@ -2562,6 +2691,9 @@ int cmd_inject(int argc, const char **argv) OPT_STRING(0, "guestmount", &symbol_conf.guestmount, "directory", "guest mount directory under which every guest os" " instance has a subdir"), + OPT_CALLBACK(0, "unwind-style", NULL, "unwind style", + "unwind styles (libdw,libunwind)", + unwind__option), OPT_BOOLEAN(0, "convert-callchain", &inject.convert_callchain, "Generate callchains using DWARF and drop register/stack data"), OPT_END() diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 4a5eba498c025..1593cf3a834bd 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -865,6 +865,7 @@ static int record__auxtrace_init(struct record *rec) } if (!rec->itr) { + err = -EINVAL; rec->itr = auxtrace_record__init(rec->evlist, &err); if (err) return err; @@ -3079,6 +3080,7 @@ static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits) static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask) { bitmap_free(mask->bits); + mask->bits = NULL; mask->nbits = 0; } diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index 95c0bdba6b116..3a5cf1b64adb0 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -48,6 +48,7 @@ #include "util/time-utils.h" #include "util/auxtrace.h" #include "util/units.h" +#include "util/unwind.h" #include "util/util.h" // perf_tip() #include "ui/ui.h" #include "ui/progress.h" @@ -299,7 +300,8 @@ static int process_sample_event(const struct perf_tool *tool, if (symbol_conf.hide_unresolved && al.sym == NULL) goto out_put; - if (rep->cpu_list && !test_bit(sample->cpu, rep->cpu_bitmap)) + if (rep->cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, rep->cpu_bitmap))) goto out_put; if (sort__mode == SORT_MODE__BRANCH) { @@ -1449,6 +1451,9 @@ int cmd_report(int argc, const char **argv) OPT_CALLBACK(0, "addr2line-style", NULL, "addr2line style", "addr2line styles (libdw,llvm,libbfd,addr2line)", report_parse_addr2line_config), + OPT_CALLBACK(0, "unwind-style", NULL, "unwind style", + "unwind styles (libdw,libunwind)", + unwind__option), OPT_BOOLEAN(0, "demangle", &symbol_conf.demangle, "Symbol demangling. Enabled by default, use --no-demangle to disable."), OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel, diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 555247568e7a6..b97b41c7655c6 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -54,6 +54,7 @@ #define COMM_LEN 20 #define SYM_LEN 129 #define MAX_PID 1024000 +#define PID_MAX_LIMIT 4194304 /* kernel limit on 64-bit */ #define MAX_PRIO 140 #define SEP_LEN 100 @@ -273,6 +274,7 @@ struct thread_runtime { u64 migrations; int prio; + bool color; }; /* per event run time data */ @@ -364,14 +366,25 @@ get_new_event(struct task_desc *task, u64 timestamp) struct sched_atom *event = zalloc(sizeof(*event)); unsigned long idx = task->nr_events; size_t size; + struct sched_atom **atoms_p; + + if (event == NULL) { + pr_err("ERROR: sched: failed to allocate event\n"); + return NULL; + } event->timestamp = timestamp; event->nr = idx; + size = sizeof(struct sched_atom *) * (task->nr_events + 1); + atoms_p = realloc(task->atoms, size); + if (!atoms_p) { + pr_err("ERROR: sched: failed to grow atoms array\n"); + free(event); + return NULL; + } + task->atoms = atoms_p; task->nr_events++; - size = sizeof(struct sched_atom *) * task->nr_events; - task->atoms = realloc(task->atoms, size); - BUG_ON(!task->atoms); task->atoms[idx] = event; @@ -402,6 +415,8 @@ static void add_sched_event_run(struct perf_sched *sched, struct task_desc *task } event = get_new_event(task, timestamp); + if (event == NULL) + return; event->type = SCHED_EVENT_RUN; event->duration = duration; @@ -415,6 +430,8 @@ static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *t struct sched_atom *event, *wakee_event; event = get_new_event(task, timestamp); + if (event == NULL) + return; event->type = SCHED_EVENT_WAKEUP; event->wakee = wakee; @@ -429,6 +446,10 @@ static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *t } wakee_event->wait_sem = zalloc(sizeof(*wakee_event->wait_sem)); + if (!wakee_event->wait_sem) { + pr_err("ERROR: sched: failed to allocate semaphore\n"); + return; + } sem_init(wakee_event->wait_sem, 0, 0); event->wait_sem = wakee_event->wait_sem; @@ -440,6 +461,9 @@ static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *ta { struct sched_atom *event = get_new_event(task, timestamp); + if (event == NULL) + return; + event->type = SCHED_EVENT_SLEEP; sched->nr_sleep_events++; @@ -448,17 +472,28 @@ static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *ta static struct task_desc *register_pid(struct perf_sched *sched, unsigned long pid, const char *comm) { - struct task_desc *task; + struct task_desc *task, **tasks_p; static int pid_max; + /* perf.data is untrusted — cap pid to prevent overflow in size calculations */ + if (pid >= PID_MAX_LIMIT) { + pr_err("pid %lu exceeds limit %d, skipping\n", pid, PID_MAX_LIMIT); + return NULL; + } + if (sched->pid_to_task == NULL) { if (sysctl__read_int("kernel/pid_max", &pid_max) < 0) pid_max = MAX_PID; - BUG_ON((sched->pid_to_task = calloc(pid_max, sizeof(struct task_desc *))) == NULL); + sched->pid_to_task = calloc(pid_max, sizeof(struct task_desc *)); + if (sched->pid_to_task == NULL) + return NULL; } if (pid >= (unsigned long)pid_max) { - BUG_ON((sched->pid_to_task = realloc(sched->pid_to_task, (pid + 1) * - sizeof(struct task_desc *))) == NULL); + void *p = realloc(sched->pid_to_task, (pid + 1) * sizeof(struct task_desc *)); + + if (p == NULL) + return NULL; + sched->pid_to_task = p; while (pid >= (unsigned long)pid_max) sched->pid_to_task[pid_max++] = NULL; } @@ -469,9 +504,11 @@ static struct task_desc *register_pid(struct perf_sched *sched, return task; task = zalloc(sizeof(*task)); + if (task == NULL) + return NULL; task->pid = pid; - task->nr = sched->nr_tasks; - strcpy(task->comm, comm); + if (comm) + strlcpy(task->comm, comm, sizeof(task->comm)); /* * every task starts in sleeping state - this gets ignored * if there's no wakeup pointing to this sleep state: @@ -479,10 +516,12 @@ static struct task_desc *register_pid(struct perf_sched *sched, add_sched_event_sleep(sched, task, 0); sched->pid_to_task[pid] = task; - sched->nr_tasks++; - sched->tasks = realloc(sched->tasks, sched->nr_tasks * sizeof(struct task_desc *)); - BUG_ON(!sched->tasks); - sched->tasks[task->nr] = task; + tasks_p = realloc(sched->tasks, (sched->nr_tasks + 1) * sizeof(struct task_desc *)); + if (!tasks_p) + return NULL; + sched->tasks = tasks_p; + sched->tasks[sched->nr_tasks] = task; + task->nr = sched->nr_tasks++; if (verbose > 0) printf("registered task #%ld, PID %ld (%s)\n", sched->nr_tasks, pid, comm); @@ -841,6 +880,8 @@ replay_wakeup_event(struct perf_sched *sched, waker = register_pid(sched, sample->tid, ""); wakee = register_pid(sched, pid, comm); + if (waker == NULL || wakee == NULL) + return -1; add_sched_event_wakeup(sched, waker, sample->time, wakee); return 0; @@ -882,6 +923,8 @@ static int replay_switch_event(struct perf_sched *sched, prev = register_pid(sched, prev_pid, prev_comm); next = register_pid(sched, next_pid, next_comm); + if (prev == NULL || next == NULL) + return -1; sched->cpu_last_switched[cpu] = timestamp; @@ -1147,7 +1190,12 @@ static int latency_switch_event(struct perf_sched *sched, int cpu = sample->cpu, err = -1; s64 delta; - BUG_ON(cpu >= MAX_CPUS || cpu < 0); + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (cpu >= MAX_CPUS || cpu < 0) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, cpu); + return 0; + } timestamp0 = sched->cpu_last_switched[cpu]; sched->cpu_last_switched[cpu] = timestamp; @@ -1177,7 +1225,7 @@ static int latency_switch_event(struct perf_sched *sched, } } if (add_sched_out_event(out_events, prev_state, timestamp)) - return -1; + goto out_put; in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid); if (!in_events) { @@ -1211,14 +1259,22 @@ static int latency_runtime_event(struct perf_sched *sched, const u32 pid = evsel__intval(evsel, sample, "pid"); const u64 runtime = evsel__intval(evsel, sample, "runtime"); struct thread *thread = machine__findnew_thread(machine, -1, pid); - struct work_atoms *atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid); + struct work_atoms *atoms; u64 timestamp = sample->time; int cpu = sample->cpu, err = -1; if (thread == NULL) return -1; - BUG_ON(cpu >= MAX_CPUS || cpu < 0); + atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid); + + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (cpu >= MAX_CPUS || cpu < 0) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, cpu); + err = 0; + goto out_put; + } if (!atoms) { if (thread_atoms_insert(sched, thread)) goto out_put; @@ -1541,22 +1597,32 @@ static int process_sched_wakeup_ignore(const struct perf_tool *tool __maybe_unus static bool thread__has_color(struct thread *thread) { - return thread__priv(thread) != NULL; + struct thread_runtime *tr = thread__priv(thread); + + return tr != NULL && tr->color; } static struct thread* map__findnew_thread(struct perf_sched *sched, struct machine *machine, pid_t pid, pid_t tid) { struct thread *thread = machine__findnew_thread(machine, pid, tid); - bool color = false; - if (!sched->map.color_pids || !thread || thread__priv(thread)) + if (!sched->map.color_pids || !thread) return thread; - if (thread_map__has(sched->map.color_pids, tid)) - color = true; + /* + * Always check the color-pids map, even if thread__priv() is + * already set. COMM events processed before the first sched_switch + * allocate a thread_runtime via thread__get_runtime(), so priv is + * non-NULL before we ever get here. Skipping the check on non-NULL + * priv would prevent those threads from being colored. + */ + if (thread_map__has(sched->map.color_pids, tid)) { + struct thread_runtime *tr = thread__get_runtime(thread); - thread__set_priv(thread, color ? ((void*)1) : NULL); + if (tr) + tr->color = true; + } return thread; } @@ -1647,7 +1713,12 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, const char *str; int ret = -1; - BUG_ON(this_cpu.cpu >= MAX_CPUS || this_cpu.cpu < 0); + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (this_cpu.cpu >= MAX_CPUS || this_cpu.cpu < 0) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, this_cpu.cpu); + return 0; + } if (this_cpu.cpu > sched->max_cpu.cpu) sched->max_cpu = this_cpu; @@ -1659,7 +1730,7 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, new_cpu = true; } } else - cpus_nr = sched->max_cpu.cpu; + cpus_nr = sched->max_cpu.cpu + 1; timestamp0 = sched->cpu_last_switched[this_cpu.cpu]; sched->cpu_last_switched[this_cpu.cpu] = timestamp; @@ -2175,7 +2246,8 @@ static void timehist_print_sample(struct perf_sched *sched, char nstr[30]; u64 wait_time; - if (cpu_list && !test_bit(sample->cpu, cpu_bitmap)) + if (cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, cpu_bitmap))) return; timestamp__scnprintf_usec(t, tstr, sizeof(tstr)); @@ -2439,10 +2511,13 @@ static void free_idle_threads(void) struct idle_thread_runtime *itr; itr = thread__priv(idle); - if (itr) + if (itr) { thread__put(itr->last_thread); + free_callchain(&itr->callchain); + callchain_cursor_cleanup(&itr->cursor); + } - thread__delete(idle); + thread__put(idle); } } @@ -2475,8 +2550,11 @@ static struct thread *get_idle_thread(int cpu) idle_threads[cpu] = thread__new(0, 0); if (idle_threads[cpu]) { - if (init_idle_thread(idle_threads[cpu]) < 0) + if (init_idle_thread(idle_threads[cpu]) < 0) { + /* clean up so next call doesn't find a half-initialized thread */ + thread__zput(idle_threads[cpu]); return NULL; + } } } @@ -2528,12 +2606,16 @@ static struct thread *timehist_get_thread(struct perf_sched *sched, idle = get_idle_thread(sample->cpu); if (idle == NULL) { pr_err("Failed to get idle thread for cpu %d.\n", sample->cpu); + thread__put(thread); return NULL; } itr = thread__priv(idle); - if (itr == NULL) + if (itr == NULL) { + thread__put(idle); + thread__put(thread); return NULL; + } thread__put(itr->last_thread); itr->last_thread = thread__get(thread); @@ -2541,6 +2623,8 @@ static struct thread *timehist_get_thread(struct perf_sched *sched, /* copy task callchain when entering to idle */ if (evsel__intval(evsel, sample, "next_pid") == 0) save_idle_callchain(sched, itr, sample); + + thread__put(idle); } } @@ -2574,7 +2658,9 @@ static bool timehist_skip_sample(struct perf_sched *sched, else if (evsel__name_is(evsel, "sched:sched_switch")) prio = evsel__intval(evsel, sample, "prev_prio"); - if (prio != -1 && !test_bit(prio, sched->prio_bitmap)) { + /* negative prio means no info; out-of-range prio can't match the filter */ + if (prio >= 0 && + (prio >= MAX_PRIO || !test_bit(prio, sched->prio_bitmap))) { rc = true; sched->skipped_samples++; } @@ -2856,8 +2942,15 @@ static int timehist_sched_change_event(const struct perf_tool *tool, t = ptime->end; } - if (!sched->idle_hist || thread__tid(thread) == 0) { - if (!cpu_list || test_bit(sample->cpu, cpu_bitmap)) + /* + * Use is_idle_sample() not thread__tid() == 0: a crafted perf.data + * can set common_pid=0 with prev_pid!=0, giving us a machine thread + * whose priv is thread_runtime, not idle_thread_runtime — the cast + * below would read past the allocation. + */ + if (!sched->idle_hist || is_idle_sample(sample, evsel)) { + if (!cpu_list || (sample->cpu < MAX_NR_CPUS && + test_bit(sample->cpu, cpu_bitmap))) timehist_update_runtime_stats(tr, t, tprev); if (sched->idle_hist) { @@ -2887,7 +2980,7 @@ static int timehist_sched_change_event(const struct perf_tool *tool, if (itr->cursor.nr) callchain_append(&itr->callchain, &itr->cursor, t - tprev); - itr->last_thread = NULL; + thread__zput(itr->last_thread); } if (!sched->summary_only) @@ -3051,7 +3144,8 @@ static size_t timehist_print_idlehist_callchain(struct rb_root_cached *root) size_t ret = 0; FILE *fp = stdout; struct callchain_node *chain; - struct rb_node *rb_node = rb_first_cached(root); + /* sort() uses rb_insert_color() on rb_root, not rb_root_cached */ + struct rb_node *rb_node = rb_first(&root->rb_root); printf(" %16s %8s %s\n", "Idle time (msec)", "Count", "Callchains"); printf(" %.16s %.8s %.50s\n", graph_dotted_line, graph_dotted_line, @@ -3193,7 +3287,9 @@ static int perf_timehist__process_sample(const struct perf_tool *tool, .cpu = sample->cpu, }; - if (this_cpu.cpu > sched->max_cpu.cpu) + /* max_cpu indexes arrays allocated with MAX_CPUS entries */ + if (this_cpu.cpu >= 0 && this_cpu.cpu < MAX_CPUS && + this_cpu.cpu > sched->max_cpu.cpu) sched->max_cpu = this_cpu; if (evsel->handler != NULL) { @@ -3299,6 +3395,7 @@ static int perf_sched__timehist(struct perf_sched *sched) */ sched->tool.sample = perf_timehist__process_sample; sched->tool.mmap = perf_event__process_mmap; + sched->tool.mmap2 = perf_event__process_mmap2; sched->tool.comm = perf_event__process_comm; sched->tool.exit = perf_event__process_exit; sched->tool.fork = perf_event__process_fork; @@ -3362,8 +3459,8 @@ static int perf_sched__timehist(struct perf_sched *sched) perf_session__set_tracepoints_handlers(session, migrate_handlers)) goto out; - /* pre-allocate struct for per-CPU idle stats */ - sched->max_cpu.cpu = env->nr_cpus_online; + /* pre-allocate struct for per-CPU idle stats; cap to array bounds */ + sched->max_cpu.cpu = min(env->nr_cpus_online, MAX_CPUS); if (sched->max_cpu.cpu == 0) sched->max_cpu.cpu = 4; if (init_idle_threads(sched->max_cpu.cpu)) @@ -3556,10 +3653,8 @@ static int perf_sched__lat(struct perf_sched *sched) static int setup_map_cpus(struct perf_sched *sched) { - sched->max_cpu.cpu = sysconf(_SC_NPROCESSORS_CONF); - if (sched->map.comp) { - sched->map.comp_cpus = calloc(sched->max_cpu.cpu, sizeof(int)); + sched->map.comp_cpus = calloc(MAX_CPUS, sizeof(*sched->map.comp_cpus)); if (!sched->map.comp_cpus) return -1; } diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index c8ac9f01a36bc..ed372762276bc 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -63,6 +63,7 @@ #include #include "util/dlfilter.h" #include "util/record.h" +#include "util/unwind.h" #include "util/util.h" #include "util/cgroup.h" #include "util/annotate.h" @@ -2637,7 +2638,7 @@ static int cleanup_scripting(void) static bool filter_cpu(struct perf_sample *sample) { - if (cpu_list && sample->cpu != (u32)-1) + if (cpu_list && sample->cpu != (u32)-1 && sample->cpu < MAX_NR_CPUS) return !test_bit(sample->cpu, cpu_bitmap); return false; } @@ -4159,6 +4160,9 @@ int cmd_script(int argc, const char **argv) "Enable symbol demangling"), OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel, "Enable kernel symbol demangling"), + OPT_CALLBACK(0, "unwind-style", NULL, "unwind style", + "unwind styles (libdw,libunwind)", + unwind__option), OPT_STRING(0, "addr2line", &symbol_conf.addr2line_path, "path", "addr2line binary to use for line numbers"), OPT_STRING(0, "time", &script.time_str, "str", diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 99d7db372b480..a04466ea3b0a0 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -1637,9 +1637,10 @@ static struct aggr_cpu_id perf_env__get_socket_aggr_by_cpu(struct perf_cpu cpu, { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) - id.socket = env->cpu[cpu.cpu].socket_id; + if (topo) + id.socket = topo->socket_id; return id; } @@ -1648,15 +1649,16 @@ static struct aggr_cpu_id perf_env__get_die_aggr_by_cpu(struct perf_cpu cpu, voi { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { /* * die_id is relative to socket, so start * with the socket ID and then add die to * make a unique ID. */ - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; + id.socket = topo->socket_id; + id.die = topo->die_id; } return id; @@ -1704,12 +1706,13 @@ static struct aggr_cpu_id perf_env__get_cache_aggr_by_cpu(struct perf_cpu cpu, { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { u32 cache_level = (perf_stat.aggr_level) ?: stat_config.aggr_level; - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; + id.socket = topo->socket_id; + id.die = topo->die_id; perf_env__get_cache_id_for_cpu(cpu, env, cache_level, &id); } @@ -1721,11 +1724,12 @@ static struct aggr_cpu_id perf_env__get_cluster_aggr_by_cpu(struct perf_cpu cpu, { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; - id.cluster = env->cpu[cpu.cpu].cluster_id; + if (topo) { + id.socket = topo->socket_id; + id.die = topo->die_id; + id.cluster = topo->cluster_id; } return id; @@ -1735,16 +1739,17 @@ static struct aggr_cpu_id perf_env__get_core_aggr_by_cpu(struct perf_cpu cpu, vo { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { /* * core_id is relative to socket, die and cluster, we need a * global id. So we set socket, die id, cluster id and core id. */ - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; - id.cluster = env->cpu[cpu.cpu].cluster_id; - id.core = env->cpu[cpu.cpu].core_id; + id.socket = topo->socket_id; + id.die = topo->die_id; + id.cluster = topo->cluster_id; + id.core = topo->core_id; } return id; @@ -1754,18 +1759,19 @@ static struct aggr_cpu_id perf_env__get_cpu_aggr_by_cpu(struct perf_cpu cpu, voi { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { /* * core_id is relative to socket and die, * we need a global id. So we set * socket, die id and core id */ - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; - id.core = env->cpu[cpu.cpu].core_id; - id.cpu = cpu; + id.socket = topo->socket_id; + id.die = topo->die_id; + id.core = topo->core_id; } + id.cpu = cpu; return id; } diff --git a/tools/perf/pmu-events/amd_metrics.py b/tools/perf/pmu-events/amd_metrics.py index e2defaffde3e6..971f6e7af1f82 100755 --- a/tools/perf/pmu-events/amd_metrics.py +++ b/tools/perf/pmu-events/amd_metrics.py @@ -268,7 +268,7 @@ def AmdDtlb() -> Optional[MetricGroup]: def AmdItlb(): global _zen_model l2h = Event("bp_l1_tlb_miss_l2_tlb_hit", "bp_l1_tlb_miss_l2_hit") - l2m = Event("l2_itlb_misses") + l2m = Event("bp_l1_tlb_miss_l2_tlb_miss.all", "l2_itlb_misses",) l2r = l2h + l2m itlb_l1_mg = None diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py index 585454828c2fe..ac582db785fc1 100644 --- a/tools/perf/pmu-events/metric.py +++ b/tools/perf/pmu-events/metric.py @@ -25,7 +25,6 @@ def LoadEvents(directory: str) -> None: "cycles", "duration_time", "instructions", - "l2_itlb_misses", } for file in os.listdir(os.fsencode(directory)): filename = os.fsdecode(file) diff --git a/tools/perf/tests/code-reading.c b/tools/perf/tests/code-reading.c index 47043a3a2fb4f..e82ecdc957778 100644 --- a/tools/perf/tests/code-reading.c +++ b/tools/perf/tests/code-reading.c @@ -471,8 +471,11 @@ static int read_object_code(u64 addr, size_t len, u8 cpumode, goto out; } - decomp = true; - objdump_name = decomp_name; + /* empty pathname means file wasn't actually compressed */ + if (decomp_name[0] != '\0') { + decomp = true; + objdump_name = decomp_name; + } } /* Read the object code using objdump */ diff --git a/tools/perf/tests/dlfilter-test.c b/tools/perf/tests/dlfilter-test.c index e63790c61d53a..204663571943c 100644 --- a/tools/perf/tests/dlfilter-test.c +++ b/tools/perf/tests/dlfilter-test.c @@ -188,8 +188,12 @@ static int write_sample(struct test_data *td, u64 sample_type, u64 id, pid_t pid event->header.type = PERF_RECORD_SAMPLE; event->header.misc = PERF_RECORD_MISC_USER; - event->header.size = perf_event__sample_event_size(&sample, sample_type, 0); - err = perf_event__synthesize_sample(event, sample_type, 0, &sample); + event->header.size = perf_event__sample_event_size(&sample, sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0); + err = perf_event__synthesize_sample(event, sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0, &sample); if (err) return test_result("perf_event__synthesize_sample() failed", TEST_FAIL); diff --git a/tools/perf/tests/pmu-events.c b/tools/perf/tests/pmu-events.c index a997168621688..b1609a7e1d8c9 100644 --- a/tools/perf/tests/pmu-events.c +++ b/tools/perf/tests/pmu-events.c @@ -15,6 +15,7 @@ #include "util/expr.h" #include "util/hashmap.h" #include "util/parse-events.h" +#include "util/tool_pmu.h" #include "metricgroup.h" #include "stat.h" @@ -817,6 +818,26 @@ struct metric { struct metric_ref metric_ref; }; +static bool is_expected_broken_metric(const struct pmu_metric *pm) +{ + if (!strcmp(pm->metric_name, "M1") || !strcmp(pm->metric_name, "M2") || + !strcmp(pm->metric_name, "M3")) + return true; + +#if defined(__aarch64__) + /* + * Arm64 platforms may return "#slots == 0", which is treated as a + * syntax error by the parser. Don't test these metrics when running + * on such platforms. + */ + if (strstr(pm->metric_expr, "#slots") && + !tool_pmu__cpu_slots_per_cycle()) + return true; +#endif + + return false; +} + static int test__parsing_callback(const struct pmu_metric *pm, const struct pmu_metrics_table *table, void *data) @@ -852,8 +873,7 @@ static int test__parsing_callback(const struct pmu_metric *pm, err = metricgroup__parse_groups_test(evlist, table, pm->metric_name); if (err) { - if (!strcmp(pm->metric_name, "M1") || !strcmp(pm->metric_name, "M2") || - !strcmp(pm->metric_name, "M3")) { + if (is_expected_broken_metric(pm)) { (*failures)--; pr_debug("Expected broken metric %s skipping\n", pm->metric_name); err = 0; diff --git a/tools/perf/tests/sample-parsing.c b/tools/perf/tests/sample-parsing.c index a7327c942ca20..55f0b73ca20e0 100644 --- a/tools/perf/tests/sample-parsing.c +++ b/tools/perf/tests/sample-parsing.c @@ -310,7 +310,8 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) sample.read.one.lost = 1; } - sz = perf_event__sample_event_size(&sample, sample_type, read_format); + sz = perf_event__sample_event_size(&sample, sample_type, read_format, + evsel.core.attr.branch_sample_type); bufsz = sz + 4096; /* Add a bit for overrun checking */ event = malloc(bufsz); if (!event) { @@ -324,7 +325,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) event->header.size = sz; err = perf_event__synthesize_sample(event, sample_type, read_format, - &sample); + evsel.core.attr.branch_sample_type, &sample); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "perf_event__synthesize_sample", sample_type, err); diff --git a/tools/perf/trace/beauty/fcntl.c b/tools/perf/trace/beauty/fcntl.c index d075904dccced..e1b99b8f55eb0 100644 --- a/tools/perf/trace/beauty/fcntl.c +++ b/tools/perf/trace/beauty/fcntl.c @@ -9,6 +9,22 @@ #include #include +#ifndef F_GET_RW_HINT +#define F_GET_RW_HINT (F_LINUX_SPECIFIC_BASE + 11) +#endif + +#ifndef F_SET_RW_HINT +#define F_SET_RW_HINT (F_LINUX_SPECIFIC_BASE + 12) +#endif + +#ifndef F_GET_FILE_RW_HINT +#define F_GET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 13) +#endif + +#ifndef F_SET_FILE_RW_HINT +#define F_SET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 14) +#endif + static size_t fcntl__scnprintf_getfd(unsigned long val, char *bf, size_t size, bool show_prefix) { return val ? scnprintf(bf, size, "%s", "0") : diff --git a/tools/perf/ui/browser.c b/tools/perf/ui/browser.c index dc88427b4ae5a..321187b204d38 100644 --- a/tools/perf/ui/browser.c +++ b/tools/perf/ui/browser.c @@ -513,6 +513,9 @@ unsigned int ui_browser__list_head_refresh(struct ui_browser *browser) struct list_head *head = browser->entries; int row = 0; + if (browser->nr_entries == 0) + return 0; + if (browser->top == NULL || browser->top == browser->entries) browser->top = ui_browser__list_head_filter_entries(browser, head->next); diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c index ea17e6d29a7e4..0261cd9221834 100644 --- a/tools/perf/ui/browsers/annotate.c +++ b/tools/perf/ui/browsers/annotate.c @@ -449,6 +449,9 @@ static bool annotate_browser__toggle_source(struct annotate_browser *browser, struct annotation_line *al; off_t offset = browser->b.index - browser->b.top_idx; + if (browser->b.nr_entries == 0) + return false; + browser->b.seek(&browser->b, offset, SEEK_CUR); al = list_entry(browser->b.top, struct annotation_line, node); @@ -542,8 +545,8 @@ static void annotate_browser__show_full_location(struct ui_browser *browser) static void ui_browser__init_asm_mode(struct ui_browser *browser) { struct annotation *notes = browser__annotation(browser); - ui_browser__reset_index(browser); browser->nr_entries = notes->src->nr_asm_entries; + ui_browser__reset_index(browser); } static int sym_title(struct symbol *sym, struct map *map, char *title, diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 70cc91d00804d..01edfccebb88e 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -216,6 +216,7 @@ ifndef CONFIG_SETNS perf-util-y += setns.o endif +perf-util-y += unwind.o perf-util-$(CONFIG_LIBDW) += probe-finder.o perf-util-$(CONFIG_LIBDW) += dwarf-aux.o perf-util-$(CONFIG_LIBDW) += dwarf-regs.o diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c index 1eff0a27237d9..63e3c54fab421 100644 --- a/tools/perf/util/annotate-data.c +++ b/tools/perf/util/annotate-data.c @@ -74,7 +74,8 @@ void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind) break; } - dwarf_aggregate_size(die, &size); + if (dwarf_aggregate_size(die, &size) != 0) + size = 0; strbuf_init(&sb, 32); die_get_typename_from_type(die, &sb); @@ -146,9 +147,9 @@ static void pr_debug_scope(Dwarf_Die *scope_die) tag = dwarf_tag(scope_die); if (tag == DW_TAG_subprogram) - pr_info("[function] %s\n", dwarf_diename(scope_die)); + pr_info("[function] %s\n", die_name(scope_die)); else if (tag == DW_TAG_inlined_subroutine) - pr_info("[inlined] %s\n", dwarf_diename(scope_die)); + pr_info("[inlined] %s\n", die_name(scope_die)); else if (tag == DW_TAG_lexical_block) pr_info("[block]\n"); else @@ -250,9 +251,12 @@ static int __add_member_cb(Dwarf_Die *die, void *arg) if (dwarf_aggregate_size(&die_mem, &size) < 0) size = 0; - if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) - dwarf_formudata(&attr, &loc); - else { + if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) { + if (dwarf_formudata(&attr, &loc) != 0) { + if (die_get_data_member_location(die, &loc) != 0) + loc = 0; + } + } else { /* bitfield member */ if (dwarf_attr_integrate(die, DW_AT_data_bit_offset, &attr) && dwarf_formudata(&attr, &loc) == 0) @@ -273,7 +277,9 @@ static int __add_member_cb(Dwarf_Die *die, void *arg) dwarf_diename(die), (long)bit_size) < 0) member->var_name = NULL; } else { - member->var_name = strdup(dwarf_diename(die)); + const char *name = dwarf_diename(die); + + member->var_name = name ? strdup(name) : NULL; } if (member->var_name == NULL) { @@ -370,7 +376,8 @@ static struct annotated_data_type *dso__findnew_data_type(struct dso *dso, if (dwarf_tag(type_die) == DW_TAG_typedef) die_get_real_type(type_die, type_die); - dwarf_aggregate_size(type_die, &size); + if (dwarf_aggregate_size(type_die, &size) != 0) + size = 0; /* Check existing nodes in dso->data_types tree */ key.self.type_name = type_name; @@ -1569,7 +1576,7 @@ static int find_data_type_die(struct data_loc_info *dloc, Dwarf_Die *type_die) offset = loc->offset; pr_debug_dtp("CU for %s (die:%#lx)\n", - dwarf_diename(&cu_die), (long)dwarf_dieoffset(&cu_die)); + die_name(&cu_die), (long)dwarf_dieoffset(&cu_die)); if (reg == DWARF_REG_PC) { if (get_global_var_type(&cu_die, dloc, dloc->ip, dloc->var_addr, @@ -1636,7 +1643,7 @@ static int find_data_type_die(struct data_loc_info *dloc, Dwarf_Die *type_die) } pr_debug_dtp("found \"%s\" (die: %#lx) in scope=%d/%d (die: %#lx) ", - dwarf_diename(&var_die), (long)dwarf_dieoffset(&var_die), + die_name(&var_die), (long)dwarf_dieoffset(&var_die), i+1, nr_scopes, (long)dwarf_dieoffset(&scopes[i])); if (reg == DWARF_REG_PC) { diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index e5835042acdf7..3e54ed41653bd 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -482,10 +482,30 @@ static void arm_spe__prep_branch_stack(struct arm_spe_queue *speq) bstack->hw_idx = -1ULL; } -static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type) +static int arm_spe__inject_event(struct arm_spe *spe, union perf_event *event, + struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && spe->session && spe->session->evlist) + evsel = evlist__id2evsel(spe->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } static inline int @@ -497,7 +517,7 @@ arm_spe_deliver_synth_event(struct arm_spe *spe, int ret; if (spe->synth_opts.inject) { - ret = arm_spe__inject_event(event, sample, spe->sample_type); + ret = arm_spe__inject_event(spe, event, sample, spe->sample_type); if (ret) return ret; } diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index a224687ffbc1b..47e2004b91d73 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -372,7 +372,8 @@ static bool filter_cpu(struct perf_session *session, struct perf_cpu cpu) { unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap; - return cpu_bitmap && cpu.cpu != -1 && !test_bit(cpu.cpu, cpu_bitmap); + return cpu_bitmap && cpu.cpu >= 0 && cpu.cpu < MAX_NR_CPUS && + !test_bit(cpu.cpu, cpu_bitmap); } static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues, diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index a27945c279efb..fa3ebc8ea7f09 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -36,7 +36,7 @@ static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) size_t i; for (i = 0; i < len; i++) - ret += snprintf(buf + ret, size - ret, "%02x", data[i]); + ret += scnprintf(buf + ret, size - ret, "%02x", data[i]); return ret; } @@ -59,6 +59,10 @@ static int machine__process_bpf_event_load(struct machine *machine, return 0; info_linear = info_node->info_linear; + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) + return 0; + for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); u64 addr = addrs[i]; @@ -140,22 +144,26 @@ static int synthesize_bpf_prog_name(char *buf, int size, const struct btf_type *t; int name_len; - name_len = snprintf(buf, size, "bpf_prog_"); + name_len = scnprintf(buf, size, "bpf_prog_"); name_len += snprintf_hex(buf + name_len, size - name_len, prog_tags[sub_id], BPF_TAG_SIZE); - if (btf) { + if (btf && + info->func_info_rec_size >= sizeof(*finfo) && + sub_id < info->nr_func_info) { finfo = func_infos + sub_id * info->func_info_rec_size; t = btf__type_by_id(btf, finfo->type_id); - short_name = btf__name_by_offset(btf, t->name_off); + if (t) + short_name = btf__name_by_offset(btf, t->name_off); } else if (sub_id == 0 && sub_prog_cnt == 1) { /* no subprog */ if (info->name[0]) short_name = info->name; } else short_name = "F"; - if (short_name) - name_len += snprintf(buf + name_len, size - name_len, - "_%s", short_name); + if (short_name) { + name_len += scnprintf(buf + name_len, size - name_len, + "_%s", short_name); + } return name_len; } @@ -365,6 +373,15 @@ static struct bpf_metadata *bpf_metadata_alloc(__u32 nr_prog_tags, event_size = sizeof(metadata->event->bpf_metadata) + nr_variables * sizeof(metadata->event->bpf_metadata.entries[0]); + /* + * header.size is __u16. synthesize_perf_record_bpf_metadata() + * adds machine->id_hdr_size (up to ~64 bytes) after this, so + * leave headroom to prevent the final size from wrapping. + */ + if (event_size > UINT16_MAX - 256) { + bpf_metadata_free(metadata); + return NULL; + } metadata->event = zalloc(event_size); if (!metadata->event) { bpf_metadata_free(metadata); @@ -393,8 +410,10 @@ static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info) continue; metadata = bpf_metadata_alloc(info->nr_prog_tags, map.num_vars); - if (!metadata) + if (!metadata) { + bpf_metadata_free_map_data(&map); continue; + } bpf_metadata_fill_event(&map, &metadata->event->bpf_metadata); @@ -869,6 +888,7 @@ static int perf_env__add_bpf_info(struct perf_env *env, u32 id) if (!perf_env__insert_bpf_prog_info(env, info_node)) { pr_debug("%s: duplicate add bpf info request for id %u\n", __func__, btf_id); + bpf_metadata_free(info_node->metadata); free(info_linear); free(info_node); goto out; @@ -943,12 +963,15 @@ int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); } -void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, struct perf_env *env, FILE *fp) { - __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); - __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + struct bpf_prog_info *info = &info_linear->info; + __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | + (1UL << PERF_BPIL_JITED_FUNC_LENS); + __u32 *prog_lens; + __u64 *prog_addrs; char name[KSYM_NAME_LEN]; struct btf *btf = NULL; u32 sub_prog_cnt, i; @@ -958,6 +981,13 @@ void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, sub_prog_cnt != info->nr_jited_func_lens) return; + /* Ensure the arrays were present and converted by bpil_offs_to_addr() */ + if ((info_linear->arrays & required_arrays) != required_arrays) + return; + + prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); + prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + if (info->btf_id) { struct btf_node *node; diff --git a/tools/perf/util/bpf-event.h b/tools/perf/util/bpf-event.h index 60d2c6637af5d..da4eeb4a1a732 100644 --- a/tools/perf/util/bpf-event.h +++ b/tools/perf/util/bpf-event.h @@ -40,7 +40,7 @@ struct btf_node { int machine__process_bpf(struct machine *machine, union perf_event *event, struct perf_sample *sample); int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env); -void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, struct perf_env *env, FILE *fp); void bpf_metadata_free(struct bpf_metadata *metadata); @@ -58,7 +58,7 @@ static inline int evlist__add_bpf_sb_event(struct evlist *evlist __maybe_unused, return 0; } -static inline void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info __maybe_unused, +static inline void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear __maybe_unused, struct perf_env *env __maybe_unused, FILE *fp __maybe_unused) { diff --git a/tools/perf/util/bpf-utils.c b/tools/perf/util/bpf-utils.c index d6d2c9c190f7a..69148197b1ef9 100644 --- a/tools/perf/util/bpf-utils.c +++ b/tools/perf/util/bpf-utils.c @@ -264,12 +264,28 @@ void bpil_offs_to_addr(struct perf_bpil *info_linear) for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { const struct bpil_array_desc *desc = &bpil_array_desc[i]; __u64 addr, offs; + __u32 count, size; if ((info_linear->arrays & (1UL << i)) == 0) continue; offs = bpf_prog_info_read_offset_u64(&info_linear->info, desc->array_offset); + count = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->size_offset); + /* offset and extent from perf.data are untrusted — keep within data[] */ + if (offs >= info_linear->data_len || + (u64)count * size > info_linear->data_len - offs) { + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, 0); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->count_offset, 0); + /* clear the bit so bpil_addr_to_offs() won't reverse a zeroed address */ + info_linear->arrays &= ~(1UL << i); + continue; + } addr = offs + ptr_to_u64(info_linear->data); bpf_prog_info_set_offset_u64(&info_linear->info, desc->array_offset, addr); diff --git a/tools/perf/util/build-id.c b/tools/perf/util/build-id.c index fdb35133fde43..3c286cdbe6e0e 100644 --- a/tools/perf/util/build-id.c +++ b/tools/perf/util/build-id.c @@ -93,8 +93,11 @@ int build_id__snprintf(const struct build_id *build_id, char *bf, size_t bf_size return 0; } - for (size_t i = 0; i < build_id->size && offs < bf_size; ++i) - offs += snprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); + if (bf_size > 0) + bf[0] = '\0'; + + for (size_t i = 0; i < build_id->size && offs + 1 < bf_size; ++i) + offs += scnprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); return offs; } diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index f031cbbeeba8b..5e0498a1faf8a 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -1578,6 +1578,21 @@ void free_callchain(struct callchain_root *root) free_callchain_node(&root->node); } +void callchain_cursor_cleanup(struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *node, *next; + + callchain_cursor_reset(cursor); + + for (node = cursor->first; node; node = next) { + next = node->next; + free(node); + } + cursor->first = NULL; + cursor->last = &cursor->first; + cursor->curr = NULL; +} + static u64 decay_callchain_node(struct callchain_node *node) { struct callchain_node *child; diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h index 06d463ccc7a04..60008f379b228 100644 --- a/tools/perf/util/callchain.h +++ b/tools/perf/util/callchain.h @@ -290,6 +290,7 @@ int callchain_list_counts__printf_value(struct callchain_list *clist, FILE *fp, char *bf, int bfsize); void free_callchain(struct callchain_root *root); +void callchain_cursor_cleanup(struct callchain_cursor *cursor); void decay_callchain(struct callchain_root *root); int callchain_node__make_parent_list(struct callchain_node *node); diff --git a/tools/perf/util/config.c b/tools/perf/util/config.c index 087002fb1b9bc..7988149dc7ed8 100644 --- a/tools/perf/util/config.c +++ b/tools/perf/util/config.c @@ -23,6 +23,7 @@ #include "build-id.h" #include "debug.h" #include "config.h" +#include "unwind.h" #include #include #include @@ -525,6 +526,9 @@ int perf_default_config(const char *var, const char *value, if (strstarts(var, "addr2line.")) return addr2line_configure(var, value, dummy); + if (strstarts(var, "unwind.")) + return unwind__configure(var, value, dummy); + /* Add other config variables here. */ return 0; } diff --git a/tools/perf/util/cpumap.c b/tools/perf/util/cpumap.c index 11922e1ded844..dbbec195cffdc 100644 --- a/tools/perf/util/cpumap.c +++ b/tools/perf/util/cpumap.c @@ -420,6 +420,12 @@ static int get_max_num(char *path, int *max) buf[num] = '\0'; + /* empty file — nothing to parse */ + if (num == 0) { + err = -1; + goto out; + } + /* start on the right, to find highest node num */ while (--num) { if ((buf[num] == ',') || (buf[num] == '-')) { @@ -466,6 +472,16 @@ static void set_max_cpu_num(void) if (ret) goto out; + /* + * struct perf_cpu.cpu is int16_t (libperf ABI) — clamp to avoid + * truncation to negative. See tools/lib/perf/TODO for the ABI + * widening plan. + */ + if (max > INT16_MAX) { + pr_warning("WARNING: max possible cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; + } max_cpu_num.cpu = max; /* get the highest present cpu number for a sparse allocation */ @@ -478,11 +494,12 @@ static void set_max_cpu_num(void) ret = get_max_num(path, &max); if (!ret && max > INT16_MAX) { - pr_err("Read out of bounds max cpus of %d\n", max); - ret = -1; + pr_warning("WARNING: max present cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; } if (!ret) - max_present_cpu_num.cpu = (int16_t)max; + max_present_cpu_num.cpu = max; out: if (ret) pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); @@ -548,6 +565,10 @@ int cpu__get_node(struct perf_cpu cpu) return -1; } + /* cpunode_map allocated for max_cpu_num entries; input may be untrusted */ + if (cpu.cpu < 0 || cpu.cpu >= max_cpu_num.cpu) + return -1; + return cpunode_map[cpu.cpu]; } @@ -615,7 +636,9 @@ int cpu__setup_cpunode_map(void) while ((dent2 = readdir(dir2)) != NULL) { if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) continue; - cpunode_map[cpu] = mem; + /* cpunode_map allocated for max_cpu_num entries */ + if (cpu < (unsigned int)max_cpu_num.cpu) + cpunode_map[cpu] = mem; } closedir(dir2); } @@ -641,21 +664,21 @@ size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) if (start == -1) { start = i; if (last) { - ret += snprintf(buf + ret, size - ret, - "%s%d", COMMA, - perf_cpu_map__cpu(map, i).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, i).cpu); } } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { int end = i - 1; if (start == end) { - ret += snprintf(buf + ret, size - ret, - "%s%d", COMMA, - perf_cpu_map__cpu(map, start).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, start).cpu); } else { - ret += snprintf(buf + ret, size - ret, - "%s%d-%d", COMMA, - perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d-%d", COMMA, + perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); } first = false; start = i; diff --git a/tools/perf/util/cs-etm-base.c b/tools/perf/util/cs-etm-base.c index 4abe416e3febd..aebef71d3a0a1 100644 --- a/tools/perf/util/cs-etm-base.c +++ b/tools/perf/util/cs-etm-base.c @@ -170,7 +170,9 @@ int cs_etm__process_auxtrace_info(union perf_event *event, u64 *ptr = NULL; u64 hdr_version; - if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE)) + /* Ensure priv[] is large enough for the global header entries */ + if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE + + CS_ETM_HEADER_SIZE)) return -EINVAL; /* First the global part */ diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index dee3020ceaa91..26940f1f1b0bf 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -402,6 +402,8 @@ cs_etm_decoder__buffer_packet(struct cs_etm_queue *etmq, packet_queue->packet_buffer[et].flags = 0; packet_queue->packet_buffer[et].exception_number = UINT32_MAX; packet_queue->packet_buffer[et].trace_chan_id = trace_chan_id; + packet_queue->packet_buffer[et].el = ocsd_EL_unknown; + packet_queue->packet_buffer[et].tid = -1; if (packet_queue->packet_count == CS_ETM_PACKET_MAX_BUFFER - 1) return OCSD_RESP_WAIT; @@ -449,6 +451,7 @@ cs_etm_decoder__buffer_range(struct cs_etm_queue *etmq, packet->last_instr_type = elem->last_i_type; packet->last_instr_subtype = elem->last_i_subtype; packet->last_instr_cond = elem->last_instr_cond; + packet->el = elem->context.exception_level; if (elem->last_i_type == OCSD_INSTR_BR || elem->last_i_type == OCSD_INSTR_BR_INDIRECT) packet->last_instr_taken_branch = elem->last_instr_exec; @@ -525,7 +528,9 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, const ocsd_generic_trace_elem *elem, const uint8_t trace_chan_id) { + struct cs_etm_packet *packet; pid_t tid = -1; + int ret; /* * Process the PE_CONTEXT packets if we have a valid contextID or VMID. @@ -546,12 +551,18 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, break; } - if (cs_etm__etmq_set_tid_el(etmq, tid, trace_chan_id, - elem->context.exception_level)) + if (cs_etm__etmq_update_decode_context(etmq, trace_chan_id, + elem->context.exception_level, tid)) return OCSD_RESP_FATAL_SYS_ERR; - if (tid == -1) - return OCSD_RESP_CONT; + ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id, + CS_ETM_CONTEXT); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &packet_queue->packet_buffer[packet_queue->tail]; + packet->tid = tid; + packet->el = elem->context.exception_level; /* * A timestamp is generated after a PE_CONTEXT element so make sure @@ -559,7 +570,7 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, */ cs_etm_decoder__reset_timestamp(packet_queue); - return OCSD_RESP_CONT; + return ret; } static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 8a639d2e51a4c..5ca55b5271839 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -6,6 +6,7 @@ * Author: Mathieu Poirier */ +#include #include #include #include @@ -85,15 +86,22 @@ struct cs_etm_traceid_queue { u64 period_instructions; size_t last_branch_pos; union perf_event *event_buf; - struct thread *thread; - struct thread *prev_packet_thread; - ocsd_ex_level prev_packet_el; - ocsd_ex_level el; struct branch_stack *last_branch; struct branch_stack *last_branch_rb; struct cs_etm_packet *prev_packet; struct cs_etm_packet *packet; struct cs_etm_packet_queue packet_queue; + + struct thread *decode_thread; + ocsd_ex_level decode_el; + + /* + * The frontend accesses the EL from '[prev_]packet' because it needs + * previous EL for branch and current EL for instruction samples. It's + * not possible to change thread in a single branch sample so no need to + * store or access the thread through the packet. + */ + struct thread *frontend_thread; }; enum cs_etm_format { @@ -284,8 +292,11 @@ static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int c { if (etm->per_thread_decoding) return etm->queues.queue_array[0].priv; - else - return etm->queues.queue_array[cpu].priv; + + if (cpu < 0 || cpu >= (int)etm->queues.nr_queues) + return NULL; + + return etm->queues.queue_array[cpu].priv; } static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id, @@ -298,6 +309,9 @@ static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id * queue associated with that CPU so only one decoder is made. */ etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]); + if (!etmq) + return -EINVAL; + if (etmq->format == UNFORMATTED) return cs_etm__insert_trace_id_node(etmq, trace_chan_id, cpu_metadata); @@ -310,6 +324,9 @@ static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id int ret; etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + ret = cs_etm__insert_trace_id_node(etmq, trace_chan_id, cpu_metadata); if (ret) @@ -350,6 +367,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, u32 sink_id = FIELD_GET(CS_AUX_HW_ID_SINK_ID_MASK, hw_id); u8 trace_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); + if (!etmq) + return -EINVAL; + /* * Check sink id hasn't changed in per-cpu mode. In per-thread mode, * let it pass for now until an actual overlapping trace ID is hit. In @@ -367,6 +387,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv; + if (!other_etmq) + continue; + /* Different sinks, skip */ if (other_etmq->sink_id != etmq->sink_id) continue; @@ -388,6 +411,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, } cpu_data = get_cpu_data(etm, cpu); + if (!cpu_data) + return -EINVAL; + ret = cs_etm__insert_trace_id_node(etmq, trace_id, cpu_data); if (ret) return ret; @@ -614,10 +640,11 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; tidq->trace_chan_id = trace_chan_id; - tidq->el = tidq->prev_packet_el = ocsd_EL_unknown; - tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1, + tidq->decode_el = ocsd_EL_unknown; + tidq->frontend_thread = machine__findnew_thread(&etm->session->machines.host, -1, + queue->tid); + tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, queue->tid); - tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host); tidq->packet = zalloc(sizeof(struct cs_etm_packet)); if (!tidq->packet) @@ -750,21 +777,10 @@ static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, /* * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for * the next incoming packet. - * - * Threads and exception levels are also tracked for both the - * previous and current packets. This is because the previous - * packet is used for the 'from' IP for branch samples, so the - * thread at that time must also be assigned to that sample. - * Across discontinuity packets the thread can change, so by - * tracking the thread for the previous packet the branch sample - * will have the correct info. */ tmp = tidq->packet; tidq->packet = tidq->prev_packet; tidq->prev_packet = tmp; - tidq->prev_packet_el = tidq->el; - thread__put(tidq->prev_packet_thread); - tidq->prev_packet_thread = thread__get(tidq->thread); } } @@ -937,8 +953,8 @@ static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) /* Free this traceid_queue from the array */ tidq = etmq->traceid_queues[idx]; - thread__zput(tidq->thread); - thread__zput(tidq->prev_packet_thread); + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); zfree(&tidq->event_buf); zfree(&tidq->last_branch); zfree(&tidq->last_branch_rb); @@ -1083,47 +1099,43 @@ static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address, } } -static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, - u64 address, size_t size, u8 *buffer, - const ocsd_mem_space_acc_t mem_space) +static u32 __cs_etm__mem_access(struct cs_etm_queue *etmq, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space, + ocsd_ex_level el, struct thread *thread) { u8 cpumode; u64 offset; int len; struct addr_location al; struct dso *dso; - struct cs_etm_traceid_queue *tidq; int ret = 0; if (!etmq) return 0; addr_location__init(&al); - tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); - if (!tidq) - goto out; /* - * We've already tracked EL along side the PID in cs_etm__set_thread() - * so double check that it matches what OpenCSD thinks as well. It - * doesn't distinguish between EL0 and EL1 for this mem access callback - * so we had to do the extra tracking. Skip validation if it's any of - * the 'any' values. + * We track EL for the frontend and the backend when receiving context + * and range packets. OpenCSD doesn't distinguish between EL0 and EL1 + * for this mem access callback so we had to do the extra tracking. Skip + * validation if it's any of the 'any' values. */ if (!(mem_space == OCSD_MEM_SPACE_ANY || mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) { if (mem_space & OCSD_MEM_SPACE_EL1N) { /* Includes both non secure EL1 and EL0 */ - assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0); + assert(el == ocsd_EL1 || el == ocsd_EL0); } else if (mem_space & OCSD_MEM_SPACE_EL2) - assert(tidq->el == ocsd_EL2); + assert(el == ocsd_EL2); else if (mem_space & OCSD_MEM_SPACE_EL3) - assert(tidq->el == ocsd_EL3); + assert(el == ocsd_EL3); } - cpumode = cs_etm__cpu_mode(etmq, address, tidq->el); + cpumode = cs_etm__cpu_mode(etmq, address, el); - if (!thread__find_map(tidq->thread, cpumode, address, &al)) + if (!thread__find_map(thread, cpumode, address, &al)) goto out; dso = map__dso(al.map); @@ -1138,7 +1150,7 @@ static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, map__load(al.map); - len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)), + len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)), offset, buffer, size); if (len <= 0) { @@ -1158,6 +1170,30 @@ static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, return ret; } +static u32 cs_etm__frontend_mem_access(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 address, size_t size, u8 *buffer) +{ + return __cs_etm__mem_access(etmq, address, size, buffer, 0, packet->el, + tidq->frontend_thread); +} + +static u32 cs_etm__decoder_mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space) +{ + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) + return 0; + + return __cs_etm__mem_access(etmq, address, size, buffer, + mem_space, tidq->decode_el, + tidq->decode_thread); +} + static struct cs_etm_queue *cs_etm__alloc_queue(void) { struct cs_etm_queue *etmq = zalloc(sizeof(*etmq)); @@ -1333,12 +1369,13 @@ void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq) } static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, - u8 trace_chan_id, u64 addr) + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr) { u8 instrBytes[2]; - cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes), - instrBytes, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + ARRAY_SIZE(instrBytes), instrBytes); /* * T32 instruction size is indicated by bits[15:11] of the first * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 @@ -1371,16 +1408,16 @@ u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) } static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, - u64 trace_chan_id, - const struct cs_etm_packet *packet, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 offset) { if (packet->isa == CS_ETM_ISA_T32) { u64 addr = packet->start_addr; while (offset) { - addr += cs_etm__t32_instr_size(etmq, - trace_chan_id, addr); + addr += cs_etm__t32_instr_size(etmq, tidq, packet, + addr); offset--; } return addr; @@ -1422,11 +1459,29 @@ static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq, bs->nr += 1; } -static int cs_etm__inject_event(union perf_event *event, +static int cs_etm__inject_event(struct cs_etm_auxtrace *etm, union perf_event *event, struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && etm->session && etm->session->evlist) + evsel = evlist__id2evsel(etm->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } @@ -1472,34 +1527,51 @@ cs_etm__get_trace(struct cs_etm_queue *etmq) return etmq->buf_len; } -static void cs_etm__set_thread(struct cs_etm_queue *etmq, - struct cs_etm_traceid_queue *tidq, pid_t tid, - ocsd_ex_level el) +/* + * Convert a raw thread number to a thread struct and assign it to **thread. + */ +static int cs_etm__etmq_update_thread(struct cs_etm_queue *etmq, + ocsd_ex_level el, pid_t tid, + struct thread **thread) { struct machine *machine = cs_etm__get_machine(etmq, el); + if (!machine || !*thread) + return -EINVAL; + if (tid != -1) { - thread__zput(tidq->thread); - tidq->thread = machine__find_thread(machine, -1, tid); + thread__zput(*thread); + *thread = machine__find_thread(machine, -1, tid); } /* Couldn't find a known thread */ - if (!tidq->thread) - tidq->thread = machine__idle_thread(machine); + if (!*thread) + *thread = machine__idle_thread(machine); - tidq->el = el; + return 0; } -int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid, - u8 trace_chan_id, ocsd_ex_level el) +/* + * Set the thread and EL of the decode context which is ahead in time of the + * frontend context. + */ +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, + ocsd_ex_level el, pid_t tid) { struct cs_etm_traceid_queue *tidq; + int ret; tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); if (!tidq) return -EINVAL; - cs_etm__set_thread(etmq, tidq, tid, el); + ret = cs_etm__etmq_update_thread(etmq, el, tid, + &tidq->decode_thread); + if (ret) + return ret; + + tidq->decode_el = el; return 0; } @@ -1509,8 +1581,8 @@ bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) } static void cs_etm__copy_insn(struct cs_etm_queue *etmq, - u64 trace_chan_id, - const struct cs_etm_packet *packet, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, struct perf_sample *sample) { /* @@ -1527,14 +1599,14 @@ static void cs_etm__copy_insn(struct cs_etm_queue *etmq, * cs_etm__t32_instr_size(). */ if (packet->isa == CS_ETM_ISA_T32) - sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id, + sample->insn_len = cs_etm__t32_instr_size(etmq, tidq, packet, sample->ip); /* Otherwise, A64 and A32 instruction size are always 32-bit. */ else sample->insn_len = 4; - cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len, - (void *)sample->insn, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip, + sample->insn_len, (void *)sample->insn); } u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp) @@ -1561,6 +1633,7 @@ static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq, static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr, u64 period) { int ret = 0; @@ -1570,29 +1643,29 @@ static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, perf_sample__init(&sample, /*all=*/true); event->sample.header.type = PERF_RECORD_SAMPLE; - event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el); + event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, packet->el); event->sample.header.size = sizeof(struct perf_event_header); /* Set time field based on etm auxtrace config. */ sample.time = cs_etm__resolve_sample_time(etmq, tidq); sample.ip = addr; - sample.pid = thread__pid(tidq->thread); - sample.tid = thread__tid(tidq->thread); + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); sample.id = etmq->etm->instructions_id; sample.stream_id = etmq->etm->instructions_id; sample.period = period; - sample.cpu = tidq->packet->cpu; + sample.cpu = packet->cpu; sample.flags = tidq->prev_packet->flags; sample.cpumode = event->sample.header.misc; - cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample); + cs_etm__copy_insn(etmq, tidq, packet, &sample); if (etm->synth_opts.last_branch) sample.branch_stack = tidq->last_branch; if (etm->synth_opts.inject) { - ret = cs_etm__inject_event(event, &sample, + ret = cs_etm__inject_event(etm, event, &sample, etm->instructions_sample_type); if (ret) return ret; @@ -1631,15 +1704,15 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, event->sample.header.type = PERF_RECORD_SAMPLE; event->sample.header.misc = cs_etm__cpu_mode(etmq, ip, - tidq->prev_packet_el); + tidq->prev_packet->el); event->sample.header.size = sizeof(struct perf_event_header); /* Set time field based on etm auxtrace config. */ sample.time = cs_etm__resolve_sample_time(etmq, tidq); sample.ip = ip; - sample.pid = thread__pid(tidq->prev_packet_thread); - sample.tid = thread__tid(tidq->prev_packet_thread); + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); sample.addr = cs_etm__first_executed_instr(tidq->packet); sample.id = etmq->etm->branches_id; sample.stream_id = etmq->etm->branches_id; @@ -1648,8 +1721,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, sample.flags = tidq->prev_packet->flags; sample.cpumode = event->sample.header.misc; - cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet, - &sample); + cs_etm__copy_insn(etmq, tidq, tidq->prev_packet, &sample); /* * perf report cannot handle events without a branch stack @@ -1667,7 +1739,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, } if (etm->synth_opts.inject) { - ret = cs_etm__inject_event(event, &sample, + ret = cs_etm__inject_event(etm, event, &sample, etm->branches_sample_type); if (ret) return ret; @@ -1770,7 +1842,6 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, { struct cs_etm_auxtrace *etm = etmq->etm; int ret; - u8 trace_chan_id = tidq->trace_chan_id; u64 instrs_prev; /* Get instructions remainder from previous packet */ @@ -1856,10 +1927,10 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, * been executed, but PC has not advanced to next * instruction) */ - addr = cs_etm__instr_addr(etmq, trace_chan_id, - tidq->packet, offset - 1); + addr = cs_etm__instr_addr(etmq, tidq, tidq->packet, + offset - 1); ret = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->packet, addr, etm->instructions_sample_period); if (ret) return ret; @@ -1941,7 +2012,7 @@ static int cs_etm__flush(struct cs_etm_queue *etmq, addr = cs_etm__last_executed_instr(tidq->prev_packet); err = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->prev_packet, addr, tidq->period_instructions); if (err) return err; @@ -1996,7 +2067,7 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, addr = cs_etm__last_executed_instr(tidq->prev_packet); err = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->prev_packet, addr, tidq->period_instructions); if (err) return err; @@ -2033,9 +2104,9 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq) return etmq->buf_len; } -static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, - struct cs_etm_packet *packet, - u64 end_addr) +static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 end_addr) { /* Initialise to keep compiler happy */ u16 instr16 = 0; @@ -2057,8 +2128,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * so below only read 2 bytes as instruction size for T32. */ addr = end_addr - 2; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16), - (u8 *)&instr16, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr16), (u8 *)&instr16); if ((instr16 & 0xFF00) == 0xDF00) return true; @@ -2073,8 +2144,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * +---------+---------+-------------------------+ */ addr = end_addr - 4; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32), - (u8 *)&instr32, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); if ((instr32 & 0x0F000000) == 0x0F000000 && (instr32 & 0xF0000000) != 0xF0000000) return true; @@ -2090,8 +2161,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * +-----------------------+---------+-----------+ */ addr = end_addr - 4; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32), - (u8 *)&instr32, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); if ((instr32 & 0xFFE0001F) == 0xd4000001) return true; @@ -2107,7 +2178,6 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, u64 magic) { - u8 trace_chan_id = tidq->trace_chan_id; struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; @@ -2122,7 +2192,7 @@ static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, */ if (magic == __perf_cs_etmv4_magic) { if (packet->exception_number == CS_ETMV4_EXC_CALL && - cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, + cs_etm__is_svc_instr(etmq, tidq, prev_packet, prev_packet->end_addr)) return true; } @@ -2160,7 +2230,6 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, u64 magic) { - u8 trace_chan_id = tidq->trace_chan_id; struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; @@ -2186,7 +2255,7 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, * (SMC, HVC) are taken as sync exceptions. */ if (packet->exception_number == CS_ETMV4_EXC_CALL && - !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, + !cs_etm__is_svc_instr(etmq, tidq, prev_packet, prev_packet->end_addr)) return true; @@ -2210,7 +2279,6 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, { struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; - u8 trace_chan_id = tidq->trace_chan_id; u64 magic; int ret; @@ -2291,11 +2359,11 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT) && - cs_etm__is_svc_instr(etmq, trace_chan_id, - packet, packet->start_addr)) + cs_etm__is_svc_instr(etmq, tidq, packet, packet->start_addr)) { prev_packet->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET; + } break; case CS_ETM_DISCONTINUITY: /* @@ -2376,6 +2444,7 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT; break; + case CS_ETM_CONTEXT: case CS_ETM_EMPTY: default: break; @@ -2451,6 +2520,19 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, */ cs_etm__sample(etmq, tidq); break; + case CS_ETM_CONTEXT: + /* + * Update context but don't swap packet. Keep the + * previous one for branch source address info, if + * tracing the kernel the context packet will be emitted + * between two ranges. + */ + ret = cs_etm__etmq_update_thread(etmq, tidq->packet->el, + tidq->packet->tid, + &tidq->frontend_thread); + if (ret) + goto out; + break; case CS_ETM_EXCEPTION: case CS_ETM_EXCEPTION_RET: /* @@ -2479,6 +2561,7 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, } } +out: return ret; } @@ -2602,7 +2685,7 @@ static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, if (!tidq) continue; - if (tid == -1 || thread__tid(tidq->thread) == tid) + if (tid == -1 || thread__tid(tidq->frontend_thread) == tid) cs_etm__run_per_thread_timeless_decoder(etmq); } else cs_etm__run_per_cpu_timeless_decoder(etmq); @@ -3079,6 +3162,9 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); + if (!etmq) + return -EINVAL; + /* * If this AUX event was inside this buffer somewhere, create a new auxtrace event * based on the sizes of the aux event, and queue that fragment. @@ -3310,7 +3396,7 @@ static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq) */ if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, 0x0L, ((u64) -1L), - cs_etm__mem_access)) + cs_etm__decoder_mem_access)) goto out_free_decoder; zfree(&t_params); @@ -3366,6 +3452,18 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, /* First the global part */ ptr = (u64 *) auxtrace_info->priv; num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; + + /* + * Bound num_cpu by the event size: the global header consumes + * CS_ETM_HEADER_SIZE bytes, and each CPU needs at least one u64 + * metadata entry after that. + */ + priv_size = total_size - event_header_size - INFO_HEADER_SIZE - + CS_ETM_HEADER_SIZE; + if (num_cpu <= 0 || priv_size <= 0 || + num_cpu > priv_size / (int)sizeof(u64)) + return -EINVAL; + metadata = zalloc(sizeof(*metadata) * num_cpu); if (!metadata) return -ENOMEM; @@ -3404,7 +3502,13 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, goto err_free_metadata; } - if ((int) metadata[j][CS_ETM_CPU] > max_cpu) + /* CPU id comes from perf.data and must fit max_cpu + 1 without overflow */ + if (metadata[j][CS_ETM_CPU] >= INT_MAX) { + err = -EINVAL; + goto err_free_metadata; + } + + if ((int)metadata[j][CS_ETM_CPU] > max_cpu) max_cpu = metadata[j][CS_ETM_CPU]; } diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h index aa9bb4a32ecaf..b81099c2b301c 100644 --- a/tools/perf/util/cs-etm.h +++ b/tools/perf/util/cs-etm.h @@ -158,6 +158,7 @@ enum cs_etm_sample_type { CS_ETM_DISCONTINUITY, CS_ETM_EXCEPTION, CS_ETM_EXCEPTION_RET, + CS_ETM_CONTEXT, }; enum cs_etm_isa { @@ -184,6 +185,8 @@ struct cs_etm_packet { u8 last_instr_size; u8 trace_chan_id; int cpu; + int el; + pid_t tid; }; #define CS_ETM_PACKET_MAX_BUFFER 1024 @@ -259,8 +262,9 @@ enum cs_etm_pid_fmt { #include int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu); enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq); -int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid, - u8 trace_chan_id, ocsd_ex_level el); +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, ocsd_ex_level el, + pid_t tid); bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq); void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, u8 trace_chan_id); diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c index d526c91312ed8..6a8d00c803942 100644 --- a/tools/perf/util/data-convert-json.c +++ b/tools/perf/util/data-convert-json.c @@ -177,6 +177,7 @@ static int process_sample_event(const struct perf_tool *tool, if (perf_time__ranges_skip_sample(c->ptime_range, c->range_num, sample->time)) { ++c->skipped; + addr_location__exit(&al); return 0; } diff --git a/tools/perf/util/debuginfo.c b/tools/perf/util/debuginfo.c index 0e35c13abd041..84a78b30ceac1 100644 --- a/tools/perf/util/debuginfo.c +++ b/tools/perf/util/debuginfo.c @@ -42,6 +42,7 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, { GElf_Addr dummy; int fd; + bool fd_consumed = false; fd = open(path, O_RDONLY); if (fd < 0) @@ -55,6 +56,7 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd); if (!dbg->mod) goto error; + fd_consumed = true; dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias); if (!dbg->dbg) @@ -62,13 +64,14 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, dwfl_module_build_id(dbg->mod, &dbg->build_id, &dummy); - dwfl_report_end(dbg->dwfl, NULL, NULL); + if (dwfl_report_end(dbg->dwfl, NULL, NULL) != 0) + goto error; return 0; error: if (dbg->dwfl) dwfl_end(dbg->dwfl); - else + if (!fd_consumed) close(fd); memset(dbg, 0, sizeof(*dbg)); @@ -167,7 +170,7 @@ int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, /* Search the relocation related .text section */ for (i = 0; i < n; i++) { p = dwfl_module_relocation_info(dbg->mod, i, &shndx); - if (strcmp(p, ".text") == 0) { + if (p && strcmp(p, ".text") == 0) { /* OK, get the section header */ scn = elf_getscn(elf, shndx); if (!scn) diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index 59ba88e1f7443..0a1a7e9cf3efe 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -1577,8 +1577,11 @@ int symbol__disassemble(struct symbol *sym, struct annotate_args *args) if (dso__decompress_kmodule_path(dso, symfs_filename, tmp, sizeof(tmp)) < 0) return -1; - decomp = true; - strcpy(symfs_filename, tmp); + /* empty pathname means file wasn't actually compressed */ + if (tmp[0] != '\0') { + decomp = true; + strcpy(symfs_filename, tmp); + } } /* diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index b791e1b6b2cf0..170637a9b8e0a 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -343,10 +343,16 @@ int filename__decompress(const char *name, char *pathname, * To keep this transparent, we detect this and return the file * descriptor to the uncompressed file. */ - if (!compressions[comp].is_compressed(name)) - return open(name, O_RDONLY); + if (!compressions[comp].is_compressed(name)) { + fd = open(name, O_RDONLY | O_CLOEXEC); + if (fd < 0) + *err = errno; + if (pathname && len > 0) + pathname[0] = '\0'; + return fd; + } - fd = mkstemp(tmpbuf); + fd = mkostemp(tmpbuf, O_CLOEXEC); if (fd < 0) { *err = errno; return -1; @@ -594,12 +600,28 @@ static char *dso__get_filename(struct dso *dso, const char *root_dir, size_t len = sizeof(newpath); if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { - errno = *dso__load_errno(dso); + /* + * Use a standard errno value, not the negative custom + * DSO_LOAD_ERRNO stored in dso__load_errno(dso): + * __open_dso() computes fd = -errno, so a negative + * errno produces a positive fd that looks valid. + */ + errno = EIO; goto out; } - *decomp = true; - strcpy(name, newpath); + /* empty pathname means file wasn't actually compressed */ + if (newpath[0] != '\0') { + char *tmp = strdup(newpath); + + if (!tmp) { + unlink(newpath); + goto out; + } + free(name); + name = tmp; + *decomp = true; + } } return name; @@ -864,6 +886,12 @@ static ssize_t bpf_read(struct dso *dso, u64 offset, char *data) return -1; } + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(node->info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + len = node->info_linear->info.jited_prog_len; buf = (u8 *)(uintptr_t)node->info_linear->info.jited_prog_insns; @@ -1766,7 +1794,7 @@ void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine) if (machine__is_default_guest(machine)) return; - sprintf(path, "%s/sys/kernel/notes", machine->root_dir); + snprintf(path, sizeof(path), "%s/sys/kernel/notes", machine->root_dir); sysfs__read_build_id(path, &bid); dso__set_build_id(dso, &bid); } @@ -1904,7 +1932,7 @@ static const u8 *__dso__read_symbol(struct dso *dso, const char *symfs_filename, int saved_errno; nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); - fd = open(symfs_filename, O_RDONLY); + fd = open(symfs_filename, O_RDONLY | O_CLOEXEC); saved_errno = errno; nsinfo__mountns_exit(&nsc); if (fd < 0) { @@ -1972,6 +2000,10 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, return NULL; } info_linear = info_node->info_linear; + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } assert(len <= info_linear->info.jited_prog_len); *out_buf_len = len; return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c index 92db2fccc788a..d7160f87ac7d7 100644 --- a/tools/perf/util/dwarf-aux.c +++ b/tools/perf/util/dwarf-aux.c @@ -125,7 +125,8 @@ int cu_find_lineinfo(Dwarf_Die *cu_die, Dwarf_Addr addr, && die_entrypc(&die_mem, &faddr) == 0 && faddr == addr) { *fname = die_get_decl_file(&die_mem); - dwarf_decl_line(&die_mem, lineno); + if (dwarf_decl_line(&die_mem, lineno) != 0) + return -ENOENT; goto out; } @@ -171,7 +172,6 @@ int cu_walk_functions_at(Dwarf_Die *cu_die, Dwarf_Addr addr, } return ret; - } /** @@ -460,7 +460,7 @@ int die_get_data_member_location(Dwarf_Die *mb_die, Dwarf_Word *offs) size_t nexpr; int ret; - if (dwarf_attr(mb_die, DW_AT_data_member_location, &attr) == NULL) + if (dwarf_attr_integrate(mb_die, DW_AT_data_member_location, &attr) == NULL) return -ENOENT; if (dwarf_formudata(&attr, offs) != 0) { @@ -620,7 +620,7 @@ Dwarf_Die *die_find_tailfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, ad.addr = addr; ad.die_mem = die_mem; /* dwarf_getscopes can't find subprogram. */ - if (!dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0)) + if (dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0) <= 0) return NULL; else return die_mem; @@ -659,7 +659,7 @@ Dwarf_Die *die_find_realfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, ad.addr = addr; ad.die_mem = die_mem; /* dwarf_getscopes can't find subprogram. */ - if (!dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0)) + if (dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0) <= 0) return NULL; else return die_mem; @@ -796,8 +796,7 @@ static int __die_walk_instances_cb(Dwarf_Die *inst, void *data) /* Ignore redundant instances */ if (dwarf_tag(inst) == DW_TAG_inlined_subroutine) { - dwarf_decl_line(origin, &tmp); - if (die_get_call_lineno(inst) == tmp) { + if (dwarf_decl_line(origin, &tmp) == 0 && die_get_call_lineno(inst) == tmp) { tmp = die_get_decl_fileno(origin); if (die_get_call_fileno(inst) == tmp) return DIE_FIND_CB_CONTINUE; @@ -951,11 +950,6 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) cu_die = dwarf_diecu(rt_die, &die_mem, NULL, NULL); dwarf_decl_line(rt_die, &decl); decf = die_get_decl_file(rt_die); - if (!decf) { - pr_debug2("Failed to get the declared file name of %s\n", - dwarf_diename(rt_die)); - return -EINVAL; - } } else cu_die = rt_die; if (!cu_die) { @@ -999,11 +993,12 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) if (die_find_inlinefunc(rt_die, addr, &die_mem)) { /* Call-site check */ inf = die_get_call_file(&die_mem); - if ((inf && !strcmp(inf, decf)) && + if ((inf == decf || (inf && decf && !strcmp(inf, decf))) && die_get_call_lineno(&die_mem) == lineno) goto found; - dwarf_decl_line(&die_mem, &inl); + if (dwarf_decl_line(&die_mem, &inl) != 0) + inl = 0; if (inl != decl || decf != die_get_decl_file(&die_mem)) continue; @@ -1035,8 +1030,10 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) .data = data, .retval = 0, }; - dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0); - ret = param.retval; + if (dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0) < 0) + ret = -EINVAL; + else + ret = param.retval; } return ret; @@ -1940,10 +1937,12 @@ static bool die_get_postprologue_addr(unsigned long entrypc_idx, break; } - dwarf_lineaddr(line, postprologue_addr); - if (*postprologue_addr >= highpc) - dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), - postprologue_addr); + if (dwarf_lineaddr(line, postprologue_addr) != 0) + return false; + if (*postprologue_addr >= highpc) { + if (dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), postprologue_addr) != 0) + return false; + } return true; } diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h index a79968a2e573b..161f0bf980b6e 100644 --- a/tools/perf/util/dwarf-aux.h +++ b/tools/perf/util/dwarf-aux.h @@ -10,6 +10,11 @@ #include #include +static inline const char *die_name(Dwarf_Die *die) +{ + return dwarf_diename(die) ?: ""; +} + struct strbuf; /* Find the realpath of the target file */ diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h index c7052ac1f8562..25ba16290280d 100644 --- a/tools/perf/util/env.h +++ b/tools/perf/util/env.h @@ -185,6 +185,20 @@ const char *perf_env__pmu_mappings(struct perf_env *env); int perf_env__read_cpu_topology_map(struct perf_env *env); +/* + * Safe accessor for env->cpu[] topology array. env->cpu can be NULL when + * reading old-format perf.data that predates topology information — + * process_cpu_topology() in header.c frees it while nr_cpus_avail remains + * set, so callers must not index env->cpu[] without this check. + */ +static inline struct cpu_topology_map * +perf_env__get_cpu_topology(struct perf_env *env, struct perf_cpu cpu) +{ + if (env->cpu && cpu.cpu >= 0 && cpu.cpu < env->nr_cpus_avail) + return &env->cpu[cpu.cpu]; + return NULL; +} + void cpu_cache_level__free(struct cpu_cache_level *cache); const char *perf_env__arch(struct perf_env *env); diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c index 66f4843bb235d..ea75816d126a1 100644 --- a/tools/perf/util/event.c +++ b/tools/perf/util/event.c @@ -14,6 +14,7 @@ #include #include "cpumap.h" #include "dso.h" +#include "env.h" #include "event.h" #include "debug.h" #include "hist.h" @@ -836,8 +837,18 @@ int machine__resolve(struct machine *machine, struct addr_location *al, if (al->cpu >= 0) { struct perf_env *env = machine->env; - if (env && env->cpu) - al->socket = env->cpu[al->cpu].socket_id; + /* + * Bounds-check al->cpu (s32) before casting to struct perf_cpu + * (int16_t): without this, e.g. 65536 truncates to 0 and silently + * returns CPU 0's topology. Can go once perf_cpu.cpu is widened. + */ + if (env && al->cpu < env->nr_cpus_avail) { + struct cpu_topology_map *topo; + + topo = perf_env__get_cpu_topology(env, (struct perf_cpu){ al->cpu }); + if (topo) + al->socket = topo->socket_id; + } } /* Account for possible out-of-order switch events. */ diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index f30e48eb3fc32..7bea1c41ced79 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -158,15 +158,25 @@ int do_write(struct feat_fd *ff, const void *buf, size_t size) /* Return: 0 if succeeded, -ERR if failed. */ static int do_write_bitmap(struct feat_fd *ff, unsigned long *set, u64 size) { - u64 *p = (u64 *) set; + size_t byte_size = BITS_TO_LONGS(size) * sizeof(unsigned long); int i, ret; ret = do_write(ff, &size, sizeof(size)); if (ret < 0) return ret; + /* + * The on-disk format uses u64 elements, but the in-memory bitmap + * uses unsigned long, which is only 4 bytes on 32-bit architectures. + * Copy with bounded size so the last element doesn't read past the + * bitmap allocation when BITS_TO_LONGS(size) is odd. + */ for (i = 0; (u64) i < BITS_TO_U64(size); i++) { - ret = do_write(ff, p + i, sizeof(*p)); + u64 val = 0; + size_t off = i * sizeof(val); + + memcpy(&val, (char *)set + off, min(sizeof(val), byte_size - off)); + ret = do_write(ff, &val, sizeof(val)); if (ret < 0) return ret; } @@ -297,7 +307,20 @@ static int do_read_bitmap(struct feat_fd *ff, unsigned long **pset, u64 *psize) if (ret) return ret; - set = bitmap_zalloc(size); + /* Bitmap APIs use int for nbits; reject u64 values that truncate. */ + if (size > INT_MAX || + BITS_TO_U64(size) > (ff->size - ff->offset) / sizeof(u64)) { + pr_debug("do_read_bitmap: size %" PRIu64 " exceeds section bounds\n", size); + return -1; + } + + /* + * bitmap_zalloc() allocates in unsigned long units, which are only + * 4 bytes on 32-bit architectures. The read loop below casts the + * buffer to u64 * and writes 8-byte elements, so allocate in u64 + * units to ensure the buffer is large enough. + */ + set = calloc(BITS_TO_U64(size), sizeof(u64)); if (!set) return -ENOMEM; @@ -2035,8 +2058,7 @@ static void print_bpf_prog_info(struct feat_fd *ff __maybe_unused, FILE *fp) node = rb_entry(next, struct bpf_prog_info_node, rb_node); next = rb_next(&node->rb_node); - __bpf_event__print_bpf_prog_info(&node->info_linear->info, - env, fp); + __bpf_event__print_bpf_prog_info(node->info_linear, env, fp); } up_read(&env->bpf_progs.lock); @@ -2135,9 +2157,28 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (do_read_u32(ff, &nre)) goto error; + /* Size of each of the nre attributes. */ if (do_read_u32(ff, &sz)) goto error; + /* + * Require at least one event with an attr no smaller than the + * first published struct, and reject sz values where + * sz + sizeof(u32) would overflow size_t (possible on 32-bit) + * or nre == UINT32_MAX where nre + 1 wraps to 0 in the calloc. + * + * The minimum section footprint per event is sz bytes for the + * attr plus a u32 for the id count, check that nre events fit. + */ + if (!nre || sz < PERF_ATTR_SIZE_VER0 || + sz > ff->size || (size_t)sz > SIZE_MAX - sizeof(u32) || + nre == UINT32_MAX || + nre > (ff->size - ff->offset) / (sz + sizeof(u32))) { + pr_err("Invalid HEADER_EVENT_DESC: nre=%u sz=%u (min %d)\n", + nre, sz, PERF_ATTR_SIZE_VER0); + goto error; + } + /* buffer to hold on file attr struct */ buf = malloc(sz); if (!buf) @@ -2153,6 +2194,9 @@ static struct evsel *read_event_desc(struct feat_fd *ff) msz = sz; for (i = 0, evsel = events; i < nre; evsel++, i++) { + struct perf_event_attr *attr = buf; + u32 attr_size; + evsel->core.idx = i; /* @@ -2162,6 +2206,32 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (__do_read(ff, buf, sz)) goto error; + /* Reject before attr_swap to prevent OOB via bswap_safe() */ + attr_size = ff->ph->needs_swap ? bswap_32(attr->size) : attr->size; + /* ABI0: size == 0 means the producer didn't set it */ + if (!attr_size) { + attr_size = PERF_ATTR_SIZE_VER0; + /* + * Write back so free_event_desc() doesn't + * treat this event as the end-of-array sentinel + * (it iterates while attr.size != 0). + * + * Only for native — the swap path must NOT + * write native-endian VER0 here because + * perf_event__attr_swap() would re-swap it + * to 0x40000000, defeating bswap_safe() bounds. + * perf_event__attr_swap() has its own ABI0 + * fallback that sets VER0 after swapping. + */ + if (!ff->ph->needs_swap) + attr->size = attr_size; + } + if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size > sz) { + pr_err("Event %d attr.size (%u) invalid (min: %d, max: %u)\n", + i, attr_size, PERF_ATTR_SIZE_VER0, sz); + goto error; + } + if (ff->ph->needs_swap) perf_event__attr_swap(buf); @@ -2183,6 +2253,12 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (!nr) continue; + /* Prevent oversized allocation from crafted nr */ + if (nr > (ff->size - ff->offset) / sizeof(*id)) { + pr_err("Event %d: id count %u exceeds remaining section\n", i, nr); + goto error; + } + id = calloc(nr, sizeof(*id)); if (!id) goto error; @@ -3357,7 +3433,8 @@ static int process_mem_topology(struct feat_fd *ff, return -1; } - if (ff->size < 3 * sizeof(u64) + nr * 2 * sizeof(u64)) { + /* Per node: node_id(u64) + mem_size(u64) + bitmap_nr_bits(u64) */ + if (ff->size < 3 * sizeof(u64) + nr * 3 * sizeof(u64)) { pr_err("Invalid HEADER_MEM_TOPOLOGY: section too small (%zu) for %llu nodes\n", ff->size, (unsigned long long)nr); return -1; @@ -3392,7 +3469,7 @@ static int process_mem_topology(struct feat_fd *ff, out: if (ret) - free(nodes); + memory_node__delete_nodes(nodes, nr); return ret; } diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 747fdc455c80e..aca9ff15518a2 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -2963,9 +2963,10 @@ int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool sh ev_name, sample_freq_str, enable_ref ? ref : " ", nr_events); - if (hists->uid_filter_str) - printed += snprintf(bf + printed, size - printed, - ", UID: %s", hists->uid_filter_str); + if (hists->uid_filter_str) { + printed += scnprintf(bf + printed, size - printed, + ", UID: %s", hists->uid_filter_str); + } if (thread) { if (hists__has(hists, thread)) { printed += scnprintf(bf + printed, size - printed, @@ -3040,7 +3041,7 @@ static void hists__delete_remaining_entries(struct rb_root_cached *root) } } -static void hists__delete_all_entries(struct hists *hists) +void hists__delete_all_entries(struct hists *hists) { hists__delete_entries(hists); hists__delete_remaining_entries(&hists->entries_in_array[0]); diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index d97a4efb92505..3a59200d2517e 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -392,6 +392,7 @@ int hists__collapse_resort(struct hists *hists, struct ui_progress *prog); void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel); void hists__delete_entries(struct hists *hists); +void hists__delete_all_entries(struct hists *hists); void hists__output_recalc_col_len(struct hists *hists, int max_rows); struct hist_entry *hists__get_entry(struct hists *hists, int idx); diff --git a/tools/perf/util/hwmon_pmu.c b/tools/perf/util/hwmon_pmu.c index fb3ffa8d32ad2..ed544dca70c38 100644 --- a/tools/perf/util/hwmon_pmu.c +++ b/tools/perf/util/hwmon_pmu.c @@ -202,7 +202,8 @@ bool parse_hwmon_filename(const char *filename, fn_item_len = strlen(fn_item); if (fn_item_len > 6 && !strcmp(&fn_item[fn_item_len - 6], "_alarm")) { assert(strlen(LONGEST_HWMON_ITEM_STR) < sizeof(fn_type)); - strlcpy(fn_type, fn_item, fn_item_len - 5); + /* fn_item_len - 5 strips "_alarm"; clamp to buffer size */ + strlcpy(fn_type, fn_item, min_t(size_t, fn_item_len - 5, sizeof(fn_type))); fn_item = fn_type; *alarm = true; } @@ -289,13 +290,16 @@ static int hwmon_pmu__read_events(struct hwmon_pmu *pmu) if (fd < 0) continue; - read_len = read(fd, buf, sizeof(buf)); + read_len = read(fd, buf, sizeof(buf) - 1); while (read_len > 0 && buf[read_len - 1] == '\n') read_len--; - if (read_len > 0) - buf[read_len] = '\0'; + if (read_len <= 0) { + close(fd); + continue; + } + buf[read_len] = '\0'; if (buf[0] == '\0') { pr_debug("hwmon_pmu: empty label file %s %s\n", @@ -431,8 +435,8 @@ static size_t hwmon_pmu__describe_items(struct hwmon_pmu *hwm, char *out_buf, si hwmon_item_strs[bit], is_alarm ? "_alarm" : ""); fd = openat(dir, buf, O_RDONLY); - if (fd > 0) { - ssize_t read_len = read(fd, buf, sizeof(buf)); + if (fd >= 0) { + ssize_t read_len = read(fd, buf, sizeof(buf) - 1); while (read_len > 0 && buf[read_len - 1] == '\n') read_len--; @@ -442,12 +446,12 @@ static size_t hwmon_pmu__describe_items(struct hwmon_pmu *hwm, char *out_buf, si buf[read_len] = '\0'; val = strtoll(buf, /*endptr=*/NULL, 10); - len += snprintf(out_buf + len, out_buf_len - len, "%s%s%s=%g%s", - len == 0 ? " " : ", ", - hwmon_item_strs[bit], - is_alarm ? "_alarm" : "", - (double)val / 1000.0, - hwmon_units[key.type]); + len += scnprintf(out_buf + len, out_buf_len - len, "%s%s%s=%g%s", + len == 0 ? " " : ", ", + hwmon_item_strs[bit], + is_alarm ? "_alarm" : "", + (double)val / 1000.0, + hwmon_units[key.type]); } close(fd); } @@ -514,14 +518,14 @@ int hwmon_pmu__for_each_event(struct perf_pmu *pmu, void *state, pmu_event_callb int ret; size_t len; - len = snprintf(alias_buf, sizeof(alias_buf), "%s%d", - hwmon_type_strs[key.type], key.num); + scnprintf(alias_buf, sizeof(alias_buf), "%s%d", + hwmon_type_strs[key.type], key.num); if (!info.name) { info.name = info.alias; info.alias = NULL; } - len = snprintf(desc_buf, sizeof(desc_buf), "%s in unit %s named %s.", + len = scnprintf(desc_buf, sizeof(desc_buf), "%s in unit %s named %s.", hwmon_desc[key.type], pmu->name + 6, value->label ?: info.name); @@ -816,7 +820,7 @@ int evsel__hwmon_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) count = perf_counts(evsel->counts, cpu_map_idx, thread); fd = FD(evsel, cpu_map_idx, thread); - len = pread(fd, buf, sizeof(buf), 0); + len = pread(fd, buf, sizeof(buf) - 1, 0); if (len <= 0) { count->lost++; return -EINVAL; diff --git a/tools/perf/util/intel-bts.c b/tools/perf/util/intel-bts.c index 382255393fb3b..0b18ebd13f7c8 100644 --- a/tools/perf/util/intel-bts.c +++ b/tools/perf/util/intel-bts.c @@ -303,7 +303,8 @@ static int intel_bts_synth_branch_sample(struct intel_bts_queue *btsq, event.sample.header.size = bts->branches_event_size; ret = perf_event__synthesize_sample(&event, bts->branches_sample_type, - 0, &sample); + /*read_format=*/0, /*branch_sample_type=*/0, + &sample); if (ret) return ret; } diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c index 72c7a4e15d617..f90fcbc4302df 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c @@ -220,7 +220,6 @@ const char *dump_insn(struct perf_insn *x, uint64_t ip __maybe_unused, { struct insn insn; int n, i, ret; - int left; ret = insn_decode(&insn, inbuf, inlen, x->is64bit ? INSN_MODE_64 : INSN_MODE_32); @@ -229,13 +228,9 @@ const char *dump_insn(struct perf_insn *x, uint64_t ip __maybe_unused, return ""; if (lenp) *lenp = insn.length; - left = sizeof(x->out); - n = snprintf(x->out, left, "insn: "); - left -= n; - for (i = 0; i < insn.length; i++) { - n += snprintf(x->out + n, left, "%02x ", inbuf[i]); - left -= n; - } + n = scnprintf(x->out, sizeof(x->out), "insn: "); + for (i = 0; i < insn.length; i++) + n += scnprintf(x->out + n, sizeof(x->out) - n, "%02x ", inbuf[i]); return x->out; } diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index fc9eec8b54b82..d9c86ac497486 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -1307,7 +1307,8 @@ static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, goto out_free; } - if (pt->synth_opts.last_branch || pt->synth_opts.other_events) { + if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch || + pt->synth_opts.other_events) { unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz); ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt); @@ -1728,11 +1729,30 @@ static void intel_pt_prep_b_sample(struct intel_pt *pt, event->sample.header.misc = sample->cpumode; } -static int intel_pt_inject_event(union perf_event *event, +static int intel_pt_inject_event(struct intel_pt *pt, union perf_event *event, struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && pt->session && pt->session->evlist) + evsel = evlist__id2evsel(pt->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } static inline int intel_pt_opt_inject(struct intel_pt *pt, @@ -1742,7 +1762,7 @@ static inline int intel_pt_opt_inject(struct intel_pt *pt, if (!pt->synth_opts.inject) return 0; - return intel_pt_inject_event(event, sample, type); + return intel_pt_inject_event(pt, event, sample, type); } static int intel_pt_deliver_synth_event(struct intel_pt *pt, @@ -2486,7 +2506,7 @@ static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evse intel_pt_add_xmm(intr_regs, pos, items, regs_mask); } - if (sample_type & PERF_SAMPLE_BRANCH_STACK) { + if ((sample_type | evsel->synth_sample_type) & PERF_SAMPLE_BRANCH_STACK) { if (items->mask[INTEL_PT_LBR_0_POS] || items->mask[INTEL_PT_LBR_1_POS] || items->mask[INTEL_PT_LBR_2_POS]) { @@ -2557,7 +2577,8 @@ static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evse sample.transaction = txn; } - ret = intel_pt_deliver_synth_event(pt, event, &sample, sample_type); + ret = intel_pt_deliver_synth_event(pt, event, &sample, + sample_type | evsel->synth_sample_type); perf_sample__exit(&sample); return ret; } diff --git a/tools/perf/util/libdw.c b/tools/perf/util/libdw.c index 2169778841030..15bf646d104f5 100644 --- a/tools/perf/util/libdw.c +++ b/tools/perf/util/libdw.c @@ -60,7 +60,10 @@ struct Dwfl *dso__libdw_dwfl(struct dso *dso) return NULL; } - dwfl_report_end(dwfl, /*removed=*/NULL, /*arg=*/NULL); + if (dwfl_report_end(dwfl, /*removed=*/NULL, /*arg=*/NULL) != 0) { + dwfl_end(dwfl); + return NULL; + } dso__set_libdw(dso, dwfl); return dwfl; @@ -72,18 +75,19 @@ struct libdw_a2l_cb_args { struct inline_node *node; char *leaf_srcline; bool leaf_srcline_used; + int err; }; static int libdw_a2l_cb(Dwarf_Die *die, void *_args) { struct libdw_a2l_cb_args *args = _args; - struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, dwarf_diename(die)); + struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, die_name(die)); const char *call_fname = die_get_call_file(die); char *call_srcline = srcline__unknown; struct inline_list *ilist; if (!inline_sym) - return -ENOMEM; + goto abort_enomem; /* Assign caller information to the parent. */ if (call_fname) @@ -103,12 +107,27 @@ static int libdw_a2l_cb(Dwarf_Die *die, void *_args) /* Add this symbol to the chain as the leaf. */ if (!args->leaf_srcline_used) { - inline_list__append_tail(inline_sym, args->leaf_srcline, args->node); + if (inline_list__append_tail(inline_sym, args->leaf_srcline, args->node) != 0) + goto abort_delete_sym; args->leaf_srcline_used = true; } else { - inline_list__append_tail(inline_sym, strdup(args->leaf_srcline), args->node); + char *srcline = strdup(args->leaf_srcline); + + if (!srcline) + goto abort_delete_sym; + if (inline_list__append_tail(inline_sym, srcline, args->node) != 0) { + free(srcline); + goto abort_delete_sym; + } } return 0; + +abort_delete_sym: + if (inline_sym->inlined) + symbol__delete(inline_sym); +abort_enomem: + args->err = -ENOMEM; + return DWARF_CB_ABORT; } int libdw__addr2line(u64 addr, char **file, unsigned int *line_nr, @@ -162,11 +181,29 @@ int libdw__addr2line(u64 addr, char **file, unsigned int *line_nr, .leaf_srcline = srcline_from_fileline(src ?: "", lineno), }; + if (!args.leaf_srcline) { + if (file && *file) { + free(*file); + *file = NULL; + } + return 0; + } + /* Walk from the parent down to the leaf. */ - cu_walk_functions_at(cudie, addr, libdw_a2l_cb, &args); + if (cudie) + cu_walk_functions_at(cudie, addr, libdw_a2l_cb, &args); if (!args.leaf_srcline_used) free(args.leaf_srcline); + + if (args.err) { + if (file && *file) { + free(*file); + *file = NULL; + } + inline_node__clear_frames(node); + return 0; + } } return 1; } diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index e76f8c86e62a6..d48d96e4b37dc 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -327,7 +327,7 @@ struct machine *machines__findnew(struct machines *machines, pid_t pid) if ((pid != HOST_KERNEL_ID) && (pid != DEFAULT_GUEST_KERNEL_ID) && (symbol_conf.guestmount)) { - sprintf(path, "%s/%d", symbol_conf.guestmount, pid); + snprintf(path, sizeof(path), "%s/%d", symbol_conf.guestmount, pid); if (access(path, R_OK)) { static struct strlist *seen; @@ -1239,9 +1239,9 @@ int machines__create_guest_kernel_maps(struct machines *machines) namelist[i]->d_name); continue; } - sprintf(path, "%s/%s/proc/kallsyms", - symbol_conf.guestmount, - namelist[i]->d_name); + snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", + symbol_conf.guestmount, + namelist[i]->d_name); ret = access(path, R_OK); if (ret) { pr_debug("Can't access file %s\n", path); @@ -1319,7 +1319,7 @@ static char *get_kernel_version(const char *root_dir) char *name, *tmp; const char *prefix = "Linux version "; - sprintf(version, "%s/proc/version", root_dir); + snprintf(version, sizeof(version), "%s/proc/version", root_dir); file = fopen(version, "r"); if (!file) return NULL; @@ -1522,22 +1522,30 @@ static void machine__set_kernel_mmap(struct machine *machine, map__set_end(machine->vmlinux_map, ~0ULL); } -static int machine__update_kernel_mmap(struct machine *machine, - u64 start, u64 end) +struct kernel_mmap_mutation_ctx { + u64 start; + u64 end; +}; + +static int kernel_mmap_mutate_cb(struct map *map, void *data) { - struct map *orig, *updated; - int err; + struct kernel_mmap_mutation_ctx *ctx = data; - orig = machine->vmlinux_map; - updated = map__get(orig); + map__set_start(map, ctx->start); + map__set_end(map, ctx->end); + if (ctx->start == 0 && ctx->end == 0) + map__set_end(map, ~0ULL); + return 0; +} - machine->vmlinux_map = updated; - maps__remove(machine__kernel_maps(machine), orig); - machine__set_kernel_mmap(machine, start, end); - err = maps__insert(machine__kernel_maps(machine), updated); - map__put(orig); +static int machine__update_kernel_mmap(struct machine *machine, + u64 start, u64 end) +{ + struct kernel_mmap_mutation_ctx ctx = { .start = start, .end = end }; - return err; + return maps__mutate_mapping(machine__kernel_maps(machine), + machine->vmlinux_map, + kernel_mmap_mutate_cb, &ctx); } int machine__create_kernel_maps(struct machine *machine) diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index 81a97ac34077e..6cdafded61bcb 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -575,6 +575,48 @@ void maps__remove(struct maps *maps, struct map *map) #endif } +/** + * maps__mutate_mapping - Apply write-protected mutations to a map. + * @maps: The maps collection containing the map. + * @map: The map to mutate. + * @mutate_cb: Callback function that performs the actual mutations. + * @data: Private data passed to the callback. + * + * This acquires the write lock on the maps semaphore to safely protect + * concurrent readers from seeing partially mutated or unsorted map boundaries. + * + * WARNING: Acquiring down_write() here can trigger a recursive self-deadlock if + * the caller already holds the read lock (e.g., during maps__for_each_map() or + * maps__find() iteration paths that trigger lazy symbol loading). To completely + * avoid this deadlock, all kernel/module maps must be pre-loaded up-front (via + * maps__load_maps()) under a clean, single-threaded context before entering + * multi-threaded event processing loops. + */ +int maps__mutate_mapping(struct maps *maps, struct map *map, + int (*mutate_cb)(struct map *map, void *data), void *data) +{ + int err = 0; + + if (maps) { + down_write(maps__lock(maps)); + + err = mutate_cb(map, data); + + RC_CHK_ACCESS(maps)->maps_by_address_sorted = false; + RC_CHK_ACCESS(maps)->maps_by_name_sorted = false; + + up_write(maps__lock(maps)); + +#ifdef HAVE_LIBDW_SUPPORT + libdw__invalidate_dwfl(maps, maps__libdw_addr_space_dwfl(maps)); +#endif + } else { + err = mutate_cb(map, data); + } + + return err; +} + bool maps__empty(struct maps *maps) { bool res; @@ -625,6 +667,41 @@ int maps__for_each_map(struct maps *maps, int (*cb)(struct map *map, void *data) return ret; } +int maps__load_maps(struct maps *maps) +{ + struct map **maps_copy; + unsigned int nr_maps; + int err = 0; + + if (!maps) + return 0; + + down_read(maps__lock(maps)); + nr_maps = maps__nr_maps(maps); + if (nr_maps == 0) { + up_read(maps__lock(maps)); + return 0; + } + maps_copy = calloc(nr_maps, sizeof(*maps_copy)); + if (!maps_copy) { + up_read(maps__lock(maps)); + return -ENOMEM; + } + for (unsigned int i = 0; i < nr_maps; i++) + maps_copy[i] = map__get(maps__maps_by_address(maps)[i]); + up_read(maps__lock(maps)); + + for (unsigned int i = 0; i < nr_maps; i++) { + if (map__load(maps_copy[i]) < 0) { + pr_warning("Failed to load map %s\n", dso__name(map__dso(maps_copy[i]))); + err = -1; + } + map__put(maps_copy[i]); + } + free(maps_copy); + return err; +} + void maps__remove_maps(struct maps *maps, bool (*cb)(struct map *map, void *data), void *data) { struct map **maps_by_address; @@ -667,40 +744,57 @@ struct symbol *maps__find_symbol(struct maps *maps, u64 addr, struct map **mapp) return result; } -struct maps__find_symbol_by_name_args { - struct map **mapp; - const char *name; - struct symbol *sym; -}; - -static int maps__find_symbol_by_name_cb(struct map *map, void *data) +struct symbol *maps__find_symbol_by_name(struct maps *maps, const char *name, struct map **mapp) { - struct maps__find_symbol_by_name_args *args = data; + struct map **maps_copy; + unsigned int nr_maps; + struct symbol *sym = NULL; - args->sym = map__find_symbol_by_name(map, args->name); - if (!args->sym) - return 0; + if (!maps) + return NULL; - if (!map__contains_symbol(map, args->sym)) { - args->sym = NULL; - return 0; + /* + * First, ensure all maps are loaded. We pre-load them outside of any + * read-to-write locks to avoid deadlocks. Even if some fail, we proceed. + */ + maps__load_maps(maps); + + /* + * Create a local snapshot of the maps while holding the read lock. + * This prevents deadlocking if iteration triggers further map insertions. + */ + down_read(maps__lock(maps)); + nr_maps = maps__nr_maps(maps); + maps_copy = calloc(nr_maps, sizeof(*maps_copy)); + if (maps_copy) { + for (unsigned int i = 0; i < nr_maps; i++) { + struct map *map = maps__maps_by_address(maps)[i]; + + maps_copy[i] = map__get(map); + } } + up_read(maps__lock(maps)); - if (args->mapp != NULL) - *args->mapp = map__get(map); - return 1; -} + if (!maps_copy) + return NULL; -struct symbol *maps__find_symbol_by_name(struct maps *maps, const char *name, struct map **mapp) -{ - struct maps__find_symbol_by_name_args args = { - .mapp = mapp, - .name = name, - .sym = NULL, - }; + for (unsigned int i = 0; i < nr_maps; i++) { + struct map *map = maps_copy[i]; + + sym = map__find_symbol_by_name(map, name); + if (sym && map__contains_symbol(map, sym)) { + if (mapp) + *mapp = map__get(map); + break; + } + sym = NULL; + } + + for (unsigned int i = 0; i < nr_maps; i++) + map__put(maps_copy[i]); - maps__for_each_map(maps, maps__find_symbol_by_name_cb, &args); - return args.sym; + free(maps_copy); + return sym; } int maps__find_ams(struct maps *maps, struct addr_map_symbol *ams) diff --git a/tools/perf/util/maps.h b/tools/perf/util/maps.h index 20c52084ba9e6..b3f00f5c32fcf 100644 --- a/tools/perf/util/maps.h +++ b/tools/perf/util/maps.h @@ -59,8 +59,11 @@ void maps__set_libdw_addr_space_dwfl(struct maps *maps, void *dwfl); size_t maps__fprintf(struct maps *maps, FILE *fp); +int maps__load_maps(struct maps *maps); int maps__insert(struct maps *maps, struct map *map); void maps__remove(struct maps *maps, struct map *map); +int maps__mutate_mapping(struct maps *maps, struct map *map, + int (*mutate_cb)(struct map *map, void *data), void *data); struct map *maps__find(struct maps *maps, u64 addr); struct symbol *maps__find_symbol(struct maps *maps, u64 addr, struct map **mapp); diff --git a/tools/perf/util/mmap.c b/tools/perf/util/mmap.c index b69f926d314b1..358e70c4f3edd 100644 --- a/tools/perf/util/mmap.c +++ b/tools/perf/util/mmap.c @@ -89,10 +89,10 @@ static int perf_mmap__aio_alloc(struct mmap *map, int idx) static void perf_mmap__aio_free(struct mmap *map, int idx) { - if (map->aio.data[idx]) { - munmap(map->aio.data[idx], mmap__mmap_len(map)); - map->aio.data[idx] = NULL; - } + if (!map->aio.data || !map->aio.data[idx]) + return; + munmap(map->aio.data[idx], mmap__mmap_len(map)); + map->aio.data[idx] = NULL; } static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, int affinity) @@ -104,9 +104,15 @@ static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, i int err = 0; if (affinity != PERF_AFFINITY_SYS && cpu__max_node() > 1) { + int node; + data = map->aio.data[idx]; mmap_len = mmap__mmap_len(map); - node_index = cpu__get_node(cpu); + node = cpu__get_node(cpu); + /* -1 sign-extends to ULONG_MAX, wrapping bitmap_zalloc(0) and OOB __set_bit */ + if (node < 0) + return 0; + node_index = node; node_mask = bitmap_zalloc(node_index + 1); if (!node_mask) { pr_err("Failed to allocate node mask for mbind: error %m\n"); @@ -135,6 +141,8 @@ static int perf_mmap__aio_alloc(struct mmap *map, int idx) static void perf_mmap__aio_free(struct mmap *map, int idx) { + if (!map->aio.data) + return; zfree(&(map->aio.data[idx])); } @@ -230,6 +238,8 @@ static void perf_mmap__aio_munmap(struct mmap *map __maybe_unused) void mmap__munmap(struct mmap *map) { bitmap_free(map->affinity_mask.bits); + map->affinity_mask.bits = NULL; + map->affinity_mask.nbits = 0; zstd_fini(&map->zstd_data); diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 23337d2fa2819..64544e1e6810f 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -314,7 +314,7 @@ static int perf_pmu__parse_scale(struct perf_pmu *pmu, struct perf_pmu_alias *al goto error; sret = read(fd, scale, sizeof(scale)-1); - if (sret < 0) + if (sret <= 0) goto error; if (scale[sret - 1] == '\n') @@ -346,7 +346,7 @@ static int perf_pmu__parse_unit(struct perf_pmu *pmu, struct perf_pmu_alias *ali return -1; sret = read(fd, alias->unit, UNIT_MAX_LEN); - if (sret < 0) + if (sret <= 0) goto error; close(fd); @@ -865,6 +865,12 @@ static char *pmu_id(const char *name) if (filename__read_str(path, &str, &len) < 0) return NULL; + /* empty identifier file — nothing useful */ + if (len == 0) { + free(str); + return NULL; + } + str[len - 1] = 0; /* remove line feed */ return str; diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c index 64328abeef8b2..f3f9a1573502c 100644 --- a/tools/perf/util/probe-finder.c +++ b/tools/perf/util/probe-finder.c @@ -79,7 +79,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, unsigned int regn; Dwarf_Word offs = 0; bool ref = false; - const char *regs; + const char *regs, *name; int ret, ret2 = 0; if (dwarf_attr(vr_die, DW_AT_external, &attr) != NULL) @@ -93,7 +93,8 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, if (!tvar) return 0; - dwarf_formsdata(&attr, &snum); + if (dwarf_formsdata(&attr, &snum) != 0) + return -ENOENT; ret = asprintf(&tvar->value, "\\%ld", (long)snum); return ret < 0 ? -ENOMEM : 0; @@ -103,8 +104,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, if (dwarf_attr(vr_die, DW_AT_location, &attr) == NULL) return -EINVAL; /* Broken DIE ? */ if (dwarf_getlocation_addr(&attr, addr, &op, &nops, 1) <= 0) { - ret = dwarf_entrypc(sp_die, &tmp); - if (ret) + if (dwarf_entrypc(sp_die, &tmp) != 0) return -ENOENT; if (probe_conf.show_location_range && @@ -115,8 +115,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, return -ENOENT; } - ret = dwarf_highpc(sp_die, &tmp); - if (ret) + if (dwarf_highpc(sp_die, &tmp) != 0) return -ENOENT; /* * This is fuzzed by fentry mcount. We try to find the @@ -138,12 +137,16 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, static_var: if (!tvar) return ret2; + /* Static variables on memory (not stack), make @varname */ - ret = strlen(dwarf_diename(vr_die)); + name = dwarf_diename(vr_die); + if (!name) + return -ENOENT; + ret = strlen(name); tvar->value = zalloc(ret + 2); if (tvar->value == NULL) return -ENOMEM; - snprintf(tvar->value, ret + 2, "@%s", dwarf_diename(vr_die)); + snprintf(tvar->value, ret + 2, "@%s", name); tvar->ref = alloc_trace_arg_ref((long)offs); if (tvar->ref == NULL) return -ENOMEM; @@ -234,13 +237,14 @@ static int convert_variable_type(Dwarf_Die *vr_die, } if (die_get_real_type(vr_die, &type) == NULL) { - pr_warning("Failed to get a type information of %s.\n", - dwarf_diename(vr_die)); + const char *name = dwarf_diename(vr_die); + + pr_warning("Failed to get a type information of %s.\n", name ?: ""); return -ENOENT; } pr_debug("%s type is %s.\n", - dwarf_diename(vr_die), dwarf_diename(&type)); + die_name(vr_die), die_name(&type)); if (cast && (!strcmp(cast, "string") || !strcmp(cast, "ustring"))) { /* String type */ @@ -249,7 +253,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, ret != DW_TAG_array_type) { pr_warning("Failed to cast into string: " "%s(%s) is not a pointer nor array.\n", - dwarf_diename(vr_die), dwarf_diename(&type)); + die_name(vr_die), die_name(&type)); return -EINVAL; } if (die_get_real_type(&type, &type) == NULL) { @@ -272,7 +276,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, !die_compare_name(&type, "unsigned char")) { pr_warning("Failed to cast into string: " "%s is not (unsigned) char *.\n", - dwarf_diename(vr_die)); + die_name(vr_die)); return -EINVAL; } tvar->type = strdup(cast); @@ -299,7 +303,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, /* Check the bitwidth */ if (ret > MAX_BASIC_TYPE_BITS) { pr_info("%s exceeds max-bitwidth. Cut down to %d bits.\n", - dwarf_diename(&type), MAX_BASIC_TYPE_BITS); + die_name(&type), MAX_BASIC_TYPE_BITS); ret = MAX_BASIC_TYPE_BITS; } ret = snprintf(buf, 16, "%c%d", prefix, ret); @@ -333,12 +337,14 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, pr_warning("Failed to get the type of %s.\n", varname); return -ENOENT; } - pr_debug2("Var real type: %s (%x)\n", dwarf_diename(&type), + pr_debug2("Var real type: %s (%x)\n", die_name(&type), (unsigned)dwarf_dieoffset(&type)); tag = dwarf_tag(&type); if (field->name[0] == '[' && (tag == DW_TAG_array_type || tag == DW_TAG_pointer_type)) { + int bsize; + /* Save original type for next field or type */ memcpy(die_mem, &type, sizeof(*die_mem)); /* Get the type of this array */ @@ -346,7 +352,7 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, pr_warning("Failed to get the type of %s.\n", varname); return -ENOENT; } - pr_debug2("Array real type: %s (%x)\n", dwarf_diename(&type), + pr_debug2("Array real type: %s (%x)\n", die_name(&type), (unsigned)dwarf_dieoffset(&type)); if (tag == DW_TAG_pointer_type) { ref = zalloc(sizeof(struct probe_trace_arg_ref)); @@ -357,7 +363,15 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, else *ref_ptr = ref; } - ref->offset += dwarf_bytesize(&type) * field->index; + bsize = dwarf_bytesize(&type); + + if (bsize < 0) + return -EINVAL; + if (!ref) { + pr_warning("Array indexing not supported for variables in registers.\n"); + return -ENOTSUP; + } + ref->offset += bsize * field->index; ref->user_access = user_access; goto next; } else if (tag == DW_TAG_pointer_type) { @@ -414,7 +428,7 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, if (die_find_member(&type, field->name, die_mem) == NULL) { pr_warning("%s(type:%s) has no member %s.\n", varname, - dwarf_diename(&type), field->name); + die_name(&type), field->name); return -EINVAL; } @@ -461,7 +475,7 @@ static int convert_variable(Dwarf_Die *vr_die, struct probe_finder *pf) int ret; pr_debug("Converting variable %s into trace event.\n", - dwarf_diename(vr_die)); + die_name(vr_die)); ret = convert_variable_location(vr_die, pf->addr, pf->fb_ops, &pf->sp_die, pf, pf->tvar); @@ -542,7 +556,7 @@ static int convert_to_trace_point(Dwarf_Die *sp_die, Dwfl_Module *mod, /* Verify the address is correct */ if (!dwarf_haspc(sp_die, paddr)) { pr_warning("Specified offset is out of %s\n", - dwarf_diename(sp_die)); + die_name(sp_die)); return -EINVAL; } @@ -599,7 +613,7 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) if (!die_find_realfunc(&pf->cu_die, pf->addr, &pf->sp_die)) { if (die_find_tailfunc(&pf->cu_die, pf->addr, &pf->sp_die)) { pr_warning("Ignoring tail call from %s\n", - dwarf_diename(&pf->sp_die)); + die_name(&pf->sp_die)); return 0; } else { pr_warning("Failed to find probe point in any " @@ -611,10 +625,16 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) memcpy(&pf->sp_die, sc_die, sizeof(Dwarf_Die)); /* Get the frame base attribute/ops from subprogram */ - dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr); - ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); - if (ret <= 0 || nops == 0) { + if (dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr) == NULL) { pf->fb_ops = NULL; + } else { + ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); + if (ret <= 0 || nops == 0) + pf->fb_ops = NULL; + } + + if (pf->fb_ops == NULL) { + /* Not supported */ } else if (nops == 1 && pf->fb_ops[0].atom == DW_OP_call_frame_cfa && (pf->cfi_eh != NULL || pf->cfi_dbg != NULL)) { if ((dwarf_cfi_addrframe(pf->cfi_eh, pf->addr, &frame) != 0 && @@ -667,8 +687,8 @@ static int find_best_scope_cb(Dwarf_Die *fn_die, void *data) } } else { /* With the line number, find the nearest declared DIE */ - dwarf_decl_line(fn_die, &lno); - if (lno < fsp->line && fsp->diff > fsp->line - lno) { + if (dwarf_decl_line(fn_die, &lno) == 0 && lno < fsp->line && + fsp->diff > fsp->line - lno) { /* Keep a candidate and continue */ fsp->diff = fsp->line - lno; memcpy(fsp->die_mem, fn_die, sizeof(Dwarf_Die)); @@ -924,12 +944,12 @@ static int probe_point_inline_cb(Dwarf_Die *in_die, void *data) /* Get probe address */ if (die_entrypc(in_die, &addr) != 0) { pr_warning("Failed to get entry address of %s.\n", - dwarf_diename(in_die)); + die_name(in_die)); return -ENOENT; } if (addr == 0) { pr_debug("%s has no valid entry address. skipped.\n", - dwarf_diename(in_die)); + die_name(in_die)); return -ENOENT; } pf->addr = addr; @@ -971,12 +991,13 @@ static int probe_point_search_cb(Dwarf_Die *sp_die, void *data) if (pp->file && fname && strtailcmp(pp->file, fname)) return DWARF_CB_OK; - pr_debug("Matched function: %s [%lx]\n", dwarf_diename(sp_die), + pr_debug("Matched function: %s [%lx]\n", die_name(sp_die), (unsigned long)dwarf_dieoffset(sp_die)); pf->fname = fname; pf->abstrace_dieoffset = dwarf_dieoffset(sp_die); if (pp->line) { /* Function relative line */ - dwarf_decl_line(sp_die, &pf->lno); + if (dwarf_decl_line(sp_die, &pf->lno) != 0) + return DWARF_CB_OK; pf->lno += pp->line; param->retval = find_probe_point_by_line(pf); } else if (die_is_func_instance(sp_die)) { @@ -985,7 +1006,7 @@ static int probe_point_search_cb(Dwarf_Die *sp_die, void *data) /* But in some case the entry address is 0 */ if (pf->addr == 0) { pr_debug("%s has no entry PC. Skipped\n", - dwarf_diename(sp_die)); + die_name(sp_die)); param->retval = 0; /* Real function */ } else if (pp->lazy_line) @@ -1018,7 +1039,8 @@ static int find_probe_point_by_func(struct probe_finder *pf) { struct dwarf_callback_param _param = {.data = (void *)pf, .retval = 0}; - dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0); + if (dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0) < 0) + pr_debug("Failed to get functions from CU\n"); return _param.retval; } @@ -1207,7 +1229,8 @@ static int copy_variables_cb(Dwarf_Die *die_mem, void *data) * points to correct die. */ if (dwarf_attr(die_mem, DW_AT_abstract_origin, &attr)) { - dwarf_formref_die(&attr, &var_die); + if (dwarf_formref_die(&attr, &var_die) == NULL) + goto out; if (pf->abstrace_dieoffset != dwarf_dieoffset(&var_die)) goto out; } @@ -1293,13 +1316,16 @@ static int add_probe_trace_event(Dwarf_Die *sc_die, struct probe_finder *pf) if (ret < 0) goto end; - tev->point.realname = strdup(dwarf_diename(sc_die)); + tev->point.realname = strdup(die_name(sc_die)); if (!tev->point.realname) { ret = -ENOMEM; goto end; } - tev->lang = dwarf_srclang(dwarf_diecu(sc_die, &pf->cu_die, NULL, NULL)); + if (dwarf_diecu(sc_die, &pf->cu_die, NULL, NULL) != NULL) + tev->lang = dwarf_srclang(&pf->cu_die); + else + tev->lang = DW_LANG_C; /* Fallback */ pr_debug("Probe point found: %s+%lu\n", tev->point.symbol, tev->point.offset); @@ -1794,7 +1820,8 @@ static int line_range_search_cb(Dwarf_Die *sp_die, void *data) if (die_match_name(sp_die, lr->function) && die_is_func_def(sp_die)) { lf->fname = die_get_decl_file(sp_die); - dwarf_decl_line(sp_die, &lr->offset); + if (dwarf_decl_line(sp_die, &lr->offset) != 0) + lr->offset = 0; /* Fallback if no line info */ pr_debug("fname: %s, lineno:%d\n", lf->fname, lr->offset); lf->lno_s = lr->offset + lr->start; if (lf->lno_s < 0) /* Overflow */ @@ -1818,7 +1845,8 @@ static int line_range_search_cb(Dwarf_Die *sp_die, void *data) static int find_line_range_by_func(struct line_finder *lf) { struct dwarf_callback_param param = {.data = (void *)lf, .retval = 0}; - dwarf_getfuncs(&lf->cu_die, line_range_search_cb, ¶m, 0); + if (dwarf_getfuncs(&lf->cu_die, line_range_search_cb, ¶m, 0) < 0) + pr_debug("Failed to get functions from CU\n"); return param.retval; } diff --git a/tools/perf/util/sample.h b/tools/perf/util/sample.h index e556c9b656ea9..c4eae8b2fd060 100644 --- a/tools/perf/util/sample.h +++ b/tools/perf/util/sample.h @@ -158,6 +158,8 @@ struct perf_sample { u64 code_page_size; /** @cgroup: The sample event PERF_SAMPLE_CGROUP value. */ u64 cgroup; + /** @file_offset: Byte offset of this event in the perf.data file. */ + u64 file_offset; /** @flags: Extra flag data from auxiliary events like intel-pt. */ u32 flags; /** @machine_pid: The guest machine pid derived from the sample id. */ diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index fe0de2a0277f0..2bc093bd2f241 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -1352,6 +1352,7 @@ static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool struct deferred_event { struct list_head list; union perf_event *event; + u64 file_offset; }; /* @@ -1387,6 +1388,7 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, perf_sample__exit(&orig_sample); break; } + orig_sample.file_offset = de->file_offset; if (sample->tid != orig_sample.tid) { perf_sample__exit(&orig_sample); @@ -1435,6 +1437,7 @@ static int session__flush_deferred_samples(struct perf_session *session, perf_sample__exit(&sample); break; } + sample.file_offset = de->file_offset; sample.evsel = evlist__id2evsel(evlist, sample.id); ret = evlist__deliver_sample(evlist, tool, de->event, @@ -1496,6 +1499,7 @@ static int machines__deliver_event(struct machines *machines, return -ENOMEM; } memcpy(de->event, event, sz); + de->file_offset = sample->file_offset; list_add_tail(&de->list, &evlist->deferred_samples); return 0; } @@ -1589,6 +1593,7 @@ static int perf_session__deliver_event(struct perf_session *session, ret = 0; goto out; } + sample.file_offset = file_offset; ret = machines__deliver_event(&session->machines, session->evlist, event, &sample, tool, file_offset, file_path); diff --git a/tools/perf/util/srcline.c b/tools/perf/util/srcline.c index db164d2581634..62884428fb5a0 100644 --- a/tools/perf/util/srcline.c +++ b/tools/perf/util/srcline.c @@ -429,10 +429,13 @@ struct inline_node *dso__parse_addr_inlines(struct dso *dso, u64 addr, return addr2inlines(dso_name, addr, dso, sym); } -void inline_node__delete(struct inline_node *node) +void inline_node__clear_frames(struct inline_node *node) { struct inline_list *ilist, *tmp; + if (node == NULL) + return; + list_for_each_entry_safe(ilist, tmp, &node->val, list) { list_del_init(&ilist->list); zfree_srcline(&ilist->srcline); @@ -441,7 +444,11 @@ void inline_node__delete(struct inline_node *node) symbol__delete(ilist->symbol); free(ilist); } +} +void inline_node__delete(struct inline_node *node) +{ + inline_node__clear_frames(node); free(node); } diff --git a/tools/perf/util/srcline.h b/tools/perf/util/srcline.h index 7c37b3bf9ce79..1018cbc886d62 100644 --- a/tools/perf/util/srcline.h +++ b/tools/perf/util/srcline.h @@ -47,6 +47,7 @@ struct inline_node *dso__parse_addr_inlines(struct dso *dso, u64 addr, struct symbol *sym); /* free resources associated to the inline node list */ void inline_node__delete(struct inline_node *node); +void inline_node__clear_frames(struct inline_node *node); /* insert the inline node list into the DSO, which will take ownership */ void inlines__tree_insert(struct rb_root_cached *tree, diff --git a/tools/perf/util/svghelper.c b/tools/perf/util/svghelper.c index e360e7736c7ba..826bd2577344b 100644 --- a/tools/perf/util/svghelper.c +++ b/tools/perf/util/svghelper.c @@ -47,13 +47,13 @@ static double cpu2slot(int cpu) } static int *topology_map; +static int topology_map_size; static double cpu2y(int cpu) { - if (topology_map) + if (topology_map && cpu >= 0 && cpu < topology_map_size) return cpu2slot(topology_map[cpu]) * SLOT_MULT; - else - return cpu2slot(cpu) * SLOT_MULT; + return cpu2slot(cpu) * SLOT_MULT; } static double time2pixels(u64 __time) @@ -736,7 +736,8 @@ static int str_to_bitmap(char *s, cpumask_t *b, int nr_cpus) return -1; perf_cpu_map__for_each_cpu(cpu, idx, map) { - if (cpu.cpu >= nr_cpus) { + /* perf_cpu_map__new("") returns cpu.cpu == -1 */ + if (cpu.cpu < 0 || cpu.cpu >= nr_cpus) { ret = -1; break; } @@ -794,6 +795,7 @@ int svg_build_topology_map(struct perf_env *env) fprintf(stderr, "topology: no memory\n"); goto exit; } + topology_map_size = nr_cpus; for (i = 0; i < nr_cpus; i++) topology_map[i] = -1; diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 7afa8a1171396..2007f8f003e3b 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -217,7 +217,7 @@ bool filename__has_section(const char *filename, const char *sec) GElf_Shdr shdr; bool found = false; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) return false; @@ -834,10 +834,24 @@ static int elf_read_build_id(Elf *elf, void *bf, size_t size) ptr = data->d_buf; while (ptr < (data->d_buf + data->d_size)) { GElf_Nhdr *nhdr = ptr; - size_t namesz = NOTE_ALIGN(nhdr->n_namesz), - descsz = NOTE_ALIGN(nhdr->n_descsz); + size_t namesz, descsz, remaining; const char *name; + /* ensure the note header fits within the section */ + if (ptr + sizeof(*nhdr) > data->d_buf + data->d_size) + break; + + namesz = NOTE_ALIGN(nhdr->n_namesz); + descsz = NOTE_ALIGN(nhdr->n_descsz); + + /* validate individually to avoid size_t overflow on 32-bit */ + remaining = data->d_buf + data->d_size - ptr - sizeof(*nhdr); + if (namesz > remaining || descsz > remaining - namesz) { + pr_warning("%s: oversized note: n_namesz=%u, n_descsz=%u\n", + __func__, nhdr->n_namesz, nhdr->n_descsz); + break; + } + ptr += sizeof(*nhdr); name = ptr; ptr += namesz; @@ -871,7 +885,7 @@ static int read_build_id(const char *filename, struct build_id *bid) if (size < BUILD_ID_SIZE) goto out; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -919,12 +933,14 @@ int filename__read_build_id(const char *filename, struct build_id *bid) return -1; } close(fd); - filename = path; + /* non-empty path means a temp file was created */ + if (path[0] != '\0') + filename = path; } err = read_build_id(filename, bid); - if (m.comp) + if (m.comp && filename == path) unlink(filename); return err; } @@ -934,7 +950,7 @@ int sysfs__read_build_id(const char *filename, struct build_id *bid) size_t size = sizeof(bid->data); int fd, err = -1; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -960,17 +976,28 @@ int sysfs__read_build_id(const char *filename, struct build_id *bid) err = 0; break; } - } else if (read(fd, bf, descsz) != (ssize_t)descsz) - break; + } else { + /* descsz from untrusted file — clamp to buffer */ + if (descsz > sizeof(bf)) + break; + if (read(fd, bf, descsz) != (ssize_t)descsz) + break; + } } else { - int n = namesz + descsz; + size_t n; - if (n > (int)sizeof(bf)) { + /* int sum of namesz+descsz can overflow negative, bypassing size check */ + if (namesz > sizeof(bf) || descsz > sizeof(bf) - namesz) { n = sizeof(bf); pr_debug("%s: truncating reading of build id in sysfs file %s: n_namesz=%u, n_descsz=%u.\n", __func__, filename, nhdr.n_namesz, nhdr.n_descsz); + } else { + n = namesz + descsz; } - if (read(fd, bf, n) != n) + /* no valid note has both namesz and descsz zero */ + if (n == 0) + break; + if (read(fd, bf, n) != (ssize_t)n) break; } } @@ -994,7 +1021,7 @@ int filename__read_debuglink(const char *filename, char *debuglink, if (err >= 0) goto out; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -1023,7 +1050,14 @@ int filename__read_debuglink(const char *filename, char *debuglink, goto out_elf_end; /* the start of this section is a zero-terminated string */ - strncpy(debuglink, data->d_buf, size); + if (data->d_size > 0) { + size_t len = min(size - 1, data->d_size); + + memcpy(debuglink, data->d_buf, len); + debuglink[len] = '\0'; + } else { + debuglink[0] = '\0'; + } err = 0; @@ -1108,9 +1142,9 @@ static Elf *read_gnu_debugdata(struct dso *dso, Elf *elf, const char *name, int return NULL; } - temp_fd = mkstemp(temp_filename); + temp_fd = mkostemp(temp_filename, O_CLOEXEC); if (temp_fd < 0) { - pr_debug("%s: mkstemp: %m\n", __func__); + pr_debug("%s: mkostemp: %m\n", __func__); *dso__load_errno(dso) = -errno; fclose(wrapped); return NULL; @@ -1152,7 +1186,7 @@ int symsrc__init(struct symsrc *ss, struct dso *dso, const char *name, type = dso__symtab_type(dso); } else { - fd = open(name, O_RDONLY); + fd = open(name, O_RDONLY | O_CLOEXEC); if (fd < 0) { *dso__load_errno(dso) = errno; return -1; @@ -1341,6 +1375,24 @@ static u64 ref_reloc(struct kmap *kmap) void __weak arch__sym_update(struct symbol *s __maybe_unused, GElf_Sym *sym __maybe_unused) { } +struct remap_kernel_ctx { + u64 sh_addr; + u64 sh_size; + u64 sh_offset; + struct kmap *kmap; +}; + +static int remap_kernel_cb(struct map *map, void *data) +{ + struct remap_kernel_ctx *ctx = data; + + map__set_start(map, ctx->sh_addr + ref_reloc(ctx->kmap)); + map__set_end(map, map__start(map) + ctx->sh_size); + map__set_pgoff(map, ctx->sh_offset); + map__set_mapping_type(map, MAPPING_TYPE__DSO); + return 0; +} + static int dso__process_kernel_symbol(struct dso *dso, struct map *map, GElf_Sym *sym, GElf_Shdr *shdr, struct maps *kmaps, struct kmap *kmap, @@ -1371,22 +1423,15 @@ static int dso__process_kernel_symbol(struct dso *dso, struct map *map, * map to the kernel dso. */ if (*remap_kernel && dso__kernel(dso) && !kmodule) { + struct remap_kernel_ctx ctx = { + .sh_addr = shdr->sh_addr, + .sh_size = shdr->sh_size, + .sh_offset = shdr->sh_offset, + .kmap = kmap + }; + *remap_kernel = false; - map__set_start(map, shdr->sh_addr + ref_reloc(kmap)); - map__set_end(map, map__start(map) + shdr->sh_size); - map__set_pgoff(map, shdr->sh_offset); - map__set_mapping_type(map, MAPPING_TYPE__DSO); - /* Ensure maps are correctly ordered */ - if (kmaps) { - int err; - struct map *tmp = map__get(map); - - maps__remove(kmaps, map); - err = maps__insert(kmaps, map); - map__put(tmp); - if (err) - return err; - } + maps__mutate_mapping(kmaps, map, remap_kernel_cb, &ctx); } /* @@ -1945,7 +1990,7 @@ static int kcore__open(struct kcore *kcore, const char *filename) { GElf_Ehdr *ehdr; - kcore->fd = open(filename, O_RDONLY); + kcore->fd = open(filename, O_RDONLY | O_CLOEXEC); if (kcore->fd == -1) return -1; @@ -1976,9 +2021,9 @@ static int kcore__init(struct kcore *kcore, char *filename, int elfclass, kcore->elfclass = elfclass; if (temp) - kcore->fd = mkstemp(filename); + kcore->fd = mkostemp(filename, O_CLOEXEC); else - kcore->fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0400); + kcore->fd = open(filename, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0400); if (kcore->fd == -1) return -1; @@ -2454,11 +2499,11 @@ static int kcore_copy__compare_files(const char *from_filename, { int from, to, err = -1; - from = open(from_filename, O_RDONLY); + from = open(from_filename, O_RDONLY | O_CLOEXEC); if (from < 0) return -1; - to = open(to_filename, O_RDONLY); + to = open(to_filename, O_RDONLY | O_CLOEXEC); if (to < 0) goto out_close_from; @@ -2876,7 +2921,7 @@ int get_sdt_note_list(struct list_head *head, const char *target) Elf *elf; int fd, ret; - fd = open(target, O_RDONLY); + fd = open(target, O_RDONLY | O_CLOEXEC); if (fd < 0) return -EBADF; diff --git a/tools/perf/util/symbol-minimal.c b/tools/perf/util/symbol-minimal.c index 8221dc9868f7c..0a71d14639527 100644 --- a/tools/perf/util/symbol-minimal.c +++ b/tools/perf/util/symbol-minimal.c @@ -1,3 +1,4 @@ +#include "debug.h" #include "dso.h" #include "symbol.h" #include "symsrc.h" @@ -44,7 +45,7 @@ static int read_build_id(void *note_data, size_t note_len, struct build_id *bid, ptr = note_data; while ((ptr + sizeof(*nhdr)) < (note_data + note_len)) { const char *name; - size_t namesz, descsz; + size_t namesz, descsz, remaining; nhdr = ptr; if (need_swap) { @@ -56,6 +57,14 @@ static int read_build_id(void *note_data, size_t note_len, struct build_id *bid, namesz = NOTE_ALIGN(nhdr->n_namesz); descsz = NOTE_ALIGN(nhdr->n_descsz); + /* validate individually to avoid size_t overflow on 32-bit */ + remaining = note_data + note_len - ptr - sizeof(*nhdr); + if (namesz > remaining || descsz > remaining - namesz) { + pr_warning("%s: oversized note: n_namesz=%u, n_descsz=%u\n", + __func__, nhdr->n_namesz, nhdr->n_descsz); + break; + } + ptr += sizeof(*nhdr); name = ptr; ptr += namesz; @@ -166,7 +175,7 @@ int filename__read_build_id(const char *filename, struct build_id *bid) if (elf32) { hdrs.phdr32[i].p_type = bswap_32(hdrs.phdr32[i].p_type); hdrs.phdr32[i].p_offset = bswap_32(hdrs.phdr32[i].p_offset); - hdrs.phdr32[i].p_filesz = bswap_32(hdrs.phdr32[i].p_offset); + hdrs.phdr32[i].p_filesz = bswap_32(hdrs.phdr32[i].p_filesz); } else { hdrs.phdr64[i].p_type = bswap_32(hdrs.phdr64[i].p_type); hdrs.phdr64[i].p_offset = bswap_64(hdrs.phdr64[i].p_offset); @@ -177,6 +186,9 @@ int filename__read_build_id(const char *filename, struct build_id *bid) continue; p_filesz = elf32 ? hdrs.phdr32[i].p_filesz : hdrs.phdr64[i].p_filesz; + /* ssize_t can go negative with crafted ELF p_filesz values */ + if (p_filesz <= 0) + continue; if (p_filesz > buf_size) { void *tmp; diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index fcaeeddbbb6b9..85bdb3cbe0b82 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -48,6 +48,13 @@ #include #include +static int map_fixup_cb(struct map *map, void *data __maybe_unused) +{ + map__fixup_start(map); + map__fixup_end(map); + return 0; +} + static int dso__load_kernel_sym(struct dso *dso, struct map *map); static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map); static bool symbol__is_idle(const char *name); @@ -2121,10 +2128,11 @@ static int dso__load_kernel_sym(struct dso *dso, struct map *map) free(kallsyms_allocated_filename); if (err > 0 && !dso__is_kcore(dso)) { + struct maps *kmaps = map__kmaps(map); + dso__set_binary_type(dso, DSO_BINARY_TYPE__KALLSYMS); dso__set_long_name(dso, DSO__NAME_KALLSYMS, false); - map__fixup_start(map); - map__fixup_end(map); + maps__mutate_mapping(kmaps, map, map_fixup_cb, NULL); } return err; @@ -2156,7 +2164,7 @@ static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map) if (!kallsyms_filename) return -1; } else { - sprintf(path, "%s/proc/kallsyms", machine->root_dir); + snprintf(path, sizeof(path), "%s/proc/kallsyms", machine->root_dir); kallsyms_filename = path; } @@ -2164,10 +2172,11 @@ static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map) if (err > 0) pr_debug("Using %s for symbols\n", kallsyms_filename); if (err > 0 && !dso__is_kcore(dso)) { + struct maps *kmaps = map__kmaps(map); + dso__set_binary_type(dso, DSO_BINARY_TYPE__GUEST_KALLSYMS); dso__set_long_name(dso, machine->mmap_name, false); - map__fixup_start(map); - map__fixup_end(map); + maps__mutate_mapping(kmaps, map, map_fixup_cb, NULL); } return err; diff --git a/tools/perf/util/symbol_conf.h b/tools/perf/util/symbol_conf.h index 6cd454d7c98e6..0dee5aa6a5340 100644 --- a/tools/perf/util/symbol_conf.h +++ b/tools/perf/util/symbol_conf.h @@ -9,6 +9,15 @@ struct strlist; struct intlist; +enum unwind_style { + UNWIND_STYLE_UNKNOWN = 0, + UNWIND_STYLE_LIBDW, + UNWIND_STYLE_LIBUNWIND, +}; + +#define MAX_UNWIND_STYLE (UNWIND_STYLE_LIBUNWIND + 1) + + enum a2l_style { A2L_STYLE_UNKNOWN = 0, A2L_STYLE_LIBDW, @@ -81,6 +90,7 @@ struct symbol_conf { const char *addr2line_path; enum a2l_style addr2line_style[MAX_A2L_STYLE]; int addr2line_timeout_ms; + enum unwind_style unwind_style[MAX_UNWIND_STYLE]; unsigned long time_quantum; struct strlist *dso_list, *comm_list, diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index 85bee747f4cd2..2461f25a4d7dc 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -1455,7 +1455,8 @@ int perf_event__synthesize_stat_round(const struct perf_tool *tool, return process(tool, (union perf_event *) &event, NULL, machine); } -size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format) +size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format, + u64 branch_sample_type) { size_t sz, result = sizeof(struct perf_record_sample); @@ -1515,8 +1516,10 @@ size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, if (type & PERF_SAMPLE_BRANCH_STACK) { sz = sample->branch_stack->nr * sizeof(struct branch_entry); - /* nr, hw_idx */ - sz += 2 * sizeof(u64); + /* nr */ + sz += sizeof(u64); + if (branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) + sz += sizeof(u64); result += sz; } @@ -1605,7 +1608,7 @@ static __u64 *copy_read_group_values(__u64 *array, __u64 read_format, } int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, - const struct perf_sample *sample) + u64 branch_sample_type, const struct perf_sample *sample) { __u64 *array; size_t sz; @@ -1719,9 +1722,17 @@ int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_fo if (type & PERF_SAMPLE_BRANCH_STACK) { sz = sample->branch_stack->nr * sizeof(struct branch_entry); - /* nr, hw_idx */ - sz += 2 * sizeof(u64); - memcpy(array, sample->branch_stack, sz); + + *array++ = sample->branch_stack->nr; + + if (branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) { + if (sample->no_hw_idx) + *array++ = 0; + else + *array++ = sample->branch_stack->hw_idx; + } + + memcpy(array, perf_sample__branch_entries((struct perf_sample *)sample), sz); array = (void *)array + sz; } diff --git a/tools/perf/util/synthetic-events.h b/tools/perf/util/synthetic-events.h index b0edad0c31001..8c7f49f9ccf54 100644 --- a/tools/perf/util/synthetic-events.h +++ b/tools/perf/util/synthetic-events.h @@ -81,7 +81,8 @@ int perf_event__synthesize_mmap_events(const struct perf_tool *tool, union perf_ int perf_event__synthesize_modules(const struct perf_tool *tool, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_namespaces(const struct perf_tool *tool, union perf_event *event, pid_t pid, pid_t tgid, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_cgroups(const struct perf_tool *tool, perf_event__handler_t process, struct machine *machine); -int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, const struct perf_sample *sample); +int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, + u64 branch_sample_type, const struct perf_sample *sample); int perf_event__synthesize_stat_config(const struct perf_tool *tool, struct perf_stat_config *config, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_stat_events(struct perf_stat_config *config, const struct perf_tool *tool, struct evlist *evlist, perf_event__handler_t process, bool attrs); int perf_event__synthesize_stat_round(const struct perf_tool *tool, u64 time, u64 type, perf_event__handler_t process, struct machine *machine); @@ -97,7 +98,8 @@ void perf_event__synthesize_final_bpf_metadata(struct perf_session *session, int perf_tool__process_synth_event(const struct perf_tool *tool, union perf_event *event, struct machine *machine, perf_event__handler_t process); -size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format); +size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, + u64 read_format, u64 branch_sample_type); int __machine__synthesize_threads(struct machine *machine, const struct perf_tool *tool, struct target *target, struct perf_thread_map *threads, diff --git a/tools/perf/util/thread.c b/tools/perf/util/thread.c index 22be77225bb0c..2faa258f9c15f 100644 --- a/tools/perf/util/thread.c +++ b/tools/perf/util/thread.c @@ -294,6 +294,11 @@ int thread__set_comm_from_proc(struct thread *thread) if (!(snprintf(path, sizeof(path), "%d/task/%d/comm", thread__pid(thread), thread__tid(thread)) >= (int)sizeof(path)) && procfs__read_str(path, &comm, &sz) == 0) { + /* sz==0: read got nothing, e.g. race during exit teardown */ + if (sz == 0) { + free(comm); + return -1; + } comm[sz - 1] = '\0'; err = thread__set_comm(thread, comm, 0); } diff --git a/tools/perf/util/tool.c b/tools/perf/util/tool.c index 013c7839e2cfd..ff2150517b755 100644 --- a/tools/perf/util/tool.c +++ b/tools/perf/util/tool.c @@ -285,6 +285,7 @@ void perf_tool__init(struct perf_tool *tool, bool ordered_events) tool->no_warn = false; tool->show_feat_hdr = SHOW_FEAT_NO_HEADER; tool->merge_deferred_callchains = true; + tool->dont_split_sample_group = false; tool->sample = process_event_sample_stub; tool->mmap = process_event_stub; @@ -433,6 +434,8 @@ CREATE_DELEGATE_OP2(stat_config); CREATE_DELEGATE_OP2(stat_round); CREATE_DELEGATE_OP2(thread_map); CREATE_DELEGATE_OP2(time_conv); +CREATE_DELEGATE_OP2(schedstat_cpu); +CREATE_DELEGATE_OP2(schedstat_domain); CREATE_DELEGATE_OP2(tracing_data); #define CREATE_DELEGATE_OP3(name) \ @@ -470,6 +473,7 @@ void delegate_tool__init(struct delegate_tool *tool, struct perf_tool *delegate) tool->tool.no_warn = delegate->no_warn; tool->tool.show_feat_hdr = delegate->show_feat_hdr; tool->tool.merge_deferred_callchains = delegate->merge_deferred_callchains; + tool->tool.dont_split_sample_group = delegate->dont_split_sample_group; tool->tool.sample = delegate_sample; tool->tool.read = delegate_read; @@ -516,4 +520,6 @@ void delegate_tool__init(struct delegate_tool *tool, struct perf_tool *delegate) tool->tool.bpf_metadata = delegate_bpf_metadata; tool->tool.compressed = delegate_compressed; tool->tool.auxtrace = delegate_auxtrace; + tool->tool.schedstat_cpu = delegate_schedstat_cpu; + tool->tool.schedstat_domain = delegate_schedstat_domain; } diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index 05e8e68bd49c4..7f35042be5677 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -339,7 +339,7 @@ frame_callback(Dwfl_Frame *state, void *arg) DWARF_CB_ABORT : DWARF_CB_OK; } -int unwind__get_entries(unwind_entry_cb_t cb, void *arg, +int libdw__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, struct perf_sample *data, int max_stack, @@ -353,10 +353,10 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, static struct unwind_info *ui; Dwfl *dwfl; Dwarf_Word ip; - int err = -EINVAL, i; + int err = -EINVAL, i, entries; if (!data->user_regs || !data->user_regs->regs) - return -EINVAL; + return 0; ui = zalloc(sizeof(*ui) + sizeof(ui->entries[0]) * max_stack); if (!ui) @@ -430,6 +430,18 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, map_symbol__exit(&ui->entries[i].ms); dwfl_ui_ti->ui = NULL; + entries = (int)ui->idx; free(ui); - return 0; + /* + * Unwinder return contract: + * > 0 : unwinding succeeded (stops fallback). If we found frames but hit an error + * (e.g. truncated stack), report success to preserve existing frames. + * 0 : unwinding failed without yielding frames. Ignore non-fatal errors + * (e.g. missing debug info, DWARF corruption) to allow fallback unwinder or + * kernel callchain resolution to proceed. + * < 0 : fatal error (e.g. -ENOMEM). Aborts unwinding entirely. + */ + if (err) + return (err == -ENOMEM) ? -ENOMEM : (entries > 0 ? 1 : 0); + return entries; } diff --git a/tools/perf/util/unwind-libunwind-local.c b/tools/perf/util/unwind-libunwind-local.c index 87d496e9dfa66..27e2f7b317895 100644 --- a/tools/perf/util/unwind-libunwind-local.c +++ b/tools/perf/util/unwind-libunwind-local.c @@ -744,7 +744,7 @@ static int get_entries(struct unwind_info *ui, unwind_entry_cb_t cb, ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), perf_arch_reg_ip(e_machine)); if (ret) - return ret; + return 0; ips[i++] = (unw_word_t) val; @@ -757,7 +757,7 @@ static int get_entries(struct unwind_info *ui, unwind_entry_cb_t cb, addr_space = maps__addr_space(thread__maps(ui->thread)); if (addr_space == NULL) - return -1; + return 0; ret = unw_init_remote(&c, addr_space, ui); if (ret && !ui->best_effort) @@ -785,15 +785,30 @@ static int get_entries(struct unwind_info *ui, unwind_entry_cb_t cb, /* * Display what we got based on the order setup. */ + int entries = 0; for (i = 0; i < max_stack && !ret; i++) { int j = i; if (callchain_param.order == ORDER_CALLER) j = max_stack - i - 1; - ret = ips[j] ? entry(ips[j], ui->thread, cb, arg) : 0; + if (ips[j]) { + ret = entry(ips[j], ui->thread, cb, arg); + if (ret) + break; + entries++; + } } - return ret; + /* + * Unwinder return contract: + * > 0 : unwinding succeeded (stops fallback). + * 0 : unwinding failed without yielding frames. Ignore non-fatal errors + * (e.g. stepping failure) to allow fallback unwinder or kernel callchains. + * < 0 : fatal error (e.g. -ENOMEM). Aborts unwinding entirely. + */ + if (ret == -ENOMEM) + return -ENOMEM; + return (entries > 0 || ret == 0) ? entries : 0; } static int _unwind__get_entries(unwind_entry_cb_t cb, void *arg, @@ -809,10 +824,10 @@ static int _unwind__get_entries(unwind_entry_cb_t cb, void *arg, }; if (!data->user_regs || !data->user_regs->regs) - return -EINVAL; + return 0; if (max_stack <= 0) - return -EINVAL; + return 0; return get_entries(&ui, cb, arg, max_stack); } diff --git a/tools/perf/util/unwind-libunwind.c b/tools/perf/util/unwind-libunwind.c index cb8be6acfb6f5..a0016b897dae2 100644 --- a/tools/perf/util/unwind-libunwind.c +++ b/tools/perf/util/unwind-libunwind.c @@ -79,7 +79,7 @@ void unwind__finish_access(struct maps *maps) ops->finish_access(maps); } -int unwind__get_entries(unwind_entry_cb_t cb, void *arg, +int libunwind__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, struct perf_sample *data, int max_stack, bool best_effort) diff --git a/tools/perf/util/unwind.c b/tools/perf/util/unwind.c new file mode 100644 index 0000000000000..4ed4b1d55c69f --- /dev/null +++ b/tools/perf/util/unwind.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "debug.h" +#include "symbol_conf.h" +#include "unwind.h" +#include +#include +#include + +int unwind__get_entries(unwind_entry_cb_t cb __maybe_unused, void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) +{ + int ret = 0; + +#if defined(HAVE_LIBDW_SUPPORT) || defined(HAVE_LIBUNWIND_SUPPORT) + if (symbol_conf.unwind_style[0] == UNWIND_STYLE_UNKNOWN) { + int i = 0; +#ifdef HAVE_LIBDW_SUPPORT + symbol_conf.unwind_style[i++] = UNWIND_STYLE_LIBDW; +#endif +#ifdef HAVE_LIBUNWIND_SUPPORT + symbol_conf.unwind_style[i++] = UNWIND_STYLE_LIBUNWIND; +#endif + } +#endif //defined(HAVE_LIBDW_SUPPORT) || defined(HAVE_LIBUNWIND_SUPPORT) + + for (size_t i = 0; i < ARRAY_SIZE(symbol_conf.unwind_style); i++) { + switch (symbol_conf.unwind_style[i]) { + case UNWIND_STYLE_LIBDW: + ret = libdw__get_entries(cb, arg, thread, data, max_stack, best_effort); + break; + case UNWIND_STYLE_LIBUNWIND: + ret = libunwind__get_entries(cb, arg, thread, data, max_stack, best_effort); + break; + case UNWIND_STYLE_UNKNOWN: + default: +#if !defined(HAVE_LIBDW_SUPPORT) && !defined(HAVE_LIBUNWIND_SUPPORT) + pr_warning_once( + "Error: dwarf unwinding not supported, build perf with libdw or libunwind.\n"); +#endif + ret = 0; + break; + } + if (ret > 0) { + ret = 0; + break; + } + if (ret < 0) + break; + } + return ret; +} + +int unwind__configure(const char *var, const char *value, void *cb __maybe_unused) +{ + static const char * const unwind_style_names[] = { + [UNWIND_STYLE_LIBDW] = "libdw", + [UNWIND_STYLE_LIBUNWIND] = "libunwind", + NULL + }; + char *s, *p, *saveptr; + size_t i = 0; + + if (strcmp(var, "unwind.style")) + return 0; + + if (!value) + return -1; + + s = strdup(value); + if (!s) + return -1; + + memset(symbol_conf.unwind_style, 0, sizeof(symbol_conf.unwind_style)); + + p = strtok_r(s, ",", &saveptr); + while (p && i < ARRAY_SIZE(symbol_conf.unwind_style)) { + bool found = false; + char *q = strim(p); + + for (size_t j = UNWIND_STYLE_LIBDW; j < MAX_UNWIND_STYLE; j++) { + if (!strcasecmp(q, unwind_style_names[j])) { + symbol_conf.unwind_style[i++] = j; + found = true; + break; + } + } + if (!found) + pr_warning("Unknown unwind style: %s\n", q); + p = strtok_r(NULL, ",", &saveptr); + } + + free(s); + return 0; +} + +int unwind__option(const struct option *opt __maybe_unused, + const char *arg, + int unset __maybe_unused) +{ + return unwind__configure("unwind.style", arg, NULL); +} diff --git a/tools/perf/util/unwind.h b/tools/perf/util/unwind.h index 9f7164c6d9aa9..69ba08afda792 100644 --- a/tools/perf/util/unwind.h +++ b/tools/perf/util/unwind.h @@ -4,9 +4,10 @@ #include #include -#include "util/map_symbol.h" +#include "map_symbol.h" struct maps; +struct option; struct perf_sample; struct thread; @@ -26,7 +27,9 @@ struct unwind_libunwind_ops { struct perf_sample *data, int max_stack, bool best_effort); }; -#ifdef HAVE_DWARF_UNWIND_SUPPORT +int unwind__configure(const char *var, const char *value, void *cb); +int unwind__option(const struct option *opt, const char *arg, int unset); + /* * When best_effort is set, don't report errors and fail silently. This could * be expanded in the future to be more permissive about things other than @@ -36,8 +39,31 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, struct perf_sample *data, int max_stack, bool best_effort); -/* libunwind specific */ + +#ifdef HAVE_LIBDW_SUPPORT +int libdw__get_entries(unwind_entry_cb_t cb, void *arg, + struct thread *thread, + struct perf_sample *data, int max_stack, + bool best_effort); +#else +#include "debug.h" +static inline int libdw__get_entries(unwind_entry_cb_t cb __maybe_unused, void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) +{ + pr_warning_once("Error: libdw dwarf unwinding not built into perf\n"); + return 0; +} +#endif + #ifdef HAVE_LIBUNWIND_SUPPORT +/* libunwind specific */ +int libunwind__get_entries(unwind_entry_cb_t cb, void *arg, + struct thread *thread, + struct perf_sample *data, int max_stack, + bool best_effort); #ifndef LIBUNWIND__ARCH_REG_ID #define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__arch_reg_id(regnum) #endif @@ -47,25 +73,15 @@ int unwind__prepare_access(struct maps *maps, struct map *map, bool *initialized void unwind__flush_access(struct maps *maps); void unwind__finish_access(struct maps *maps); #else -static inline int unwind__prepare_access(struct maps *maps __maybe_unused, - struct map *map __maybe_unused, - bool *initialized __maybe_unused) -{ - return 0; -} - -static inline void unwind__flush_access(struct maps *maps __maybe_unused) {} -static inline void unwind__finish_access(struct maps *maps __maybe_unused) {} -#endif -#else -static inline int -unwind__get_entries(unwind_entry_cb_t cb __maybe_unused, - void *arg __maybe_unused, - struct thread *thread __maybe_unused, - struct perf_sample *data __maybe_unused, - int max_stack __maybe_unused, - bool best_effort __maybe_unused) +#include "debug.h" +static inline int libunwind__get_entries(unwind_entry_cb_t cb __maybe_unused, + void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) { + pr_warning_once("Error: libunwind dwarf unwinding not built into perf\n"); return 0; } @@ -78,5 +94,6 @@ static inline int unwind__prepare_access(struct maps *maps __maybe_unused, static inline void unwind__flush_access(struct maps *maps __maybe_unused) {} static inline void unwind__finish_access(struct maps *maps __maybe_unused) {} -#endif /* HAVE_DWARF_UNWIND_SUPPORT */ +#endif + #endif /* __UNWIND_H */ diff --git a/tools/power/x86/intel-speed-select/isst-daemon.c b/tools/power/x86/intel-speed-select/isst-daemon.c index 66df21b2b5737..acedb74328494 100644 --- a/tools/power/x86/intel-speed-select/isst-daemon.c +++ b/tools/power/x86/intel-speed-select/isst-daemon.c @@ -148,6 +148,7 @@ static void daemonize(char *rundir, char *pidfile) { int pid, sid, i; char str[10]; + struct stat st; struct sigaction sig_actions; sigset_t sig_set; int ret; @@ -200,11 +201,17 @@ static void daemonize(char *rundir, char *pidfile) if (ret == -1) exit(EXIT_FAILURE); - pid_file_handle = open(pidfile, O_RDWR | O_CREAT, 0600); + pid_file_handle = open(pidfile, O_RDWR | O_CREAT | O_NOFOLLOW, 0600); if (pid_file_handle == -1) { /* Couldn't open lock file */ exit(1); } + + if (fstat(pid_file_handle, &st) == -1) + exit(1); + + if (!S_ISREG(st.st_mode)) + exit(1); /* Try to lock file */ #ifdef LOCKF_SUPPORT if (lockf(pid_file_handle, F_TLOCK, 0) == -1) { diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index 296516eecfd6a..5733a526f7dcb 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -433,12 +433,16 @@ static void depopulate_all_mock_resources(void) static struct cxl_mock_res *alloc_mock_res(resource_size_t size, int align) { - struct cxl_mock_res *res = kzalloc(sizeof(*res), GFP_KERNEL); struct genpool_data_align data = { .align = align, }; unsigned long phys; + struct cxl_mock_res *res __free(kfree) = kzalloc(sizeof(*res), + GFP_KERNEL); + if (!res) + return NULL; + INIT_LIST_HEAD(&res->list); phys = gen_pool_alloc_algo(cxl_mock_pool, size, gen_pool_first_fit_align, &data); @@ -453,7 +457,7 @@ static struct cxl_mock_res *alloc_mock_res(resource_size_t size, int align) list_add(&res->list, &mock_res); mutex_unlock(&mock_res_lock); - return res; + return no_free_ptr(res); } /* Only update CFMWS0 as this is used by the auto region. */ @@ -1951,7 +1955,7 @@ static __init int cxl_test_init(void) err_mem: cxl_mem_exit(); err_root: - platform_device_put(cxl_acpi); + platform_device_unregister(cxl_acpi); err_rch: cxl_rch_topo_exit(); err_single: diff --git a/tools/testing/cxl/test/cxl_translate.c b/tools/testing/cxl/test/cxl_translate.c index 16328b2112b23..25a27e01ac21b 100644 --- a/tools/testing/cxl/test/cxl_translate.c +++ b/tools/testing/cxl/test/cxl_translate.c @@ -236,8 +236,8 @@ static int setup_xor_mapping(void) if (!cximsd) return -ENOMEM; - memcpy(cximsd->xormaps, xormaps, nr_maps * sizeof(*cximsd->xormaps)); cximsd->nr_maps = nr_maps; + memcpy(cximsd->xormaps, xormaps, nr_maps * sizeof(*cximsd->xormaps)); return 0; } diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 271c7ad8cc32c..739343cd58022 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -312,12 +312,17 @@ static int mock_get_event(struct device *dev, struct cxl_mbox_cmd *cmd) static int mock_clear_event(struct device *dev, struct cxl_mbox_cmd *cmd) { - struct cxl_mbox_clear_event_payload *pl = cmd->payload_in; + struct cxl_mbox_clear_event_payload *pl; struct mock_event_log *log; - u8 log_type = pl->event_log; + u8 log_type; u16 handle; int nr; + if (cmd->size_in < sizeof(*pl)) + return -EINVAL; + + pl = cmd->payload_in; + log_type = pl->event_log; if (log_type >= CXL_EVENT_TYPE_MAX) return -EINVAL; @@ -574,14 +579,19 @@ static int mock_gsl(struct cxl_mbox_cmd *cmd) static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd) { struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; - struct cxl_mbox_get_log *gl = cmd->payload_in; - u32 offset = le32_to_cpu(gl->offset); - u32 length = le32_to_cpu(gl->length); uuid_t uuid = DEFINE_CXL_CEL_UUID; + struct cxl_mbox_get_log *gl; void *data = &mock_cel; + u32 offset; + u32 length; if (cmd->size_in < sizeof(*gl)) return -EINVAL; + + gl = cmd->payload_in; + offset = le32_to_cpu(gl->offset); + length = le32_to_cpu(gl->length); + if (length > cxl_mbox->payload_size) return -EINVAL; if (offset + length > sizeof(mock_cel)) @@ -1053,7 +1063,7 @@ static int mock_get_lsa(struct cxl_mockmem_data *mdata, return -EINVAL; offset = le32_to_cpu(get_lsa->offset); length = le32_to_cpu(get_lsa->length); - if (offset + length > LSA_SIZE) + if (offset > LSA_SIZE || length > LSA_SIZE - offset) return -EINVAL; if (length > cmd->size_out) return -EINVAL; @@ -1073,7 +1083,7 @@ static int mock_set_lsa(struct cxl_mockmem_data *mdata, return -EINVAL; offset = le32_to_cpu(set_lsa->offset); length = cmd->size_in - sizeof(*set_lsa); - if (offset + length > LSA_SIZE) + if (offset > LSA_SIZE || length > LSA_SIZE - offset) return -EINVAL; memcpy(lsa + offset, &set_lsa->data[0], length); @@ -1336,10 +1346,14 @@ static int mock_fw_info(struct cxl_mockmem_data *mdata, static int mock_transfer_fw(struct cxl_mockmem_data *mdata, struct cxl_mbox_cmd *cmd) { - struct cxl_mbox_transfer_fw *transfer = cmd->payload_in; + struct cxl_mbox_transfer_fw *transfer; void *fw = mdata->fw; size_t offset, length; + if (cmd->size_in < sizeof(*transfer)) + return -EINVAL; + + transfer = cmd->payload_in; offset = le32_to_cpu(transfer->offset) * CXL_FW_TRANSFER_ALIGNMENT; length = cmd->size_in - sizeof(*transfer); if (offset + length > FW_SIZE) @@ -1415,11 +1429,18 @@ static int mock_get_test_feature(struct cxl_mockmem_data *mdata, struct cxl_mbox_cmd *cmd) { struct vendor_test_feat *output = cmd->payload_out; - struct cxl_mbox_get_feat_in *input = cmd->payload_in; - u16 offset = le16_to_cpu(input->offset); - u16 count = le16_to_cpu(input->count); + struct cxl_mbox_get_feat_in *input; + u16 offset; + u16 count; u8 *ptr; + if (cmd->size_in < sizeof(*input)) + return -EINVAL; + + input = cmd->payload_in; + offset = le16_to_cpu(input->offset); + count = le16_to_cpu(input->count); + if (offset > sizeof(*output)) { cmd->return_code = CXL_MBOX_CMD_RC_INPUT; return -EINVAL; @@ -1703,7 +1724,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) return -ENOMEM; dev_set_drvdata(dev, mdata); - mdata->lsa = vmalloc(LSA_SIZE); + mdata->lsa = vzalloc(LSA_SIZE); if (!mdata->lsa) return -ENOMEM; mdata->fw = vmalloc(FW_SIZE); diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 1c61a0ed740d4..266c9dd953512 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -17,7 +17,7 @@ from enum import Enum, auto from typing import Iterable, Iterator, List, Optional, Tuple -from kunit_printer import Printer, stdout +from kunit_printer import Printer class Test: """ @@ -57,7 +57,7 @@ def __repr__(self) -> str: def add_error(self, printer: Printer, error_message: str) -> None: """Records an error that occurred while parsing this test.""" self.counts.errors += 1 - printer.print_with_timestamp(stdout.red('[ERROR]') + f' Test: {self.name}: {error_message}') + printer.print_with_timestamp(printer.red('[ERROR]') + f' Test: {self.name}: {error_message}') def ok_status(self) -> bool: """Returns true if the status was ok, i.e. passed or skipped.""" @@ -544,7 +544,7 @@ def format_test_result(test: Test, printer: Printer) -> str: return printer.yellow('[NO TESTS RUN] ') + test.name if test.status == TestStatus.TEST_CRASHED: print_log(test.log, printer) - return stdout.red('[CRASHED] ') + test.name + return printer.red('[CRASHED] ') + test.name print_log(test.log, printer) return printer.red('[FAILED] ') + test.name @@ -651,11 +651,11 @@ def print_summary_line(test: Test, printer: Printer) -> None: printer - Printer object to output results """ if test.status == TestStatus.SUCCESS: - color = stdout.green + color = printer.green elif test.status in (TestStatus.SKIPPED, TestStatus.NO_TESTS): - color = stdout.yellow + color = printer.yellow else: - color = stdout.red + color = printer.red printer.print_with_timestamp(color(f'Testing complete. {test.counts}')) # Summarize failures that might have gone off-screen since we had a lot diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 6e59b8f63e416..641a180fb35f4 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -37,6 +37,7 @@ TARGETS += filesystems/fat TARGETS += filesystems/overlayfs TARGETS += filesystems/statmount TARGETS += filesystems/mount-notify +TARGETS += filesystems/nsfs TARGETS += filesystems/fuse TARGETS += filesystems/move_mount TARGETS += filesystems/empty_mntns @@ -85,7 +86,6 @@ TARGETS += net/ppp TARGETS += net/rds TARGETS += net/tcp_ao TARGETS += nolibc -TARGETS += nsfs TARGETS += pci_endpoint TARGETS += pcie_bwctrl TARGETS += perf_events diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 776fbe3cb8f9b..37164322a1023 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -77,7 +77,7 @@ In case of linker errors when running selftests, try using static linking: .. code-block:: console - $ LDLIBS=-static PKG_CONFIG='pkg-config --static' vmtest.sh + $ LDLIBS=-static EXTRA_LDFLAGS=-static PKG_CONFIG='pkg-config --static' vmtest.sh .. note:: Some distros may not support static linking. diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c b/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c new file mode 100644 index 0000000000000..32159dc64da8b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Google LLC */ +#include +#include +#include +#include +#include +#include "cgroup_skb_direct_packet_access.skel.h" + +#define OLD_QUERY_SIZE offsetofend(union bpf_attr, query.prog_cnt) +#define FULL_QUERY_SIZE offsetofend(union bpf_attr, query.revision) + +static void test_query_size_boundaries(void) +{ + struct cgroup_skb_direct_packet_access *skel; + struct bpf_link *link = NULL; + union bpf_attr attr; + int cg_fd = -1; + int err; + + skel = cgroup_skb_direct_packet_access__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + return; + + cg_fd = test__join_cgroup("/attr_size_cg"); + if (!ASSERT_GE(cg_fd, 0, "join_cgroup")) + goto cleanup; + + link = bpf_program__attach_cgroup(skel->progs.direct_packet_access, + cg_fd); + if (!ASSERT_OK_PTR(link, "cg_attach")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.query.target_fd = cg_fd; + attr.query.attach_type = BPF_CGROUP_INET_INGRESS; + attr.query.revision = 0xdeadbeefdeadbeefULL; + + err = syscall(__NR_bpf, BPF_PROG_QUERY, &attr, OLD_QUERY_SIZE); + if (ASSERT_OK(err, "query_old_size")) { + ASSERT_EQ(attr.query.prog_cnt, 1, "prog_cnt_written_old"); + ASSERT_EQ(attr.query.revision, 0xdeadbeefdeadbeefULL, + "revision_not_written_old"); + } + + memset(&attr, 0, sizeof(attr)); + attr.query.target_fd = cg_fd; + attr.query.attach_type = BPF_CGROUP_INET_INGRESS; + + err = syscall(__NR_bpf, BPF_PROG_QUERY, &attr, FULL_QUERY_SIZE); + if (!ASSERT_OK(err, "query_full_size")) + goto cleanup; + + ASSERT_EQ(attr.query.prog_cnt, 1, "prog_cnt_written"); + ASSERT_GT(attr.query.revision, 0, "revision_written"); + +cleanup: + if (link) + bpf_link__destroy(link); + if (cg_fd >= 0) + close(cg_fd); + cgroup_skb_direct_packet_access__destroy(skel); +} + +void test_bpf_attr_size(void) +{ + if (test__start_subtest("query_size_boundaries")) + test_query_size_boundaries(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index 215878ea04dee..4b06a54907144 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -5,6 +5,8 @@ #include "test_bpf_nf.skel.h" #include "test_bpf_nf_fail.skel.h" +#define CT_OPTS_ERROR_GUARD 0x12345678 + static char log_buf[1024 * 1024]; struct { @@ -119,6 +121,10 @@ static void test_bpf_nf_ct(int mode) ASSERT_EQ(skel->bss->test_einval_reserved_new, -EINVAL, "Test EINVAL for reserved in new struct not set to 0"); ASSERT_EQ(skel->bss->test_einval_netns_id, -EINVAL, "Test EINVAL for netns_id < -1"); ASSERT_EQ(skel->bss->test_einval_len_opts, -EINVAL, "Test EINVAL for len__opts != NF_BPF_CT_OPTS_SZ"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_lookup, CT_OPTS_ERROR_GUARD, + "Test no error write for lookup opts__sz before error field"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_alloc, CT_OPTS_ERROR_GUARD, + "Test no error write for alloc opts__sz before error field"); ASSERT_EQ(skel->bss->test_eproto_l4proto, -EPROTO, "Test EPROTO for l4proto != TCP or UDP"); ASSERT_EQ(skel->bss->test_enonet_netns_id, -ENONET, "Test ENONET for bad but valid netns_id"); ASSERT_EQ(skel->bss->test_enoent_lookup, -ENOENT, "Test ENOENT for failed lookup"); diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index e401146207510..f589eefbf9fbd 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -469,7 +469,7 @@ verify_umulti_link_info(int fd, bool retprobe, __u64 *offsets, ASSERT_EQ(info.uprobe_multi.pid, getpid(), "info.uprobe_multi.pid"); ASSERT_EQ(info.uprobe_multi.count, 3, "info.uprobe_multi.count"); - ASSERT_EQ(info.uprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN, + ASSERT_EQ(info.uprobe_multi.flags & BPF_F_UPROBE_MULTI_RETURN, retprobe, "info.uprobe_multi.flags.retprobe"); ASSERT_EQ(info.uprobe_multi.path_size, strlen(path) + 1, "info.uprobe_multi.path_size"); ASSERT_STREQ(path_buf, path, "info.uprobe_multi.path"); diff --git a/tools/testing/selftests/bpf/prog_tests/htab_update.c b/tools/testing/selftests/bpf/prog_tests/htab_update.c index ea1a6766fbe98..0a28d43469240 100644 --- a/tools/testing/selftests/bpf/prog_tests/htab_update.c +++ b/tools/testing/selftests/bpf/prog_tests/htab_update.c @@ -23,7 +23,7 @@ static void test_reenter_update(void) if (!ASSERT_OK_PTR(skel, "htab_update__open")) return; - bpf_program__set_autoload(skel->progs.bpf_obj_free_fields, true); + bpf_program__set_autoload(skel->progs.bpf_obj_cancel_fields, true); err = htab_update__load(skel); if (!ASSERT_TRUE(!err, "htab_update__load") || err) goto out; @@ -50,7 +50,7 @@ static void test_reenter_update(void) /* * Second update: replace existing element with same key and trigger * the reentrancy of bpf_map_update_elem(). - * check_and_free_fields() calls bpf_obj_free_fields() on the old + * check_and_cancel_fields() calls bpf_obj_cancel_fields() on the old * value, which is where fentry program runs and performs a nested * bpf_map_update_elem(), triggering -EDEADLK. */ diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index 6f25b5f39a79c..62d974a1987ff 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -131,13 +131,14 @@ static void test_linked_list_fail_prog(const char *prog_name, const char *err_ms linked_list_fail__destroy(skel); } -static void clear_fields(struct bpf_map *map) +static void clear_fields(struct bpf_program *prog) { - char buf[24]; - int key = 0; + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; - memset(buf, 0xff, sizeof(buf)); - ASSERT_OK(bpf_map__update_elem(map, &key, sizeof(key), buf, sizeof(buf), 0), "check_and_free_fields"); + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + ASSERT_OK(ret, "clear_fields"); + ASSERT_OK(opts.retval, "clear_fields retval"); } enum { @@ -170,31 +171,31 @@ static void test_linked_list_success(int mode, bool leave_in_map) ASSERT_OK(ret, "map_list_push_pop"); ASSERT_OK(opts.retval, "map_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_push_pop), &opts); ASSERT_OK(ret, "inner_map_list_push_pop"); ASSERT_OK(opts.retval, "inner_map_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop), &opts); ASSERT_OK(ret, "global_list_push_pop"); ASSERT_OK(opts.retval, "global_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop_nested), &opts); ASSERT_OK(ret, "global_list_push_pop_nested"); ASSERT_OK(opts.retval, "global_list_push_pop_nested retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_nested_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_array_push_pop), &opts); ASSERT_OK(ret, "global_list_array_push_pop"); ASSERT_OK(opts.retval, "global_list_array_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_array_list); if (mode == PUSH_POP) goto end; @@ -204,19 +205,19 @@ static void test_linked_list_success(int mode, bool leave_in_map) ASSERT_OK(ret, "map_list_push_pop_multiple"); ASSERT_OK(opts.retval, "map_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_push_pop_multiple), &opts); ASSERT_OK(ret, "inner_map_list_push_pop_multiple"); ASSERT_OK(opts.retval, "inner_map_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop_multiple), &opts); ASSERT_OK(ret, "global_list_push_pop_multiple"); ASSERT_OK(opts.retval, "global_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); if (mode == PUSH_POP_MULT) goto end; @@ -226,19 +227,19 @@ static void test_linked_list_success(int mode, bool leave_in_map) ASSERT_OK(ret, "map_list_in_list"); ASSERT_OK(opts.retval, "map_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_in_list), &opts); ASSERT_OK(ret, "inner_map_list_in_list"); ASSERT_OK(opts.retval, "inner_map_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_in_list), &opts); ASSERT_OK(ret, "global_list_in_list"); ASSERT_OK(opts.retval, "global_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); end: linked_list__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/map_kptr.c b/tools/testing/selftests/bpf/prog_tests/map_kptr.c index 03b46f17cf537..ec6f2f2e83089 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/map_kptr.c @@ -51,7 +51,6 @@ static void test_map_kptr_success(bool test_run) ret = bpf_map__update_elem(skel->maps.array_map, &key, sizeof(key), buf, sizeof(buf), 0); ASSERT_OK(ret, "array_map update"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); @@ -59,49 +58,42 @@ static void test_map_kptr_success(bool test_run) ret = bpf_map__update_elem(skel->maps.pcpu_array_map, &key, sizeof(key), pbuf, cpu * sizeof(buf), 0); ASSERT_OK(ret, "pcpu_array_map update"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.pcpu_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "pcpu_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.hash_malloc_map, &key, sizeof(key), 0); ASSERT_OK(ret, "hash_malloc_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.pcpu_hash_malloc_map, &key, sizeof(key), 0); ASSERT_OK(ret, "pcpu_hash_malloc_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.lru_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "lru_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.lru_pcpu_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "lru_pcpu_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); @@ -175,7 +167,7 @@ void serial_test_map_kptr(void) ASSERT_OK(kern_sync_rcu(), "sync rcu"); wait_for_map_release(); - /* Observe refcount dropping to 1 on synchronous delete elem */ + /* Observe refcount dropping to 1 on map release. */ test_map_kptr_success(true); } diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c new file mode 100644 index 0000000000000..b8538fc4fc3f6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "test_kmods/bpf_testmod.h" +#include "bpf_util.h" + +static void check_attach_reject(const struct bpf_insn *program, size_t prog_len) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + char error[4096]; + int bpf_fd, tp_fd; + + opts.log_level = 2; + opts.log_buf = error; + opts.log_size = sizeof(error); + + bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", + program, prog_len, &opts); + if (!ASSERT_GE(bpf_fd, 0, "prog_load")) + return; + + tp_fd = bpf_raw_tracepoint_open("bpf_testmod_test_writable_bare_tp", bpf_fd); + ASSERT_EQ(tp_fd, -EINVAL, "bpf_raw_tracepoint_open"); + if (tp_fd >= 0) + close(tp_fd); + + close(bpf_fd); +} + +void test_raw_tp_writable_reject_bad_access(void) +{ + const struct bpf_insn program[] = { + /* r6 is our tp buffer */ + BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), + /* one byte beyond the end of the writable context */ + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, + sizeof(struct bpf_testmod_test_writable_ctx)), + BPF_EXIT_INSN(), + }; + + const struct bpf_insn negative_var_off_program[] = { + BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), + /* make var_off negative, but keep the effective access offset non-negative */ + BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, -8), + /* one byte beyond the end of the writable context */ + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, + sizeof(struct bpf_testmod_test_writable_ctx) + 8), + BPF_EXIT_INSN(), + }; + + if (test__start_subtest("past_end")) + check_attach_reject(program, ARRAY_SIZE(program)); + + if (test__start_subtest("negative_var_off_past_end")) + check_attach_reject(negative_var_off_program, + ARRAY_SIZE(negative_var_off_program)); +} diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c deleted file mode 100644 index 216b0dfac0fe6..0000000000000 --- a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "bpf_util.h" - -void test_raw_tp_writable_reject_nbd_invalid(void) -{ - __u32 duration = 0; - char error[4096]; - int bpf_fd = -1, tp_fd = -1; - - const struct bpf_insn program[] = { - /* r6 is our tp buffer */ - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), - /* one byte beyond the end of the nbd_request struct */ - BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, - sizeof(struct nbd_request)), - BPF_EXIT_INSN(), - }; - - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .log_level = 2, - .log_buf = error, - .log_size = sizeof(error), - ); - - bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", - program, ARRAY_SIZE(program), - &opts); - if (CHECK(bpf_fd < 0, "bpf_raw_tracepoint_writable load", - "failed: %d errno %d\n", bpf_fd, errno)) - return; - - tp_fd = bpf_raw_tracepoint_open("nbd_send_request", bpf_fd); - if (CHECK(tp_fd >= 0, "bpf_raw_tracepoint_writable open", - "erroneously succeeded\n")) - goto out_bpffd; - - close(tp_fd); -out_bpffd: - close(bpf_fd); -} diff --git a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c index d2c0542716a84..1737eba343234 100644 --- a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c @@ -57,6 +57,7 @@ void test_percpu_hash_refcounted_kptr_refcount_leak(void) .data_size_in = sizeof(pkt_v4), .repeat = 1, ); + LIBBPF_OPTS(bpf_test_run_opts, syscall_opts); cpu_nr = libbpf_num_possible_cpus(); if (!ASSERT_GT(cpu_nr, 0, "libbpf_num_possible_cpus")) @@ -87,8 +88,11 @@ void test_percpu_hash_refcounted_kptr_refcount_leak(void) if (!ASSERT_EQ(opts.retval, 2, "opts.retval")) goto out; - err = bpf_map__update_elem(map, &key, sizeof(key), values, values_sz, 0); - if (!ASSERT_OK(err, "bpf_map__update_elem")) + fd = bpf_program__fd(skel->progs.clear_percpu_hash_kptr); + err = bpf_prog_test_run_opts(fd, &syscall_opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto out; + if (!ASSERT_EQ(syscall_opts.retval, 1, "syscall_opts.retval")) goto out; fd = bpf_program__fd(skel->progs.check_percpu_hash_refcount); diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 06cd24e37b3f8..e67140c24b742 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -77,6 +77,7 @@ #include "verifier_precision.skel.h" #include "verifier_prevent_map_lookup.skel.h" #include "verifier_private_stack.skel.h" +#include "verifier_ptr_to_buf.skel.h" #include "verifier_raw_stack.skel.h" #include "verifier_raw_tp_writable.skel.h" #include "verifier_reg_equal.skel.h" @@ -225,6 +226,7 @@ void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); } void test_verifier_precision(void) { RUN(verifier_precision); } void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); } void test_verifier_private_stack(void) { RUN(verifier_private_stack); } +void test_verifier_ptr_to_buf(void) { RUN(verifier_ptr_to_buf); } void test_verifier_raw_stack(void) { RUN(verifier_raw_stack); } void test_verifier_raw_tp_writable(void) { RUN(verifier_raw_tp_writable); } void test_verifier_reg_equal(void) { RUN(verifier_reg_equal); } diff --git a/tools/testing/selftests/bpf/prog_tests/verifier_log.c b/tools/testing/selftests/bpf/prog_tests/verifier_log.c index c01c0114af1b7..4542bb586d723 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier_log.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier_log.c @@ -317,6 +317,7 @@ static void verif_btf_log_subtest(bool bad_btf) res = load_btf(&opts, true); ASSERT_EQ(res, -ENOSPC, "half_log_fd"); ASSERT_EQ(strlen(logs.buf), 24, "log_fixed_25"); + strscpy(op_name, "log_fixed", sizeof(op_name)); ASSERT_STRNEQ(logs.buf, logs.reference, 24, op_name); /* validate rolling verifier log logic: try all variations of log buf diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c index d64ba7ddaed54..d7fb561ed4fb9 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c @@ -52,7 +52,7 @@ SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx) bpf_d_path(&file->f_path, d_path_buf, D_PATH_BUF_SIZE); BPF_SEQ_PRINTF(seq, "%08llx ", vma->vm_pgoff << 12); - BPF_SEQ_PRINTF(seq, "%02x:%02x %u", MAJOR(dev), MINOR(dev), + BPF_SEQ_PRINTF(seq, "%02x:%02x %llu", MAJOR(dev), MINOR(dev), file->f_inode->i_ino); BPF_SEQ_PRINTF(seq, "\t%s\n", d_path_buf); } else { diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 0e04c31b91c0c..774706e7b058b 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c @@ -866,7 +866,7 @@ int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags) * access NR_CPUS, the upper bound for nr_cpus, so we infer * it from the size of cpumask_t. */ - if (nr_cpus < 0 || nr_cpus >= CPUMASK_TEST_MASKLEN * 8) { + if (nr_cpus < 0 || nr_cpus > CPUMASK_TEST_MASKLEN * 8) { err = 3; goto out; } diff --git a/tools/testing/selftests/bpf/progs/file_reader.c b/tools/testing/selftests/bpf/progs/file_reader.c index 462712ff3b8a0..aa2c05cce2b30 100644 --- a/tools/testing/selftests/bpf/progs/file_reader.c +++ b/tools/testing/selftests/bpf/progs/file_reader.c @@ -50,7 +50,7 @@ int on_open_expect_fault(void *c) goto out; local_err = bpf_dynptr_read(tmp_buf, user_buf_sz, &dynptr, user_buf_sz, 0); - if (local_err == -EFAULT) { /* Expect page fault */ + if (local_err == -EFAULT || local_err == 0) { /* Expect page fault or success */ local_err = 0; run_success = 1; } diff --git a/tools/testing/selftests/bpf/progs/htab_update.c b/tools/testing/selftests/bpf/progs/htab_update.c index 195d3b2fba00c..62c1b1325ec27 100644 --- a/tools/testing/selftests/bpf/progs/htab_update.c +++ b/tools/testing/selftests/bpf/progs/htab_update.c @@ -22,8 +22,8 @@ struct { int pid = 0; int update_err = 0; -SEC("?fentry/bpf_obj_free_fields") -int bpf_obj_free_fields(void *ctx) +SEC("?fentry/bpf_obj_cancel_fields") +int bpf_obj_cancel_fields(void *ctx) { __u32 key = 0; struct val value = { .payload = 1 }; diff --git a/tools/testing/selftests/bpf/progs/linked_list.c b/tools/testing/selftests/bpf/progs/linked_list.c index 421f40835acd7..fa97faa5358bc 100644 --- a/tools/testing/selftests/bpf/progs/linked_list.c +++ b/tools/testing/selftests/bpf/progs/linked_list.c @@ -290,6 +290,77 @@ int test_list_in_list(struct bpf_spin_lock *lock, struct bpf_list_head *head) return list_in_list(lock, head, true); } +#define MAX_LIST_CLEAR_NODES 256 + +static __always_inline +int clear_list(struct bpf_spin_lock *lock, struct bpf_list_head *head) +{ + struct bpf_list_node *n; + int i; + + for (i = 0; i < MAX_LIST_CLEAR_NODES; i++) { + bpf_spin_lock(lock); + n = bpf_list_pop_front(head); + bpf_spin_unlock(lock); + if (!n) + return 0; + bpf_obj_drop(container_of(n, struct foo, node2)); + } + return 1; +} + +SEC("syscall") +int clear_map_list(void *ctx) +{ + struct map_value *v; + + v = bpf_map_lookup_elem(&array_map, &(int){0}); + if (!v) + return 1; + return clear_list(&v->lock, &v->head); +} + +SEC("syscall") +int clear_inner_map_list(void *ctx) +{ + struct map_value *v; + void *map; + + map = bpf_map_lookup_elem(&map_of_maps, &(int){0}); + if (!map) + return 1; + v = bpf_map_lookup_elem(map, &(int){0}); + if (!v) + return 1; + return clear_list(&v->lock, &v->head); +} + +SEC("syscall") +int clear_global_list(void *ctx) +{ + return clear_list(&glock, &ghead); +} + +SEC("syscall") +int clear_global_nested_list(void *ctx) +{ + return clear_list(&ghead_nested.inner.lock, &ghead_nested.inner.head); +} + +SEC("syscall") +int clear_global_array_list(void *ctx) +{ + int ret; + + ret = clear_list(&glock_c, &ghead_array[0]); + if (ret) + return ret; + ret = clear_list(&glock_c, &ghead_array[1]); + if (ret) + return ret; + return clear_list(&glock_c, &ghead_array_one[0]); +} + SEC("tc") int map_list_push_pop(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr.c b/tools/testing/selftests/bpf/progs/refcounted_kptr.c index c847398837ccd..9837dccfc141c 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr.c @@ -615,13 +615,31 @@ int percpu_hash_refcount_leak(void *ctx) struct map_value *v; int key = 0; - v = bpf_map_lookup_elem(&percpu_hash, &key); + v = bpf_map_lookup_percpu_elem(&percpu_hash, &key, 0); if (!v) return 0; return __insert_in_list(&head, &lock, &v->node); } +SEC("syscall") +int clear_percpu_hash_kptr(void *ctx) +{ + struct node_data *n; + struct map_value *v; + int key = 0; + + v = bpf_map_lookup_percpu_elem(&percpu_hash, &key, 0); + if (!v) + return 0; + + n = bpf_kptr_xchg(&v->node, NULL); + if (!n) + return 0; + bpf_obj_drop(n); + return probe_read_refcount(); +} + SEC("tc") int check_percpu_hash_refcount(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index 076fbf03a1268..df43649ecb785 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -10,6 +10,8 @@ #define EINVAL 22 #define ENOENT 2 +#define CT_OPTS_ERROR_GUARD 0x12345678 + #define NF_CT_ZONE_DIR_ORIG (1 << IP_CT_DIR_ORIGINAL) #define NF_CT_ZONE_DIR_REPL (1 << IP_CT_DIR_REPLY) @@ -19,6 +21,8 @@ int test_einval_reserved = 0; int test_einval_reserved_new = 0; int test_einval_netns_id = 0; int test_einval_len_opts = 0; +int test_einval_len_opts_small_lookup = 0; +int test_einval_len_opts_small_alloc = 0; int test_eproto_l4proto = 0; int test_enonet_netns_id = 0; int test_enoent_lookup = 0; @@ -124,6 +128,28 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, else test_einval_len_opts = opts_def.error; + opts_def.error = CT_OPTS_ERROR_GUARD; + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def.netns_id)); + if (ct) { + bpf_ct_release(ct); + test_einval_len_opts_small_lookup = -EINVAL; + } else { + test_einval_len_opts_small_lookup = opts_def.error; + } + + opts_def.error = CT_OPTS_ERROR_GUARD; + ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def.netns_id)); + if (ct) { + ct = bpf_ct_insert_entry(ct); + if (ct) + bpf_ct_release(ct); + test_einval_len_opts_small_alloc = -EINVAL; + } else { + test_einval_len_opts_small_alloc = opts_def.error; + } + opts_def.l4proto = IPPROTO_ICMP; ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); diff --git a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c index 32127f1cd6872..30f1de458669d 100644 --- a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c +++ b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c @@ -6,6 +6,7 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ +#define BPF_NO_KFUNC_PROTOTYPES #include "vmlinux.h" #include #include @@ -36,12 +37,10 @@ enum bpf_fou_encap_type___local { FOU_BPF_ENCAP_GUE___local, }; -struct bpf_fou_encap; - int bpf_skb_set_fou_encap(struct __sk_buff *skb_ctx, - struct bpf_fou_encap *encap, int type) __ksym; + struct bpf_fou_encap___local *encap, int type) __ksym; int bpf_skb_get_fou_encap(struct __sk_buff *skb_ctx, - struct bpf_fou_encap *encap) __ksym; + struct bpf_fou_encap___local *encap) __ksym; struct xfrm_state * bpf_xdp_get_xfrm_state(struct xdp_md *ctx, struct bpf_xfrm_state_opts *opts, u32 opts__sz) __ksym; @@ -781,7 +780,7 @@ int ipip_gue_set_tunnel(struct __sk_buff *skb) encap.sport = 0; encap.dport = bpf_htons(5555); - ret = bpf_skb_set_fou_encap(skb, (struct bpf_fou_encap *)&encap, + ret = bpf_skb_set_fou_encap(skb, &encap, bpf_core_enum_value(enum bpf_fou_encap_type___local, FOU_BPF_ENCAP_GUE___local)); if (ret < 0) { @@ -820,7 +819,7 @@ int ipip_fou_set_tunnel(struct __sk_buff *skb) encap.sport = 0; encap.dport = bpf_htons(5555); - ret = bpf_skb_set_fou_encap(skb, (struct bpf_fou_encap *)&encap, + ret = bpf_skb_set_fou_encap(skb, &encap, FOU_BPF_ENCAP_FOU___local); if (ret < 0) { log_err(ret); @@ -843,7 +842,7 @@ int ipip_encap_get_tunnel(struct __sk_buff *skb) return TC_ACT_SHOT; } - ret = bpf_skb_get_fou_encap(skb, (struct bpf_fou_encap *)&encap); + ret = bpf_skb_get_fou_encap(skb, &encap); if (ret < 0) { log_err(ret); return TC_ACT_SHOT; diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index c1ae013dee29c..bc431f46aafdb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1890,25 +1890,25 @@ __naked void bounds_refinement_tnum_umax(void *ctx) /* This test covers the bounds deduction when the u64 range and the tnum * overlap only at umin. After instruction 3, the ranges look as follows: * - * 0 umin=0xe00 umax=0xeff U64_MAX + * 0 umin=0xe1 umax=0xf0 U64_MAX * | [xxxxxxxxxxxxxx] | * |----------------------------|------------------------------| * | x x | tnum values * - * The verifier can therefore deduce that the R0=0xe0=224. + * The verifier can therefore deduce that the R0=0xe1=225. */ SEC("socket") __description("bounds refinement with single-value tnum on umin") -__msg("3: (15) if r0 == 0xf0 {{.*}} R0=224") +__msg("3: (15) if r0 == 0xf1 {{.*}} R0=225") __success __log_level(2) __naked void bounds_refinement_tnum_umin(void *ctx) { asm volatile(" \ call %[bpf_get_prandom_u32]; \ - r0 |= 0xe0; \ - r0 &= 0xf0; \ - if r0 == 0xf0 goto +2; \ - if r0 == 0xe0 goto +1; \ + r0 |= 0xe1; \ + r0 &= 0xf1; \ + if r0 == 0xf1 goto +2; \ + if r0 == 0xe1 goto +1; \ r10 = 0; \ exit; \ " : diff --git a/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c b/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c new file mode 100644 index 0000000000000..12cf24db46a0a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_misc.h" + +SEC("iter/bpf_map_elem") +__description("PTR_TO_BUF: reject negative const offset") +__failure +__msg("invalid negative rdwr buffer offset") +__naked void ptr_to_buf_reject_negative_const_offset(void) +{ + asm volatile ("r0 = 0; \ + r2 = *(u64 *)(r1 + %[value_off]); \ + if r2 == 0 goto l0_%=; \ + r2 += -8; \ + r0 = *(u64 *)(r2 + 0); \ +l0_%=: \ + exit; \ + " + : + : __imm_const(value_off, + offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c b/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c index 14a0172e2141e..4055a6443bc20 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c @@ -47,4 +47,20 @@ l0_%=: /* shift the buffer pointer to a variable location */\ : __clobber_all); } +SEC("raw_tracepoint.w") +__description("raw_tracepoint_writable: reject negative const offset") +__failure +__msg("invalid negative tracepoint buffer offset") +__naked void tracepoint_writable_reject_negative_const_offset(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + 0); \ + r6 += -8; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/xdp_flowtable.c b/tools/testing/selftests/bpf/progs/xdp_flowtable.c index 7fdc7b23ee749..e67daa02749d5 100644 --- a/tools/testing/selftests/bpf/progs/xdp_flowtable.c +++ b/tools/testing/selftests/bpf/progs/xdp_flowtable.c @@ -15,7 +15,10 @@ struct bpf_flowtable_opts___local { s32 error; }; -struct flow_offload_tuple_rhash * +struct flow_offload_tuple_rhash___local { +}; + +struct flow_offload_tuple_rhash___local * bpf_xdp_flow_lookup(struct xdp_md *, struct bpf_fib_lookup *, struct bpf_flowtable_opts___local *, u32) __ksym; @@ -67,7 +70,7 @@ int xdp_flowtable_do_lookup(struct xdp_md *ctx) { void *data_end = (void *)(long)ctx->data_end; struct bpf_flowtable_opts___local opts = {}; - struct flow_offload_tuple_rhash *tuplehash; + struct flow_offload_tuple_rhash___local *tuplehash; struct bpf_fib_lookup tuple = { .ifindex = ctx->ingress_ifindex, }; diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 2f869daf8a06b..9ca8022853933 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -382,7 +382,7 @@ main() local exit_command="poweroff -f" local debug_shell="no" - while getopts ':hskl:id:j:' opt; do + while getopts ':hsl:id:j:' opt; do case ${opt} in l) LOCAL_ROOTFS_IMAGE="$OPTARG" diff --git a/tools/testing/selftests/drivers/net/hw/tso.py b/tools/testing/selftests/drivers/net/hw/tso.py index bb675e3dac88f..abc575e4b3406 100755 --- a/tools/testing/selftests/drivers/net/hw/tso.py +++ b/tools/testing/selftests/drivers/net/hw/tso.py @@ -187,28 +187,24 @@ def query_nic_features(cfg) -> None: cfg.wanted_features.add(f["name"]) cfg.hw_features = set() - hw_all_features_cmd = "" for f in features["hw"]["bits"]["bit"]: if f.get("value", False): - feature = f["name"] - cfg.hw_features.add(feature) - hw_all_features_cmd += f" {feature} on" - try: - ethtool(f"-K {cfg.ifname} {hw_all_features_cmd}") - except Exception as e: - ksft_pr(f"WARNING: failure enabling all hw features: {e}") - ksft_pr("partial gso feature detection may be impacted") + cfg.hw_features.add(f["name"]) # Check which features are supported via GSO partial cfg.partial_features = set() if 'tx-gso-partial' in cfg.hw_features: + seg_features = {f for f in cfg.hw_features if "segmentation" in f} + ethtool(f"-K {cfg.ifname} " + + " ".join(f"{f} on" for f in seg_features)) + ethtool(f"-K {cfg.ifname} tx-gso-partial off") no_partial = set() features = cfg.ethnl.features_get({"header": {"dev-index": cfg.ifindex}}) for f in features["active"]["bits"]["bit"]: no_partial.add(f["name"]) - cfg.partial_features = cfg.hw_features - no_partial + cfg.partial_features = seg_features - no_partial ethtool(f"-K {cfg.ifname} tx-gso-partial on") restore_wanted_features(cfg) diff --git a/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc b/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc index e71cc3ad0bdf9..6d00d3c0f493a 100644 --- a/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc +++ b/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc @@ -6,7 +6,7 @@ original_group=`stat -c "%g" .` original_owner=`stat -c "%u" .` -local mount_point=$(get_mount_point) +mount_point=$(get_mount_point) mount_options=$(get_mnt_options "$mount_point") diff --git a/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc b/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc index 8e905d4fe6dd2..f985ff391463a 100644 --- a/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc +++ b/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc @@ -36,15 +36,23 @@ make_str() { data=`printf -- 'X%.0s' $(seq $cnt)` - printf "${val}${data}" + # Return escape-sequence text (e.g. "\003\000..."); the caller + # converts to binary. Shell command substitution strips NUL bytes, + # so the binary form cannot survive being captured into a variable. + printf '%s' "${val}${data}" } write_buffer() { id=$1 size=$2 - # write the string into the raw marker - make_str $id $size > trace_marker_raw + str=`make_str $id $size` + len=`printf "$str" | wc -c` + # Pipe through dd to ensure a single atomic write() syscall + # on architectures with 64K pages, where shell's printf builtin + # uses stdio buffering which may split the output into multiple + # writes. + printf "$str" | dd of=trace_marker_raw bs=$len iflag=fullblock } diff --git a/tools/testing/selftests/gpio/.gitignore b/tools/testing/selftests/gpio/.gitignore index ededb077a3a69..16f74de479f17 100644 --- a/tools/testing/selftests/gpio/.gitignore +++ b/tools/testing/selftests/gpio/.gitignore @@ -2,3 +2,4 @@ gpio-mockup-cdev gpio-chip-info gpio-line-name +gpio-cdev-uaf diff --git a/tools/testing/selftests/hid/Makefile b/tools/testing/selftests/hid/Makefile index 50ec9e0406aba..357c6eb5ff5ee 100644 --- a/tools/testing/selftests/hid/Makefile +++ b/tools/testing/selftests/hid/Makefile @@ -24,7 +24,7 @@ CXX ?= $(CROSS_COMPILE)g++ HOSTPKG_CONFIG := pkg-config -CFLAGS += -g -O0 -rdynamic -Wall -Werror -I$(OUTPUT) +CFLAGS += -g -O0 -rdynamic -Wall -Werror -I$(OUTPUT) $(KHDR_INCLUDES) CFLAGS += -I$(OUTPUT)/tools/include LDLIBS += -lelf -lz -lrt -lpthread diff --git a/tools/testing/selftests/hid/hid_bpf.c b/tools/testing/selftests/hid/hid_bpf.c index 1e979fb3542ba..b851339308c21 100644 --- a/tools/testing/selftests/hid/hid_bpf.c +++ b/tools/testing/selftests/hid/hid_bpf.c @@ -86,6 +86,20 @@ static void load_programs(const struct test_program programs[], self->skel = hid__open(); ASSERT_OK_PTR(self->skel) TEARDOWN_LOG("Error while calling hid__open"); + /* + * Disable all struct_ops maps by default so libbpf does not autoload + * programs referenced by maps that are unrelated to the current test. + */ + bpf_object__for_each_map(iter_map, *self->skel->skeleton->obj) { + if (bpf_map__type(iter_map) == BPF_MAP_TYPE_STRUCT_OPS) { + err = bpf_map__set_autocreate(iter_map, false); + ASSERT_OK(err) TH_LOG("can not disable struct_ops map '%s'", + bpf_map__name(iter_map)); + } + + bpf_map__set_autoattach(iter_map, false); + } + for (int i = 0; i < progs_count; i++) { struct bpf_program *prog; struct bpf_map *map; @@ -102,6 +116,10 @@ static void load_programs(const struct test_program programs[], ASSERT_OK_PTR(map) TH_LOG("can not find struct_ops by name '%s'", programs[i].name + 4); + err = bpf_map__set_autocreate(map, true); + ASSERT_OK(err) TH_LOG("can not enable struct_ops map '%s'", + programs[i].name + 4); + /* hid_id is the first field of struct hid_bpf_ops */ ops_hid_id = bpf_map__initial_value(map, NULL); ASSERT_OK_PTR(ops_hid_id) TH_LOG("unable to retrieve struct_ops data"); @@ -109,13 +127,6 @@ static void load_programs(const struct test_program programs[], *ops_hid_id = self->hid.hid_id; } - /* we disable the auto-attach feature of all maps because we - * only want the tested one to be manually attached in the next - * call to bpf_map__attach_struct_ops() - */ - bpf_object__for_each_map(iter_map, *self->skel->skeleton->obj) - bpf_map__set_autoattach(iter_map, false); - err = hid__load(self->skel); ASSERT_OK(err) TH_LOG("hid_skel_load failed: %d", err); @@ -887,6 +898,17 @@ TEST_F(hid_bpf, test_rdesc_fixup) ASSERT_EQ(rpt_desc.value[4], 0x42); } +TEST_F(hid_bpf, test_rdesc_fixup_get_data_overflow) +{ + const struct test_program progs[] = { + { .name = "hid_rdesc_fixup_get_data_overflow" }, + }; + + LOAD_PROGRAMS(progs); + + ASSERT_EQ(self->skel->bss->get_data_overflow_check, 1); +} + static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) { diff --git a/tools/testing/selftests/hid/progs/hid.c b/tools/testing/selftests/hid/progs/hid.c index 5ecc845ef7921..b21fbb13c926f 100644 --- a/tools/testing/selftests/hid/progs/hid.c +++ b/tools/testing/selftests/hid/progs/hid.c @@ -13,6 +13,7 @@ struct attach_prog_args { __u64 callback_check = 52; __u64 callback2_check = 52; +__u64 get_data_overflow_check; SEC("?struct_ops/hid_device_event") int BPF_PROG(hid_first_event, struct hid_bpf_ctx *hid_ctx, enum hid_report_type type) @@ -240,6 +241,20 @@ struct hid_bpf_ops rdesc_fixup = { .hid_rdesc_fixup = (void *)hid_rdesc_fixup, }; +SEC("?struct_ops.s/hid_rdesc_fixup") +int BPF_PROG(hid_rdesc_fixup_get_data_overflow, struct hid_bpf_ctx *hid_ctx) +{ + if (!hid_bpf_get_data(hid_ctx, 2 /* offset */, ~0ULL /* size */)) + get_data_overflow_check = 1; + + return 0; +} + +SEC(".struct_ops.link") +struct hid_bpf_ops rdesc_fixup_get_data_overflow = { + .hid_rdesc_fixup = (void *)hid_rdesc_fixup_get_data_overflow, +}; + SEC("?struct_ops/hid_device_event") int BPF_PROG(hid_test_insert1, struct hid_bpf_ctx *hid_ctx, enum hid_report_type type) { diff --git a/tools/testing/selftests/landlock/audit.h b/tools/testing/selftests/landlock/audit.h index 834005b2b0f09..f45fdef35681a 100644 --- a/tools/testing/selftests/landlock/audit.h +++ b/tools/testing/selftests/landlock/audit.h @@ -45,17 +45,25 @@ struct audit_message { }; }; -static const struct timeval audit_tv_dom_drop = { +static const struct timeval audit_tv_default = { /* - * Because domain deallocation is tied to asynchronous credential - * freeing, receiving such event may take some time. In practice, - * on a small VM, it should not exceed 100k usec, but let's wait up - * to 1 second to be safe. + * Default socket timeout for audit_match_record() callers that expect a + * record to arrive. Asynchronous kauditd delivery can exceed 1 usec + * under heavy debug configs (KASAN, lockdep), where kauditd_thread + * scheduling between audit_log_end() and netlink_unicast() takes longer + * than the previous 1 usec timeout. 1 second is a generous ceiling: on + * the happy path, kauditd delivers within dozens of usec. */ .tv_sec = 1, }; -static const struct timeval audit_tv_default = { +static const struct timeval audit_tv_fast = { + /* + * Fast timeout for paths that expect no record (audit_init() drain, + * audit_count_records(), probes). Causes audit_recv() to return + * -EAGAIN once the socket buffer is empty, naturally terminating the + * read loop. + */ .tv_usec = 1, }; @@ -334,8 +342,13 @@ static int __maybe_unused matches_log_domain_allocated(int audit_fd, pid_t pid, * Matches a domain deallocation record. When expected_domain_id is non-zero, * the pattern includes the specific domain ID so that stale deallocation * records from a previous test (with a different domain ID) are skipped by - * audit_match_record(), and the socket timeout is temporarily increased to - * audit_tv_dom_drop to wait for the asynchronous kworker deallocation. + * audit_match_record(), waiting for the asynchronous kworker deallocation with + * the default patient timeout. + * + * When expected_domain_id is zero, the caller is probing for any dealloc record + * that may or may not arrive. Temporarily lowers the socket timeout to + * audit_tv_fast for this probe so it returns promptly when no record is + * pending; restores audit_tv_default after. */ static int __maybe_unused matches_log_domain_deallocated(int audit_fd, unsigned int num_denials, @@ -361,16 +374,21 @@ matches_log_domain_deallocated(int audit_fd, unsigned int num_denials, if (log_match_len >= sizeof(log_match)) return -E2BIG; - if (expected_domain_id) - setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, - &audit_tv_dom_drop, sizeof(audit_tv_dom_drop)); + if (!expected_domain_id) { + if (setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, + &audit_tv_fast, sizeof(audit_tv_fast))) + return -errno; + } err = audit_match_record(audit_fd, AUDIT_LANDLOCK_DOMAIN, log_match, domain_id); - if (expected_domain_id) - setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, &audit_tv_default, - sizeof(audit_tv_default)); + if (!expected_domain_id) { + if (setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, + &audit_tv_default, sizeof(audit_tv_default)) && + !err) + err = -errno; + } return err; } @@ -381,30 +399,46 @@ struct audit_records { }; /* - * WARNING: Do not assert records.domain == 0 without a preceding - * audit_match_record() call. Domain deallocation records are emitted - * asynchronously from kworker threads and can arrive after the drain in - * audit_init(), corrupting the domain count. A preceding audit_match_record() - * call consumes stale records while scanning, making the assertion safe in - * practice because stale deallocation records arrive before the expected access - * records. + * Counts remaining audit records by type, skipping domain deallocation records. + * Deallocation records are emitted asynchronously from kworker threads after a + * previous test's child has exited, so they can arrive after the drain in + * audit_init() and after the preceding audit_match_record() call. Allocation + * records are emitted synchronously during landlock_log_denial() in the current + * test's syscall context, so only those are counted in records->domain. + * + * Temporarily lowers SO_RCVTIMEO to audit_tv_fast for the read loop: this is a + * "no record expected" path that should terminate on the first -EAGAIN. The + * default patient timeout is restored on exit for subsequent + * audit_match_record() callers. */ static int audit_count_records(int audit_fd, struct audit_records *records) { + static const char dealloc_pattern[] = REGEX_LANDLOCK_PREFIX + " status=deallocated "; struct audit_message msg; - int err; + regex_t dealloc_re; + int ret, err = 0; + + ret = regcomp(&dealloc_re, dealloc_pattern, 0); + if (ret) + return -ENOMEM; records->access = 0; records->domain = 0; + if (setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, &audit_tv_fast, + sizeof(audit_tv_fast))) { + err = -errno; + goto out; + } + do { memset(&msg, 0, sizeof(msg)); err = audit_recv(audit_fd, &msg); if (err) { if (err == -EAGAIN) - return 0; - else - return err; + err = 0; + break; } switch (msg.header.nlmsg_type) { @@ -412,12 +446,24 @@ static int audit_count_records(int audit_fd, struct audit_records *records) records->access++; break; case AUDIT_LANDLOCK_DOMAIN: - records->domain++; + ret = regexec(&dealloc_re, msg.data, 0, NULL, 0); + if (ret == REG_NOMATCH) { + records->domain++; + } else if (ret != 0) { + err = -EIO; + goto out; + } break; } } while (true); - return 0; +out: + if (setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, &audit_tv_default, + sizeof(audit_tv_default)) && + !err) + err = -errno; + regfree(&dealloc_re); + return err; } static int audit_init(void) @@ -436,9 +482,9 @@ static int audit_init(void) if (err) goto err_close; - /* Sets a timeout for negative tests. */ - err = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &audit_tv_default, - sizeof(audit_tv_default)); + /* Uses the fast timeout to drain stale records below. */ + err = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &audit_tv_fast, + sizeof(audit_tv_fast)); if (err) { err = -errno; goto err_close; @@ -454,6 +500,19 @@ static int audit_init(void) while (audit_recv(fd, NULL) == 0) ; + /* + * Restores the default timeout for audit_match_record() callers that + * expect a record to arrive. Paths that expect no record restore the + * fast timeout locally (audit_count_records(), the expected_domain_id + * == 0 probe in matches_log_domain_deallocated()). + */ + err = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &audit_tv_default, + sizeof(audit_tv_default)); + if (err) { + err = -errno; + goto err_close; + } + return fd; err_close: @@ -494,10 +553,9 @@ static int audit_init_filter_exe(struct audit_filter *filter, const char *path) static int audit_cleanup(int audit_fd, struct audit_filter *filter) { struct audit_filter new_filter; + int err = 0; if (audit_fd < 0 || !filter) { - int err; - /* * Simulates audit_init_with_exe_filter() when called from * FIXTURE_TEARDOWN_PARENT(). @@ -508,23 +566,19 @@ static int audit_cleanup(int audit_fd, struct audit_filter *filter) filter = &new_filter; err = audit_init_filter_exe(filter, NULL); - if (err) { - close(audit_fd); - return err; - } + if (err) + goto err_close; } /* Filters might not be in place. */ audit_filter_exe(audit_fd, filter, AUDIT_DEL_RULE); audit_filter_drop(audit_fd, AUDIT_DEL_RULE); - /* - * Because audit_cleanup() might not be called by the test auditd - * process, it might not be possible to explicitly set it. Anyway, - * AUDIT_STATUS_ENABLED will implicitly be set to 0 when the auditd - * process will exit. - */ - return close(audit_fd); + err = audit_set_status(audit_fd, AUDIT_STATUS_ENABLED, 0); + +err_close: + close(audit_fd); + return err; } static int audit_init_with_exe_filter(struct audit_filter *filter) diff --git a/tools/testing/selftests/landlock/audit_test.c b/tools/testing/selftests/landlock/audit_test.c index 93ae5bd0dcce0..bd9f207b36e40 100644 --- a/tools/testing/selftests/landlock/audit_test.c +++ b/tools/testing/selftests/landlock/audit_test.c @@ -730,6 +730,7 @@ TEST_F(audit_flags, signal) } else { EXPECT_EQ(1, records.access); } + EXPECT_EQ(0, records.domain); /* Updates filter rules to match the drop record. */ set_cap(_metadata, CAP_AUDIT_CONTROL); @@ -849,10 +850,8 @@ FIXTURE_SETUP(audit_exec) FIXTURE_TEARDOWN(audit_exec) { set_cap(_metadata, CAP_AUDIT_CONTROL); - EXPECT_EQ(0, audit_filter_exe(self->audit_fd, &self->audit_filter, - AUDIT_DEL_RULE)); + EXPECT_EQ(0, audit_cleanup(self->audit_fd, &self->audit_filter)); clear_cap(_metadata, CAP_AUDIT_CONTROL); - EXPECT_EQ(0, close(self->audit_fd)); } TEST_F(audit_exec, signal_and_open) @@ -917,6 +916,7 @@ TEST_F(audit_exec, signal_and_open) /* Tests that there was no denial until now. */ EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); EXPECT_EQ(0, records.access); + EXPECT_EQ(0, records.domain); /* * Wait for the child to do a first denied action by layer1 and diff --git a/tools/testing/selftests/landlock/net_test.c b/tools/testing/selftests/landlock/net_test.c index 4c528154ea92b..0c256e7c86752 100644 --- a/tools/testing/selftests/landlock/net_test.c +++ b/tools/testing/selftests/landlock/net_test.c @@ -2026,4 +2026,66 @@ TEST_F(audit, connect) EXPECT_EQ(0, close(sock_fd)); } +static int matches_log_tcp_bound(int audit_fd, const char *const addr, + __u16 lport, __u16 dport) +{ + static const char log_template[] = REGEX_LANDLOCK_PREFIX + " blockers=net\\.connect_tcp laddr=%s lport=%u daddr=%s dest=%u$"; + /* Slack for two addresses and two port numbers. */ + char log_match[sizeof(log_template) + 40]; + int log_match_len; + + log_match_len = snprintf(log_match, sizeof(log_match), log_template, + addr, lport, addr, dport); + if (log_match_len > sizeof(log_match)) + return -E2BIG; + + return audit_match_record(audit_fd, AUDIT_LANDLOCK_ACCESS, log_match, + NULL); +} + +/* + * After a bind() to an allowed port, a denied connect must report laddr/lport + * from the bound socket (made available through audit_net.sk) in addition to + * the connect sockaddr's daddr/dest. + */ +TEST_F(audit, connect_tcp_bound) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr rule_bind = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv0.port, + }; + struct service_fixture srv_remote; + struct audit_records records; + int ruleset_fd, sock_fd; + + /* Uses a second port as the denied connect target. */ + ASSERT_EQ(0, set_service(&srv_remote, variant->prot, 1)); + + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &rule_bind, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + + sock_fd = socket_variant(&self->srv0); + ASSERT_LE(0, sock_fd); + EXPECT_EQ(0, bind_variant(sock_fd, &self->srv0)); + EXPECT_EQ(-EACCES, connect_variant(sock_fd, &srv_remote)); + EXPECT_EQ(0, matches_log_tcp_bound(self->audit_fd, variant->addr, + self->srv0.port, srv_remote.port)); + + EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); + EXPECT_EQ(0, records.access); + EXPECT_EQ(1, records.domain); + + EXPECT_EQ(0, close(sock_fd)); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/landlock/ptrace_test.c b/tools/testing/selftests/landlock/ptrace_test.c index 1b6c8b53bf33a..4f64c90583cd6 100644 --- a/tools/testing/selftests/landlock/ptrace_test.c +++ b/tools/testing/selftests/landlock/ptrace_test.c @@ -342,6 +342,7 @@ TEST_F(audit, trace) /* Makes sure there is no superfluous logged records. */ EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); EXPECT_EQ(0, records.access); + EXPECT_EQ(0, records.domain); yama_ptrace_scope = get_yama_ptrace_scope(); ASSERT_LE(0, yama_ptrace_scope); diff --git a/tools/testing/selftests/landlock/scoped_abstract_unix_test.c b/tools/testing/selftests/landlock/scoped_abstract_unix_test.c index c47491d2d1c14..72f97648d4a7d 100644 --- a/tools/testing/selftests/landlock/scoped_abstract_unix_test.c +++ b/tools/testing/selftests/landlock/scoped_abstract_unix_test.c @@ -312,6 +312,7 @@ TEST_F(scoped_audit, connect_to_child) /* Makes sure there is no superfluous logged records. */ EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); EXPECT_EQ(0, records.access); + EXPECT_EQ(0, records.domain); ASSERT_EQ(0, pipe2(pipe_child, O_CLOEXEC)); ASSERT_EQ(0, pipe2(pipe_parent, O_CLOEXEC)); diff --git a/tools/testing/selftests/landlock/scoped_signal_test.c b/tools/testing/selftests/landlock/scoped_signal_test.c index d8bf33417619f..2d37d0c06c069 100644 --- a/tools/testing/selftests/landlock/scoped_signal_test.c +++ b/tools/testing/selftests/landlock/scoped_signal_test.c @@ -249,12 +249,12 @@ TEST_F(scoped_domains, check_access_signal) _metadata->exit_code = KSFT_FAIL; } -enum thread_return { - THREAD_INVALID = 0, - THREAD_SUCCESS = 1, - THREAD_ERROR = 2, - THREAD_TEST_FAILED = 3, -}; +/* clang-format off */ +#define THREAD_INVALID ((void *)0) +#define THREAD_SUCCESS ((void *)1) +#define THREAD_ERROR ((void *)2) +#define THREAD_TEST_FAILED ((void *)3) +/* clang-format on */ static void *thread_sync(void *arg) { @@ -262,15 +262,15 @@ static void *thread_sync(void *arg) char buf; if (read(pipe_read, &buf, 1) != 1) - return (void *)THREAD_ERROR; + return THREAD_ERROR; - return (void *)THREAD_SUCCESS; + return THREAD_SUCCESS; } TEST(signal_scoping_thread_before) { pthread_t no_sandbox_thread; - enum thread_return ret = THREAD_INVALID; + void *ret = THREAD_INVALID; int thread_pipe[2]; drop_caps(_metadata); @@ -285,7 +285,7 @@ TEST(signal_scoping_thread_before) EXPECT_EQ(0, pthread_kill(no_sandbox_thread, 0)); EXPECT_EQ(1, write(thread_pipe[1], ".", 1)); - EXPECT_EQ(0, pthread_join(no_sandbox_thread, (void **)&ret)); + EXPECT_EQ(0, pthread_join(no_sandbox_thread, &ret)); EXPECT_EQ(THREAD_SUCCESS, ret); EXPECT_EQ(0, close(thread_pipe[0])); @@ -295,7 +295,7 @@ TEST(signal_scoping_thread_before) TEST(signal_scoping_thread_after) { pthread_t scoped_thread; - enum thread_return ret = THREAD_INVALID; + void *ret = THREAD_INVALID; int thread_pipe[2]; drop_caps(_metadata); @@ -310,7 +310,7 @@ TEST(signal_scoping_thread_after) EXPECT_EQ(0, pthread_kill(scoped_thread, 0)); EXPECT_EQ(1, write(thread_pipe[1], ".", 1)); - EXPECT_EQ(0, pthread_join(scoped_thread, (void **)&ret)); + EXPECT_EQ(0, pthread_join(scoped_thread, &ret)); EXPECT_EQ(THREAD_SUCCESS, ret); EXPECT_EQ(0, close(thread_pipe[0])); @@ -327,20 +327,20 @@ void *thread_setuid(void *ptr) char buf; if (read(arg->pipe_read, &buf, 1) != 1) - return (void *)THREAD_ERROR; + return THREAD_ERROR; /* libc's setuid() should update all thread's credentials. */ if (getuid() != arg->new_uid) - return (void *)THREAD_TEST_FAILED; + return THREAD_TEST_FAILED; - return (void *)THREAD_SUCCESS; + return THREAD_SUCCESS; } TEST(signal_scoping_thread_setuid) { struct thread_setuid_args arg; pthread_t no_sandbox_thread; - enum thread_return ret = THREAD_INVALID; + void *ret = THREAD_INVALID; int pipe_parent[2]; int prev_uid; @@ -367,7 +367,7 @@ TEST(signal_scoping_thread_setuid) EXPECT_EQ(arg.new_uid, getuid()); EXPECT_EQ(1, write(pipe_parent[1], ".", 1)); - EXPECT_EQ(0, pthread_join(no_sandbox_thread, (void **)&ret)); + EXPECT_EQ(0, pthread_join(no_sandbox_thread, &ret)); EXPECT_EQ(THREAD_SUCCESS, ret); clear_cap(_metadata, CAP_SETUID); @@ -400,6 +400,24 @@ static int setup_signal_handler(int signal) return sigaction(SIGURG, &sa, NULL); } +/* + * MSG_OOB might be disabled in the kernel via the CONFIG_AF_UNIX_OOB + * switch, so this function can be used for probing for its availability. + */ +static bool has_af_unix_oob(void) +{ + bool available = false; + int sp[2]; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) == 0) { + available = (send(sp[0], ".", 1, MSG_OOB) == 1); + close(sp[0]); + close(sp[1]); + } + + return available; +} + /* clang-format off */ FIXTURE(fown) {}; /* clang-format on */ @@ -462,6 +480,9 @@ TEST_F(fown, sigurg_socket) int pipe_parent[2], pipe_child[2]; pid_t child; + if (!has_af_unix_oob()) + SKIP(return, "CONFIG_AF_UNIX_OOB / MSG_OOB not available"); + memset(&server_address, 0, sizeof(server_address)); set_unix_address(&server_address, 0); @@ -559,4 +580,186 @@ TEST_F(fown, sigurg_socket) _metadata->exit_code = KSFT_FAIL; } +/* + * Checks that LANDLOCK_SCOPE_SIGNAL is enforced on the asynchronous SIGIO + * delivery path (fcntl(F_SETOWN)) when the file owner is a process group. + * + * A sandboxed process sitting at the head of its process group's PID hlist (the + * default position right after fork()) used to escape the fcntl(F_SETOWN, + * -pgrp) domain recording: pid_task(pgrp, PIDTYPE_PGID) resolved to the process + * itself, so the same-thread-group exemption skipped recording its Landlock + * domain. At SIGIO time that domain was then unset and the signal fanned out + * to every group member, including non-sandboxed processes outside the domain. + */ +TEST(sigio_to_pgid_members) +{ + int trigger[2], sync_child[2]; + char buf; + pid_t child; + int status, i; + + drop_caps(_metadata); + + /* + * Isolates the test in its own process group so the SIGIO fan-out stays + * bounded to this parent and the child forked below. + */ + ASSERT_EQ(0, setpgid(0, 0)); + + /* The non-sandboxed parent is the protected (out-of-domain) target. */ + ASSERT_EQ(0, setup_signal_handler(SIGURG)); + signal_received = 0; + + ASSERT_EQ(0, pipe2(trigger, O_CLOEXEC)); + ASSERT_EQ(0, pipe2(sync_child, O_CLOEXEC)); + + child = fork(); + ASSERT_LE(0, child); + if (child == 0) { + /* + * The child inherits the parent's new process group and, just + * attached with hlist_add_head_rcu(), is now the head of the + * pgid hlist: this is the case that used to skip the recording. + */ + EXPECT_EQ(0, close(sync_child[0])); + + /* In-domain positive control: the child must be signaled. */ + ASSERT_EQ(0, setup_signal_handler(SIGURG)); + signal_received = 0; + + create_scoped_domain(_metadata, LANDLOCK_SCOPE_SIGNAL); + + /* Owns the SIGIO source for the whole process group. */ + ASSERT_EQ(0, fcntl(trigger[0], F_SETSIG, SIGURG)); + ASSERT_EQ(0, fcntl(trigger[0], F_SETOWN, -getpgrp())); + ASSERT_EQ(0, fcntl(trigger[0], F_SETFL, O_ASYNC)); + + /* Fans SIGURG out to every member of the process group. */ + ASSERT_EQ(1, write(trigger[1], ".", 1)); + + /* + * The sandboxed child is in its own domain and must always be + * signaled: this proves the SIGIO actually fired. + */ + for (i = 0; i < 1000 && !signal_received; i++) + usleep(1000); + EXPECT_EQ(1, signal_received); + + ASSERT_EQ(1, write(sync_child[1], ".", 1)); + EXPECT_EQ(0, close(sync_child[1])); + + _exit(_metadata->exit_code); + return; + } + EXPECT_EQ(0, close(sync_child[1])); + EXPECT_EQ(0, close(trigger[0])); + EXPECT_EQ(0, close(trigger[1])); + + /* Waits for the child to generate the SIGIO. */ + ASSERT_EQ(1, read(sync_child[0], &buf, 1)); + EXPECT_EQ(0, close(sync_child[0])); + + /* Lets a delivered-but-pending signal run our handler, if any. */ + for (i = 0; i < 100 && !signal_received; i++) + usleep(1000); + + /* + * SCOPE_SIGNAL must block the fan-out to this non-sandboxed parent, + * which is outside the child's Landlock domain. Before the fix the + * parent was signaled here. + */ + EXPECT_EQ(0, signal_received); + + ASSERT_EQ(child, waitpid(child, &status, 0)); + if (WIFSIGNALED(status) || !WIFEXITED(status) || + WEXITSTATUS(status) != EXIT_SUCCESS) + _metadata->exit_code = KSFT_FAIL; +} + +static void *thread_setown_scoped(void *arg) +{ + const int fd = *(int *)arg; + int ruleset_fd; + const struct landlock_ruleset_attr ruleset_attr = { + .scoped = LANDLOCK_SCOPE_SIGNAL, + }; + + /* Sandboxes only this non-leader thread (no thread syncing). */ + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + if (ruleset_fd < 0) + return THREAD_ERROR; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || + landlock_restrict_self(ruleset_fd, 0)) { + close(ruleset_fd); + return THREAD_ERROR; + } + close(ruleset_fd); + + /* Makes this process group own the SIGIO source. */ + if (fcntl(fd, F_SETSIG, SIGURG) || fcntl(fd, F_SETOWN, -getpgrp()) || + fcntl(fd, F_SETFL, O_ASYNC)) + return THREAD_ERROR; + + return THREAD_SUCCESS; +} + +/* + * Checks that the SIGIO fan-out is still delivered to the file owner's own + * process when fcntl(F_SETOWN, -pgrp) was issued from a sandboxed non-leader + * thread. + * + * The Landlock domain is recorded for a process-group owner (so out-of-domain + * members stay blocked, see sigio_to_pgid_members), but the kernel signals a + * process group through its members' thread-group leaders. Here the leader is + * not sandboxed and thus has a different domain than the registering thread, so + * the registration-time check cannot tell that it belongs to the owner's own + * process. hook_file_send_sigiotask() must recognize it through the recorded + * thread group and allow the delivery, matching the same-process guarantee of + * commit 18eb75f3af40. Without that exemption the leader is wrongly denied and + * never signaled. + */ +TEST(sigio_to_pgid_self) +{ + int trigger[2]; + pthread_t thread; + void *ret = THREAD_INVALID; + int i; + + drop_caps(_metadata); + + /* Bounds the SIGIO fan-out to this process. */ + ASSERT_EQ(0, setpgid(0, 0)); + + /* The non-sandboxed thread-group leader is the SIGIO target. */ + ASSERT_EQ(0, setup_signal_handler(SIGURG)); + signal_received = 0; + + ASSERT_EQ(0, pipe2(trigger, O_CLOEXEC)); + + /* + * Registers the process-group fowner from a sibling thread that + * sandboxes only itself, so its domain differs from the leader's. + */ + ASSERT_EQ(0, pthread_create(&thread, NULL, thread_setown_scoped, + &trigger[0])); + ASSERT_EQ(0, pthread_join(thread, &ret)); + ASSERT_EQ(THREAD_SUCCESS, ret); + + /* Fans SIGURG out to the process group. */ + ASSERT_EQ(1, write(trigger[1], ".", 1)); + + for (i = 0; i < 1000 && !signal_received; i++) + usleep(1000); + + /* + * Same-process delivery must always be allowed, even though the owner + * was registered from a sandboxed sibling thread. + */ + EXPECT_EQ(1, signal_received); + + EXPECT_EQ(0, close(trigger[0])); + EXPECT_EQ(0, close(trigger[1])); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/liveupdate/liveupdate.c b/tools/testing/selftests/liveupdate/liveupdate.c index 37c808fbe1e92..90268d86684f5 100644 --- a/tools/testing/selftests/liveupdate/liveupdate.c +++ b/tools/testing/selftests/liveupdate/liveupdate.c @@ -386,4 +386,46 @@ TEST_F(liveupdate_device, prevent_double_preservation) ASSERT_EQ(close(session_fd2), 0); } +/* + * Test Case: Create Session with No Null Termination + * + * Verifies that filling the entire 64-byte name field with non-null characters + * (no '\0' terminator) is rejected by the kernel with EINVAL. + */ +TEST_F(liveupdate_device, create_session_no_null_termination) +{ + struct liveupdate_ioctl_create_session args = {}; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + /* Fill entire name field with 'X', no null terminator */ + args.size = sizeof(args); + memset(args.name, 'X', sizeof(args.name)); + + EXPECT_LT(ioctl(self->fd1, LIVEUPDATE_IOCTL_CREATE_SESSION, &args), 0); + EXPECT_EQ(errno, EINVAL); +} + +/* + * Test Case: Create Session with Empty Name + * + * Verifies that creating a session with an empty string name fails + * with EINVAL. + */ +TEST_F(liveupdate_device, create_session_empty_name) +{ + int session_fd; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, ""); + EXPECT_EQ(session_fd, -EINVAL); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh index 44f4e703deb9b..a1cfd3a349db6 100755 --- a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh +++ b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh @@ -17,6 +17,7 @@ if ! command -v killall >/dev/null 2>&1; then fi nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) +trap 'echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages' EXIT INT TERM fault_limit_file=limit_in_bytes reservation_limit_file=rsvd.limit_in_bytes @@ -70,7 +71,6 @@ function cleanup() { if [[ -e $cgroup_path/hugetlb_cgroup_test2 ]]; then rmdir $cgroup_path/hugetlb_cgroup_test2 fi - echo 0 >/proc/sys/vm/nr_hugepages echo CLEANUP DONE } @@ -94,6 +94,15 @@ function get_machine_hugepage_size() { } MB=$(get_machine_hugepage_size) +if (( MB >= 1024 )); then + # For 1GB hugepages + UNIT="GB" + MB_DISPLAY=$((MB / 1024)) +else + # For 2MB hugepages + UNIT="MB" + MB_DISPLAY=$MB +fi function setup_cgroup() { local name="$1" @@ -103,11 +112,12 @@ function setup_cgroup() { mkdir $cgroup_path/$name echo writing cgroup limit: "$cgroup_limit" - echo "$cgroup_limit" >$cgroup_path/$name/hugetlb.${MB}MB.$fault_limit_file + echo "$cgroup_limit" > \ + $cgroup_path/$name/hugetlb.${MB_DISPLAY}${UNIT}.$fault_limit_file echo writing reservation limit: "$reservation_limit" echo "$reservation_limit" > \ - $cgroup_path/$name/hugetlb.${MB}MB.$reservation_limit_file + $cgroup_path/$name/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_limit_file if [ -e "$cgroup_path/$name/cpuset.cpus" ]; then echo 0 >$cgroup_path/$name/cpuset.cpus @@ -142,7 +152,7 @@ function wait_for_file_value() { function wait_for_hugetlb_memory_to_get_depleted() { local cgroup="$1" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file" wait_for_file_value "$path" "0" } @@ -150,7 +160,7 @@ function wait_for_hugetlb_memory_to_get_depleted() { function wait_for_hugetlb_memory_to_get_reserved() { local cgroup="$1" local size="$2" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file" wait_for_file_value "$path" "$size" } @@ -158,7 +168,7 @@ function wait_for_hugetlb_memory_to_get_reserved() { function wait_for_hugetlb_memory_to_get_written() { local cgroup="$1" local size="$2" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file" wait_for_file_value "$path" "$size" } @@ -180,8 +190,8 @@ function write_hugetlbfs_and_get_usage() { hugetlb_difference=0 reserved_difference=0 - local hugetlb_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file - local reserved_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file + local hugetlb_usage=$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local reserved_usage=$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file local hugetlb_before=$(cat $hugetlb_usage) local reserved_before=$(cat $reserved_usage) @@ -312,8 +322,10 @@ function run_test() { cleanup_hugetlb_memory "hugetlb_cgroup_test" - local final_hugetlb=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$fault_usage_file) - local final_reservation=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$reservation_usage_file) + local final_hugetlb=$(cat \ + $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file) + local final_reservation=$(cat \ + $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file) echo $hugetlb_difference echo $reserved_difference @@ -369,10 +381,14 @@ function run_multiple_cgroup_test() { reservation_failed1=$reservation_failed oom_killed1=$oom_killed - local cgroup1_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$fault_usage_file - local cgroup1_reservation_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$reservation_usage_file - local cgroup2_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$fault_usage_file - local cgroup2_reservation_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$reservation_usage_file + local cgroup1_hugetlb_usage=\ + $cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local cgroup1_reservation_usage=\ + $cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file + local cgroup2_hugetlb_usage=\ + $cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local cgroup2_reservation_usage=\ + $cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file local usage_before_second_write=$(cat $cgroup1_hugetlb_usage) local reservation_usage_before_second_write=$(cat $cgroup1_reservation_usage) @@ -599,4 +615,3 @@ if [[ $do_umount ]]; then rmdir $cgroup_path fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 30209c40b6977..e1610e2fbdf64 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -261,6 +261,9 @@ int main(int argc, char **argv) mem_fragmentable_MB -= MAP_SIZE_MB; } + /* Unmap every other entry in the list to create fragmentation with + * locked pages before invoking check_compaction(). + */ for (entry = list; entry != NULL; entry = entry->next) { munmap(entry->map, MAP_SIZE); if (!entry->next) diff --git a/tools/testing/selftests/mm/droppable.c b/tools/testing/selftests/mm/droppable.c index 44940f75c461d..5f74827a714db 100644 --- a/tools/testing/selftests/mm/droppable.c +++ b/tools/testing/selftests/mm/droppable.c @@ -17,10 +17,10 @@ int main(int argc, char *argv[]) { - size_t alloc_size = 134217728; - size_t page_size = getpagesize(); + const size_t alloc_size = 2 * 1024 * 1024; + int retry_count = 10; + bool dropped; void *alloc; - pid_t child; ksft_print_header(); ksft_set_plan(1); @@ -28,26 +28,32 @@ int main(int argc, char *argv[]) alloc = mmap(0, alloc_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_DROPPABLE, -1, 0); assert(alloc != MAP_FAILED); memset(alloc, 'A', alloc_size); - for (size_t i = 0; i < alloc_size; i += page_size) - assert(*(uint8_t *)(alloc + i)); - - child = fork(); - assert(child >= 0); - if (!child) { - for (;;) - *(char *)malloc(page_size) = 'B'; - } - for (bool done = false; !done;) { - for (size_t i = 0; i < alloc_size; i += page_size) { - if (!*(uint8_t *)(alloc + i)) { - done = true; - break; + while (retry_count--) { + if (madvise(alloc, alloc_size, MADV_PAGEOUT)) { + if (errno == EINVAL) { + ksft_test_result_skip("madvise(MADV_PAGEOUT) not supported\n"); + exit(KSFT_SKIP); } + ksft_test_result_fail("madvise(MADV_PAGEOUT) error: %s\n", strerror(errno)); + exit(KSFT_FAIL); } + + dropped = memchr(alloc, 'A', alloc_size) == NULL; + + /* + * Speculative reference can temporarily prevent some + * pages from getting dropped. So sleep and retry. + * + * If a page is not droppable for 10s, something + * is seriously messed up and we want to fail. + */ + if (dropped) + break; + sleep(1); } - kill(child, SIGTERM); - ksft_test_result_pass("MAP_DROPPABLE: PASS\n"); - exit(KSFT_PASS); + ksft_test_result(dropped, "madvise(MADV_PAGEOUT) behavior\n"); + + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 77fb4c5d871bb..f16da3267d486 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1599,8 +1600,8 @@ TEST_F(hmm2, snapshot) } /* - * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that - * should be mapped by a large page table entry. + * Test the hmm_range_fault() handling of large pages (PMD or PUD) + * that should be mapped by a large page table entry. */ TEST_F(hmm, compound) { @@ -1610,6 +1611,7 @@ TEST_F(hmm, compound) unsigned long default_hsize = default_huge_page_size(); int *ptr; unsigned char *m; + unsigned char prot; int ret; unsigned long i; @@ -1646,11 +1648,20 @@ TEST_F(hmm, compound) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - /* Check what the device saw. */ + /* + * Check what the device saw. The region is backed by a single huge + * page that the device reports either at PMD or at PUD level depending + * on the configured default hugepage size. Determine that level from + * the first page and require every page in the range to match it + * exactly, so that a fragmented mapping mixing levels (or a missing + * large-page bit) is still caught and reported with its actual value. + */ m = buffer->mirror; + prot = HMM_DMIRROR_PROT_WRITE | + ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : + HMM_DMIRROR_PROT_PMD); for (i = 0; i < npages; ++i) - ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE | - HMM_DMIRROR_PROT_PMD); + ASSERT_EQ(m[i], prot); /* Make the region read-only. */ ret = mprotect(buffer->ptr, size, PROT_READ); @@ -1661,11 +1672,17 @@ TEST_F(hmm, compound) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - /* Check what the device saw. */ + /* + * Check what the device saw after mprotect(PROT_READ). Same + * approach as above: determine the mapping level from the first + * page and require every page to match it exactly. + */ m = buffer->mirror; + prot = HMM_DMIRROR_PROT_READ | + ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : + HMM_DMIRROR_PROT_PMD); for (i = 0; i < npages; ++i) - ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | - HMM_DMIRROR_PROT_PMD); + ASSERT_EQ(m[i], prot); munmap(buffer->ptr, buffer->size); buffer->ptr = NULL; @@ -1865,6 +1882,8 @@ TEST_F(hmm, exclusive_cow) unsigned long i; int *ptr; int ret; + pid_t pid; + int status; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); @@ -1893,14 +1912,37 @@ TEST_F(hmm, exclusive_cow) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - fork(); + pid = fork(); + if (pid == -1) + ASSERT_EQ(pid, 0); - /* Fault pages back to system memory and check them. */ + if (pid == 0) { + /* + * Child verifies COW independently, then _exit(0)s so it does + * not run the test teardown. A failed ASSERT_* here makes the + * harness abort() the child, so the parent sees + * !WIFEXITED(status) below and fails in turn. + */ + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i]++, i); + + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i], i + 1); + + _exit(0); + } + + /* Parent: also increment to verify COW works for both processes. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i]++, i); for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) - ASSERT_EQ(ptr[i], i+1); + ASSERT_EQ(ptr[i], i + 1); + + /* Parent: wait for child and then free the buffer. */ + ASSERT_EQ(waitpid(pid, &status, 0), pid); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(WEXITSTATUS(status), 0); hmm_buffer_free(buffer); } @@ -2332,12 +2374,21 @@ TEST_F(hmm, migrate_partial_unmap_fault) struct hmm_buffer *buffer; unsigned long npages; unsigned long size = read_pmd_pagesize(); + unsigned long unmap_size; + unsigned long offsets[3]; unsigned long i; void *old_ptr; void *map; int *ptr; int ret, j, use_thp; - int offsets[] = { 0, 512 * ONEKB, ONEMEG }; + + if (!size) + size = TWOMEG; + + unmap_size = size / 2; + offsets[0] = 0; + offsets[1] = size / 4; + offsets[2] = size / 2; for (use_thp = 0; use_thp < 2; ++use_thp) { for (j = 0; j < ARRAY_SIZE(offsets); ++j) { @@ -2379,12 +2430,12 @@ TEST_F(hmm, migrate_partial_unmap_fault) for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); - munmap(buffer->ptr + offsets[j], ONEMEG); + munmap(buffer->ptr + offsets[j], unmap_size); /* Fault pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) if (i * sizeof(int) < offsets[j] || - i * sizeof(int) >= offsets[j] + ONEMEG) + i * sizeof(int) >= offsets[j] + unmap_size) ASSERT_EQ(ptr[i], i); buffer->ptr = old_ptr; @@ -2398,12 +2449,19 @@ TEST_F(hmm, migrate_remap_fault) struct hmm_buffer *buffer; unsigned long npages; unsigned long size = read_pmd_pagesize(); + unsigned long offsets[3]; unsigned long i; void *old_ptr, *new_ptr = NULL; void *map; int *ptr; int ret, j, use_thp, dont_unmap, before; - int offsets[] = { 0, 512 * ONEKB, ONEMEG }; + + if (!size) + size = TWOMEG; + + offsets[0] = 0; + offsets[1] = size / 4; + offsets[2] = size / 2; for (before = 0; before < 2; ++before) { for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) { @@ -2806,38 +2864,45 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) { struct benchmark_results thp_results, regular_results; - size_t thp_size = 2 * 1024 * 1024; /* 2MB - typical THP size */ + size_t thp_size = read_pmd_pagesize(); int iterations = 5; + if (!thp_size) + thp_size = TWOMEG; + printf("\nHMM THP Migration Benchmark\n"); printf("---------------------------\n"); printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE)); /* Test different buffer sizes */ size_t test_sizes[] = { - thp_size / 4, /* 512KB - smaller than THP */ - thp_size / 2, /* 1MB - half THP */ - thp_size, /* 2MB - single THP */ - thp_size * 2, /* 4MB - two THPs */ - thp_size * 4, /* 8MB - four THPs */ - thp_size * 8, /* 16MB - eight THPs */ - thp_size * 128, /* 256MB - one twenty eight THPs */ + thp_size / 4, /* quarter THP */ + thp_size / 2, /* half THP */ + thp_size, /* single THP */ + thp_size * 2, /* two THPs */ + thp_size * 4, /* four THPs */ + thp_size * 8, /* eight THPs */ + thp_size * 128, /* one twenty eight THPs */ }; static const char *const test_names[] = { - "Small Buffer (512KB)", - "Half THP Size (1MB)", - "Single THP Size (2MB)", - "Two THP Size (4MB)", - "Four THP Size (8MB)", - "Eight THP Size (16MB)", - "One twenty eight THP Size (256MB)" + "Small Buffer", + "Half THP Size", + "Single THP Size", + "Two THP Size", + "Four THP Size", + "Eight THP Size", + "One twenty eight THP Size" }; int num_tests = ARRAY_SIZE(test_sizes); /* Run all tests */ for (int i = 0; i < num_tests; i++) { + /* Skip test sizes exceeding INT_MAX to avoid overflow */ + if (test_sizes[i] > INT_MAX) + break; + /* Test with THP */ ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i], iterations, &thp_results), 0); diff --git a/tools/testing/selftests/mm/hugepage-mremap.c b/tools/testing/selftests/mm/hugepage-mremap.c index b8f7d92e5a351..d8be8329868f2 100644 --- a/tools/testing/selftests/mm/hugepage-mremap.c +++ b/tools/testing/selftests/mm/hugepage-mremap.c @@ -31,7 +31,7 @@ #define MB_TO_BYTES(x) (x * 1024 * 1024) #define PROTECTION (PROT_READ | PROT_WRITE | PROT_EXEC) -#define FLAGS (MAP_SHARED | MAP_ANONYMOUS) +#define FLAGS (MAP_HUGETLB | MAP_SHARED) static void check_bytes(char *addr) { @@ -85,25 +85,14 @@ static void register_region_with_uffd(char *addr, size_t len) if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) ksft_exit_fail_msg("ioctl-UFFDIO_API: %s\n", strerror(errno)); - /* Create a private anonymous mapping. The memory will be - * demand-zero paged--that is, not yet allocated. When we - * actually touch the memory, it will be allocated via - * the userfaultfd. - */ - - addr = mmap(NULL, len, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); - - ksft_print_msg("Address returned by mmap() = %p\n", addr); - - /* Register the memory range of the mapping we just created for - * handling by the userfaultfd object. In mode, we request to track - * missing pages (i.e., pages that have not yet been faulted in). + /* Register the passed memory range for handling by the userfaultfd object. + * In mode, we request to track missing pages + * (i.e., pages that have not yet been faulted in). */ if (uffd_register(uffd, addr, len, true, false, false)) ksft_exit_fail_msg("ioctl-UFFDIO_REGISTER: %s\n", strerror(errno)); + + ksft_print_msg("Registered memory at address %p with userfaultfd\n", addr); } int main(int argc, char *argv[]) @@ -132,23 +121,20 @@ int main(int argc, char *argv[]) /* mmap to a PUD aligned address to hopefully trigger pmd sharing. */ unsigned long suggested_addr = 0x7eaa40000000; - void *haddr = mmap((void *)suggested_addr, length, PROTECTION, - MAP_HUGETLB | MAP_SHARED | MAP_POPULATE, fd, 0); + void *haddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map haddr: Returned address is %p\n", haddr); if (haddr == MAP_FAILED) ksft_exit_fail_msg("mmap1: %s\n", strerror(errno)); /* mmap again to a dummy address to hopefully trigger pmd sharing. */ suggested_addr = 0x7daa40000000; - void *daddr = mmap((void *)suggested_addr, length, PROTECTION, - MAP_HUGETLB | MAP_SHARED | MAP_POPULATE, fd, 0); + void *daddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map daddr: Returned address is %p\n", daddr); if (daddr == MAP_FAILED) ksft_exit_fail_msg("mmap3: %s\n", strerror(errno)); suggested_addr = 0x7faa40000000; - void *vaddr = - mmap((void *)suggested_addr, length, PROTECTION, FLAGS, -1, 0); + void *vaddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map vaddr: Returned address is %p\n", vaddr); if (vaddr == MAP_FAILED) ksft_exit_fail_msg("mmap2: %s\n", strerror(errno)); diff --git a/tools/testing/selftests/mm/hugetlb_reparenting_test.sh b/tools/testing/selftests/mm/hugetlb_reparenting_test.sh index 0dd31892ff679..95f517c3bd167 100755 --- a/tools/testing/selftests/mm/hugetlb_reparenting_test.sh +++ b/tools/testing/selftests/mm/hugetlb_reparenting_test.sh @@ -12,6 +12,8 @@ if [[ $(id -u) -ne 0 ]]; then fi nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) +trap 'echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages' EXIT INT TERM + usage_file=usage_in_bytes if [[ "$1" == "-cgroup-v2" ]]; then @@ -46,6 +48,13 @@ function get_machine_hugepage_size() { } MB=$(get_machine_hugepage_size) +if (( MB >= 1024 )); then + UNIT="GB" + MB_DISPLAY=$((MB / 1024)) +else + UNIT="MB" + MB_DISPLAY=$MB +fi function cleanup() { echo cleanup @@ -56,7 +65,6 @@ function cleanup() { rmdir "$CGROUP_ROOT"/a/b 2>/dev/null rmdir "$CGROUP_ROOT"/a 2>/dev/null rmdir "$CGROUP_ROOT"/test1 2>/dev/null - echo $nr_hugepgs >/proc/sys/vm/nr_hugepages set -e } @@ -87,6 +95,7 @@ function assert_with_retry() { if [[ $elapsed -ge $timeout ]]; then echo "actual = $((${actual%% *} / 1024 / 1024)) MB" echo "expected = $((${expected%% *} / 1024 / 1024)) MB" + echo FAIL cleanup exit 1 fi @@ -96,22 +105,19 @@ function assert_with_retry() { } function assert_state() { - local expected_a="$1" - local expected_a_hugetlb="$2" - local expected_b="" + local expected_a_hugetlb="$1" local expected_b_hugetlb="" - if [ ! -z ${3:-} ] && [ ! -z ${4:-} ]; then - expected_b="$3" - expected_b_hugetlb="$4" + if [ ! -z ${2:-} ]; then + expected_b_hugetlb="$2" fi - assert_with_retry "$CGROUP_ROOT/a/memory.$usage_file" "$expected_a" - assert_with_retry "$CGROUP_ROOT/a/hugetlb.${MB}MB.$usage_file" "$expected_a_hugetlb" + assert_with_retry \ + "$CGROUP_ROOT/a/hugetlb.${MB_DISPLAY}${UNIT}.$usage_file" "$expected_a_hugetlb" - if [[ -n "$expected_b" && -n "$expected_b_hugetlb" ]]; then - assert_with_retry "$CGROUP_ROOT/a/b/memory.$usage_file" "$expected_b" - assert_with_retry "$CGROUP_ROOT/a/b/hugetlb.${MB}MB.$usage_file" "$expected_b_hugetlb" + if [[ -n "$expected_b_hugetlb" ]]; then + assert_with_retry \ + "$CGROUP_ROOT/a/b/hugetlb.${MB_DISPLAY}${UNIT}.$usage_file" "$expected_b_hugetlb" fi } @@ -143,18 +149,17 @@ write_hugetlbfs() { local size="$3" if [[ $cgroup2 ]]; then - echo $$ >$CGROUP_ROOT/$cgroup/cgroup.procs + cg_file="$CGROUP_ROOT/$cgroup/cgroup.procs" else echo 0 >$CGROUP_ROOT/$cgroup/cpuset.mems echo 0 >$CGROUP_ROOT/$cgroup/cpuset.cpus - echo $$ >"$CGROUP_ROOT/$cgroup/tasks" - fi - ./write_to_hugetlbfs -p "$path" -s "$size" -m 0 -o - if [[ $cgroup2 ]]; then - echo $$ >$CGROUP_ROOT/cgroup.procs - else - echo $$ >"$CGROUP_ROOT/tasks" + cg_file="$CGROUP_ROOT/$cgroup/tasks" fi + + # Spawn helper to join cgroup before exec to ensure correct cgroup accounting + bash -c 'echo $$ > "$1"; exec ./write_to_hugetlbfs -p "$2" -s "$3" -m 0 -o' _ \ + "$cg_file" "$path" "$size" & pid=$! + wait "$pid" echo } @@ -192,21 +197,21 @@ if [[ ! $cgroup2 ]]; then write_hugetlbfs a "$MNT"/test $size echo Assert memory charged correctly for parent use. - assert_state 0 $size 0 0 + assert_state $size 0 write_hugetlbfs a/b "$MNT"/test2 $size echo Assert memory charged correctly for child use. - assert_state 0 $(($size * 2)) 0 $size + assert_state $(($size * 2)) $size rmdir "$CGROUP_ROOT"/a/b echo Assert memory reparent correctly. - assert_state 0 $(($size * 2)) + assert_state $(($size * 2)) rm -rf "$MNT"/* umount "$MNT" echo Assert memory uncharged correctly. - assert_state 0 0 + assert_state 0 cleanup fi @@ -220,16 +225,16 @@ echo write write_hugetlbfs a/b "$MNT"/test2 $size echo Assert memory charged correctly for child only use. -assert_state 0 $(($size)) 0 $size +assert_state $(($size)) $size rmdir "$CGROUP_ROOT"/a/b echo Assert memory reparent correctly. -assert_state 0 $size +assert_state $size rm -rf "$MNT"/* umount "$MNT" echo Assert memory uncharged correctly. -assert_state 0 0 +assert_state 0 cleanup @@ -240,4 +245,3 @@ if [[ $do_umount ]]; then rm -rf $CGROUP_ROOT fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages diff --git a/tools/testing/selftests/mm/ksft_process_madv.sh b/tools/testing/selftests/mm/ksft_process_madv.sh index 2c3137ae8bc85..edad2d2d888f2 100755 --- a/tools/testing/selftests/mm/ksft_process_madv.sh +++ b/tools/testing/selftests/mm/ksft_process_madv.sh @@ -1,4 +1,4 @@ #!/bin/sh -e # SPDX-License-Identifier: GPL-2.0 -./run_vmtests.sh -t mmap +./run_vmtests.sh -t process_madv diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 7f9428d6062ca..0222a3dda0297 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -1366,7 +1366,7 @@ void *thread_proc(void *mem) ksft_exit_fail_msg("pthread_barrier_wait\n"); for (i = 0; i < access_per_thread; ++i) - __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); + __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST); ret = pthread_barrier_wait(&end_barrier); if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) @@ -1401,15 +1401,15 @@ static void transact_test(int page_size) if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) ksft_exit_fail_msg("pthread_barrier_init\n"); - mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, + mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (mem == MAP_FAILED) ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); - wp_init(mem, 0x1000 * nthreads * pages_per_thread); - wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); + wp_init(mem, page_size * nthreads * pages_per_thread); + wp_addr_range(mem, page_size * nthreads * pages_per_thread); - memset(mem, 0, 0x1000 * nthreads * pages_per_thread); + memset(mem, 0, page_size * nthreads * pages_per_thread); count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); ksft_test_result(count > 0, "%s count %u\n", __func__, count); @@ -1418,7 +1418,7 @@ static void transact_test(int page_size) finish = 0; for (i = 0; i < nthreads; ++i) - pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); + pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread); extra_pages = 0; for (i = 0; i < iter_count; ++i) { diff --git a/tools/testing/selftests/mm/pkey-arm64.h b/tools/testing/selftests/mm/pkey-arm64.h index 8e9685e03c441..c5a78a2f211d5 100644 --- a/tools/testing/selftests/mm/pkey-arm64.h +++ b/tools/testing/selftests/mm/pkey-arm64.h @@ -130,9 +130,10 @@ static inline u64 get_pkey_bits(u64 reg, int pkey) static inline void aarch64_write_signal_pkey(ucontext_t *uctxt, u64 pkey) { struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uctxt); + size_t resv_size = GET_UCP_RESV_SIZE(uctxt); struct poe_context *poe_ctx = (struct poe_context *) get_header(ctx, POE_MAGIC, - sizeof(uctxt->uc_mcontext), NULL); + resv_size, NULL); if (poe_ctx) poe_ctx->por_el0 = pkey; } diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index c17b133a81d24..59e6eada431f3 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -235,32 +235,61 @@ pretty_name() { run_test() { if test_selected ${CATEGORY}; then local skip=0 + local LOADED_HWPOISON_INJECT_MOD=0 # On memory constrainted systems some tests can fail to allocate hugepages. # perform some cleanup before the test for a higher success rate. if [ ${CATEGORY} == "thp" -o ${CATEGORY} == "hugetlb" ]; then - if [ "${HAVE_HUGEPAGES}" = "1" ]; then + mem_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + mem_Mb=$((mem_kb / 1024)) + + if (( $mem_Mb < 256 )); then echo 3 > /proc/sys/vm/drop_caches sleep 2 echo 1 > /proc/sys/vm/compact_memory sleep 2 - else - echo "hugepages not supported" | tap_prefix - skip=1 fi fi + # Ensure hwpoison_inject is available for memory-failure tests + if [ "${CATEGORY}" = "memory-failure" ]; then + # Try to load hwpoison_inject if not present. + HWPOISON_DIR=/sys/kernel/debug/hwpoison/ + if [ ! -d "$HWPOISON_DIR" ]; then + if ! modprobe -n hwpoison_inject > /dev/null 2>&1; then + echo "Module hwpoison_inject not found, skipping..." \ + | tap_prefix + skip=1 + else + modprobe hwpoison_inject > /dev/null 2>&1 + LOADED_HWPOISON_INJECT_MOD=1 + if [ ! -d "$HWPOISON_DIR" ]; then + echo "hwpoison debugfs interface not present" \ + | tap_prefix + skip=1 + fi + fi + fi + + fi + local test=$(pretty_name "$*") local title="running $*" local sep=$(echo -n "$title" | tr "[:graph:][:space:]" -) printf "%s\n%s\n%s\n" "$sep" "$title" "$sep" | tap_prefix - if [ "${skip}" != "1" ]; then + if [ $skip -eq 1 ]; then + local ret=$ksft_skip + else ("$@" 2>&1) | tap_prefix local ret=${PIPESTATUS[0]} - else - local ret=$ksft_skip fi + + # Unload hwpoison_inject if we loaded it + if [ "${LOADED_HWPOISON_INJECT_MOD}" = "1" ]; then + modprobe -r hwpoison_inject > /dev/null 2>&1 + fi + count_total=$(( count_total + 1 )) if [ $ret -eq 0 ]; then count_pass=$(( count_pass + 1 )) @@ -270,7 +299,9 @@ run_test() { count_skip=$(( count_skip + 1 )) echo "[SKIP]" | tap_prefix echo "ok ${count_total} ${test} # SKIP" | tap_output - exitcode=$ksft_skip + if [ $exitcode -eq 0 ]; then + exitcode=$ksft_skip + fi else count_fail=$(( count_fail + 1 )) echo "[FAIL]" | tap_prefix @@ -531,24 +562,7 @@ CATEGORY="page_frag" run_test ./test_page_frag.sh nonaligned CATEGORY="rmap" run_test ./rmap -# Try to load hwpoison_inject if not present. -HWPOISON_DIR=/sys/kernel/debug/hwpoison/ -if [ ! -d "$HWPOISON_DIR" ]; then - if ! modprobe -q -R hwpoison_inject; then - echo "Module hwpoison_inject not found, skipping..." - else - modprobe hwpoison_inject > /dev/null 2>&1 - LOADED_MOD=1 - fi -fi - -if [ -d "$HWPOISON_DIR" ]; then - CATEGORY="memory-failure" run_test ./memory-failure -fi - -if [ -n "${LOADED_MOD}" ]; then - modprobe -r hwpoison_inject > /dev/null 2>&1 -fi +CATEGORY="memory-failure" run_test ./memory-failure if [ "${HAVE_HUGEPAGES}" = 1 ]; then echo "$orig_nr_hugepgs" > /proc/sys/vm/nr_hugepages diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 500d07c4938b1..ee256afecd8fe 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -470,13 +470,18 @@ static void split_file_backed_thp(int order) char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; const char *tmpfs_loc = mkdtemp(tmpfs_template); char testfile[INPUT_MAX]; + unsigned long size = 2 * pmd_pagesize; + char opts[64]; ssize_t num_written, num_read; - char *file_buf1, *file_buf2; + char *file_buf1 = NULL, *file_buf2 = NULL; uint64_t pgoff_start = 0, pgoff_end = 1024; int i; ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); + if (!tmpfs_loc) + ksft_exit_fail_msg("mkdtemp failed\n"); + file_buf1 = (char *)malloc(pmd_pagesize); file_buf2 = (char *)malloc(pmd_pagesize); @@ -489,10 +494,13 @@ static void split_file_backed_thp(int order) file_buf1[i] = (char)i; memset(file_buf2, 0, pmd_pagesize); - status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m"); + snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); + status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); - if (status) - ksft_exit_fail_msg("Unable to create a tmpfs for testing\n"); + if (status) { + ksft_print_msg("Unable to create a tmpfs for testing\n"); + goto out; + } status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); if (status >= INPUT_MAX) { @@ -544,10 +552,13 @@ static void split_file_backed_thp(int order) status = umount(tmpfs_loc); if (status) { - rmdir(tmpfs_loc); - ksft_exit_fail_msg("Unable to umount %s\n", tmpfs_loc); + ksft_print_msg("Unable to umount %s\n", tmpfs_loc); + goto out; } + free(file_buf1); + free(file_buf2); + status = rmdir(tmpfs_loc); if (status) ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); @@ -560,8 +571,10 @@ static void split_file_backed_thp(int order) close(fd); cleanup: umount(tmpfs_loc); - rmdir(tmpfs_loc); out: + free(file_buf1); + free(file_buf2); + rmdir(tmpfs_loc); ksft_exit_fail_msg("Error occurred\n"); } diff --git a/tools/testing/selftests/mm/uffd-stress.c b/tools/testing/selftests/mm/uffd-stress.c index 700fbaa18d44b..dabaade1eaa0d 100644 --- a/tools/testing/selftests/mm/uffd-stress.c +++ b/tools/testing/selftests/mm/uffd-stress.c @@ -491,9 +491,8 @@ int main(int argc, char **argv) gopts->nr_pages_per_cpu = bytes / gopts->page_size / gopts->nr_parallel; if (!gopts->nr_pages_per_cpu) { - _err("pages_per_cpu = 0, cannot test (%lu / %lu / %lu)", - bytes, gopts->page_size, gopts->nr_parallel); - usage(); + ksft_exit_skip("pages_per_cpu = 0, cannot test (%zu / %lu / %lu)\n", + bytes, gopts->page_size, gopts->nr_parallel); } bounces = atoi(argv[3]); diff --git a/tools/testing/selftests/net/broadcast_ether_dst.sh b/tools/testing/selftests/net/broadcast_ether_dst.sh index 334a7eca8a803..cc571f6074297 100755 --- a/tools/testing/selftests/net/broadcast_ether_dst.sh +++ b/tools/testing/selftests/net/broadcast_ether_dst.sh @@ -44,7 +44,7 @@ test_broadcast_ether_dst() { # tcpdump will exit after receiving a single packet # timeout will kill tcpdump if it is still running after 2s timeout 2s ip netns exec "${CLIENT_NS}" \ - tcpdump -i link0 -c 1 -w "${CAPFILE}" icmp &> "${OUTPUT}" & + tcpdump -i link0 -c 1 -w "${CAPFILE}" -Z root icmp &> "${OUTPUT}" & pid=$! slowwait 1 grep -qs "listening" "${OUTPUT}" diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index b3827b43782b0..d46d2cec89e45 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -70,12 +70,33 @@ ksft_exit_status_merge() $ksft_xfail $ksft_pass $ksft_skip $ksft_fail } +timestamp_ms() +{ + local now + local seconds + local nanoseconds + + now=$(date -u +%s:%N) || return + seconds=${now%:*} + nanoseconds=${now#*:} + + if [[ $nanoseconds =~ ^[0-9]+$ ]]; then + nanoseconds=${nanoseconds:0:9} + else + nanoseconds=0 + fi + + echo $((seconds * 1000 + 10#$nanoseconds / 1000000)) +} + loopy_wait() { local sleep_cmd=$1; shift local timeout_ms=$1; shift + local start_time + local current_time - local start_time="$(date -u +%s%3N)" + start_time=$(timestamp_ms) || return while true do local out @@ -84,7 +105,7 @@ loopy_wait() return 0 fi - local current_time="$(date -u +%s%3N)" + current_time=$(timestamp_ms) || return if ((current_time - start_time > timeout_ms)); then echo -n "$out" return 1 diff --git a/tools/testing/selftests/net/netfilter/nft_flowtable.sh b/tools/testing/selftests/net/netfilter/nft_flowtable.sh index 7a34ef468975d..08ad07500e8a7 100755 --- a/tools/testing/selftests/net/netfilter/nft_flowtable.sh +++ b/tools/testing/selftests/net/netfilter/nft_flowtable.sh @@ -592,7 +592,7 @@ ip -net "$nsr1" link set tun0 up ip -net "$nsr1" addr add 192.168.100.1/24 dev tun0 ip netns exec "$nsr1" sysctl net.ipv4.conf.tun0.forwarding=1 > /dev/null -ip -net "$nsr1" link add name tun6 type ip6tnl local fee1:2::1 remote fee1:2::2 +ip -net "$nsr1" link add name tun6 type ip6tnl local fee1:2::1 remote fee1:2::2 encaplimit none ip -net "$nsr1" link set tun6 up ip -net "$nsr1" addr add fee1:3::1/64 dev tun6 nodad @@ -601,7 +601,7 @@ ip -net "$nsr2" link set tun0 up ip -net "$nsr2" addr add 192.168.100.2/24 dev tun0 ip netns exec "$nsr2" sysctl net.ipv4.conf.tun0.forwarding=1 > /dev/null -ip -net "$nsr2" link add name tun6 type ip6tnl local fee1:2::2 remote fee1:2::1 || ret=1 +ip -net "$nsr2" link add name tun6 type ip6tnl local fee1:2::2 remote fee1:2::1 encaplimit none || ret=1 ip -net "$nsr2" link set tun6 up ip -net "$nsr2" addr add fee1:3::2/64 dev tun6 nodad @@ -651,7 +651,7 @@ ip -net "$nsr1" route change default via 192.168.200.2 ip netns exec "$nsr1" sysctl net.ipv4.conf.tun0/10.forwarding=1 > /dev/null ip netns exec "$nsr1" nft -a insert rule inet filter forward 'meta oif tun0.10 accept' -ip -net "$nsr1" link add name tun6.10 type ip6tnl local fee1:4::1 remote fee1:4::2 +ip -net "$nsr1" link add name tun6.10 type ip6tnl local fee1:4::1 remote fee1:4::2 encaplimit none ip -net "$nsr1" link set tun6.10 up ip -net "$nsr1" addr add fee1:5::1/64 dev tun6.10 nodad ip -6 -net "$nsr1" route delete default @@ -670,7 +670,7 @@ ip -net "$nsr2" addr add 192.168.200.2/24 dev tun0.10 ip -net "$nsr2" route change default via 192.168.200.1 ip netns exec "$nsr2" sysctl net.ipv4.conf.tun0/10.forwarding=1 > /dev/null -ip -net "$nsr2" link add name tun6.10 type ip6tnl local fee1:4::2 remote fee1:4::1 || ret=1 +ip -net "$nsr2" link add name tun6.10 type ip6tnl local fee1:4::2 remote fee1:4::1 encaplimit none || ret=1 ip -net "$nsr2" link set tun6.10 up ip -net "$nsr2" addr add fee1:5::2/64 dev tun6.10 nodad ip -6 -net "$nsr2" route delete default diff --git a/tools/testing/selftests/net/tcp_mmap.c b/tools/testing/selftests/net/tcp_mmap.c index 4fcce5150850d..2544ae35d07a8 100644 --- a/tools/testing/selftests/net/tcp_mmap.c +++ b/tools/testing/selftests/net/tcp_mmap.c @@ -313,6 +313,8 @@ void *child_thread(void *arg) tcp_info_get_rcv_mss(fd)); } error: + if (ctx) + EVP_MD_CTX_free(ctx); munmap(buffer, buffer_sz); close(fd); if (zflg) @@ -606,6 +608,8 @@ int main(int argc, char *argv[]) EVP_DigestFinal_ex(ctx, digest, &digest_len); send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0); } + if (ctx) + EVP_MD_CTX_free(ctx); close(fd); munmap(buffer, buffer_sz); return 0; diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c index 30a236b8e9f73..d805a7dfbdd51 100644 --- a/tools/testing/selftests/net/tls.c +++ b/tools/testing/selftests/net/tls.c @@ -997,6 +997,8 @@ TEST_F(tls, splice_short) char sendbuf[0x100]; char sendchar = 'S'; int pipefds[2]; + int pipe_sz; + int ret; int i; sendchar_iov.iov_base = &sendchar; @@ -1005,7 +1007,11 @@ TEST_F(tls, splice_short) memset(sendbuf, 's', sizeof(sendbuf)); ASSERT_GE(pipe2(pipefds, O_NONBLOCK), 0); - ASSERT_GE(fcntl(pipefds[0], F_SETPIPE_SZ, (MAX_FRAGS + 1) * 0x1000), 0); + pipe_sz = (MAX_FRAGS + 1) * getpagesize(); + ret = fcntl(pipefds[0], F_SETPIPE_SZ, pipe_sz); + if (ret < 0 && errno == EPERM) + SKIP(return, "insufficient pipe capacity"); + ASSERT_GE(ret, pipe_sz); for (i = 0; i < MAX_FRAGS; i++) ASSERT_GE(vmsplice(pipefds[1], &sendchar_iov, 1, 0), 0); diff --git a/tools/testing/selftests/net/vlan_bridge_binding.sh b/tools/testing/selftests/net/vlan_bridge_binding.sh index e8c02c64e03a4..d04caa14202d0 100755 --- a/tools/testing/selftests/net/vlan_bridge_binding.sh +++ b/tools/testing/selftests/net/vlan_bridge_binding.sh @@ -64,7 +64,7 @@ check_operstate() local expect=$1; shift local operstate - operstate=$(busywait 1000 \ + operstate=$(busywait 2000 \ operstate_is "$dev" "$expect") check_err $? "Got operstate of $operstate, expected $expect" } diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile index 50d69e22ee7a6..aba6317f6cb8e 100644 --- a/tools/testing/selftests/rseq/Makefile +++ b/tools/testing/selftests/rseq/Makefile @@ -5,9 +5,13 @@ CLANG_FLAGS += -no-integrated-as endif top_srcdir = ../../../.. +include $(top_srcdir)/scripts/subarch.include +ARCH ?= $(SUBARCH) +LINUX_TOOL_ARCH_INCLUDE = $(top_srcdir)/tools/arch/$(ARCH)/include CFLAGS += -O2 -Wall -g -I./ $(KHDR_INCLUDES) -L$(OUTPUT) -Wl,-rpath=./ \ - $(CLANG_FLAGS) -I$(top_srcdir)/tools/include + $(CLANG_FLAGS) -I$(top_srcdir)/tools/include \ + -I$(LINUX_TOOL_ARCH_INCLUDE) LDLIBS += -lpthread -ldl # Own dependencies because we only want to build against 1st prerequisite, but diff --git a/tools/testing/selftests/vfio/Makefile b/tools/testing/selftests/vfio/Makefile index 0684932d91bfc..0a4cfd1a6c7ed 100644 --- a/tools/testing/selftests/vfio/Makefile +++ b/tools/testing/selftests/vfio/Makefile @@ -1,6 +1,6 @@ ARCH ?= $(shell uname -m) -ifeq (,$(filter $(ARCH),aarch64 arm64 x86_64)) +ifeq (,$(filter $(ARCH),aarch64 arm64 x86 x86_64)) # Do nothing on unsupported architectures include ../lib.mk else @@ -27,10 +27,13 @@ CFLAGS += $(EXTRA_CFLAGS) LDFLAGS += -pthread -$(TEST_GEN_PROGS): %: %.o $(LIBVFIO_O) +$(TEST_GEN_PROGS): $(OUTPUT)/%: $(OUTPUT)/%.o $(LIBVFIO_O) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $< $(LIBVFIO_O) $(LDLIBS) -o $@ TEST_GEN_PROGS_O = $(patsubst %, %.o, $(TEST_GEN_PROGS)) +$(TEST_GEN_PROGS_O): $(OUTPUT)/%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@ + TEST_DEP_FILES = $(patsubst %.o, %.d, $(TEST_GEN_PROGS_O) $(LIBVFIO_O)) -include $(TEST_DEP_FILES) diff --git a/tools/testing/selftests/vfio/lib/libvfio.mk b/tools/testing/selftests/vfio/lib/libvfio.mk index 9f47bceed16f4..f90235a1043fe 100644 --- a/tools/testing/selftests/vfio/lib/libvfio.mk +++ b/tools/testing/selftests/vfio/lib/libvfio.mk @@ -19,11 +19,13 @@ LIBVFIO_OUTPUT := $(OUTPUT)/libvfio LIBVFIO_O := $(patsubst %.c, $(LIBVFIO_OUTPUT)/%.o, $(LIBVFIO_C)) LIBVFIO_O_DIRS := $(shell dirname $(LIBVFIO_O) | uniq) -$(shell mkdir -p $(LIBVFIO_O_DIRS)) + +$(LIBVFIO_O_DIRS): + mkdir -p $@ CFLAGS += -I$(LIBVFIO_SRCDIR)/include -$(LIBVFIO_O): $(LIBVFIO_OUTPUT)/%.o : $(LIBVFIO_SRCDIR)/%.c +$(LIBVFIO_O): $(LIBVFIO_OUTPUT)/%.o : $(LIBVFIO_SRCDIR)/%.c | $(LIBVFIO_O_DIRS) $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@ EXTRA_CLEAN += $(LIBVFIO_OUTPUT) diff --git a/tools/tracing/rtla/src/actions.c b/tools/tracing/rtla/src/actions.c index b0d68b5de08db..bf13d9d68f169 100644 --- a/tools/tracing/rtla/src/actions.c +++ b/tools/tracing/rtla/src/actions.c @@ -247,6 +247,8 @@ actions_perform(struct actions *self) int pid, retval; const struct action *action; + self->continue_flag = false; + for_each_action(self, action) { switch (action->type) { case ACTION_TRACE_OUTPUT: diff --git a/tools/tracing/rtla/src/common.c b/tools/tracing/rtla/src/common.c index bc9d01ddd1029..bfeccc6222e5b 100644 --- a/tools/tracing/rtla/src/common.c +++ b/tools/tracing/rtla/src/common.c @@ -10,7 +10,7 @@ #include "common.h" -struct trace_instance *trace_inst; +struct osnoise_tool *trace_tool; volatile int stop_tracing; int nr_cpus; @@ -21,12 +21,16 @@ static void stop_trace(int sig) * Stop requested twice in a row; abort event processing and * exit immediately */ - tracefs_iterate_stop(trace_inst->inst); + if (trace_tool) + tracefs_iterate_stop(trace_tool->trace.inst); return; } stop_tracing = 1; - if (trace_inst) - trace_instance_stop(trace_inst); + if (trace_tool) { + trace_instance_stop(&trace_tool->trace); + if (trace_tool->record) + trace_instance_stop(&trace_tool->record->trace); + } } /* @@ -255,11 +259,10 @@ int run_tool(struct tool_ops *ops, int argc, char *argv[]) tool->params = params; /* - * Save trace instance into global variable so that SIGINT can stop - * the timerlat tracer. + * Expose the tool to signal handlers so they can stop the trace. * Otherwise, rtla could loop indefinitely when overloaded. */ - trace_inst = &tool->trace; + trace_tool = tool; retval = ops->apply_config(tool); if (retval) { @@ -267,7 +270,7 @@ int run_tool(struct tool_ops *ops, int argc, char *argv[]) goto out_free; } - retval = enable_tracer_by_name(trace_inst->inst, ops->tracer); + retval = enable_tracer_by_name(tool->trace.inst, ops->tracer); if (retval) { err_msg("Failed to enable %s tracer\n", ops->tracer); goto out_free; diff --git a/tools/tracing/rtla/src/common.h b/tools/tracing/rtla/src/common.h index 8921807bda988..505babf386373 100644 --- a/tools/tracing/rtla/src/common.h +++ b/tools/tracing/rtla/src/common.h @@ -54,7 +54,6 @@ struct osnoise_context { int opt_workload; }; -extern struct trace_instance *trace_inst; extern volatile int stop_tracing; struct hist_params { diff --git a/tools/tracing/rtla/src/timerlat.c b/tools/tracing/rtla/src/timerlat.c index f8c057518d223..637f68d684f5f 100644 --- a/tools/tracing/rtla/src/timerlat.c +++ b/tools/tracing/rtla/src/timerlat.c @@ -202,7 +202,7 @@ void timerlat_analyze(struct osnoise_tool *tool, bool stopped) * If the trace did not stop with --aa-only, at least print * the max known latency. */ - max_lat = tracefs_instance_file_read(trace_inst->inst, "tracing_max_latency", NULL); + max_lat = tracefs_instance_file_read(tool->trace.inst, "tracing_max_latency", NULL); if (max_lat) { printf(" Max latency was %s\n", max_lat); free(max_lat); diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 4b6708e333b8f..d52fd59195cd1 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -694,7 +694,7 @@ static void timerlat_hist_usage(void) "[-d s] [-D] [-n] [-a us] [-p us] [-i us] [-T us] [-s us] \\", " [-t [file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", " [-P priority] [-E N] [-b N] [--no-irq] [--no-thread] [--no-header] [--no-summary] \\", - " [--no-index] [--with-zeros] [--dma-latency us] [-C [cgroup_name]] [--no-aa] [--dump-task] [-u|-k]", + " [--no-index] [--with-zeros] [--dma-latency us] [-C [cgroup_name]] [--no-aa] [--dump-tasks] [-u|-k]", " [--warm-up s] [--deepest-idle-state n]", NULL, }; @@ -809,7 +809,7 @@ static struct common_params {"filter", required_argument, 0, '7'}, {"dma-latency", required_argument, 0, '8'}, {"no-aa", no_argument, 0, '9'}, - {"dump-task", no_argument, 0, '\1'}, + {"dump-tasks", no_argument, 0, '\1'}, {"warm-up", required_argument, 0, '\2'}, {"trace-buffer-size", required_argument, 0, '\3'}, {"deepest-idle-state", required_argument, 0, '\4'}, diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 91f88bbebad9e..035abf01dbe63 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -467,7 +467,8 @@ static void timerlat_top_usage(void) static const char *const msg_start[] = { "[-q] [-a us] [-d s] [-D] [-n] [-p us] [-i us] [-T us] [-s us] \\", " [[-t [file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", - " [-P priority] [--dma-latency us] [--aa-only us] [-C [cgroup_name]] [-u|-k] [--warm-up s] [--deepest-idle-state n]", + " [-P priority] [--dma-latency us] [--aa-only us] [-C [cgroup_name]] [--dump-tasks] [-u|-k] [--warm-up s]\\", + " [--deepest-idle-state n]", NULL, }; diff --git a/tools/usb/usbip/libsrc/usbip_common.c b/tools/usb/usbip/libsrc/usbip_common.c index b8d7d480595a8..f4734f552d314 100644 --- a/tools/usb/usbip/libsrc/usbip_common.c +++ b/tools/usb/usbip/libsrc/usbip_common.c @@ -29,6 +29,8 @@ static const struct speed_string speed_strings[] = { { USB_SPEED_HIGH, "480", "High Speed(480Mbps)" }, { USB_SPEED_WIRELESS, "53.3-480", "Wireless"}, { USB_SPEED_SUPER, "5000", "Super Speed(5000Mbps)" }, + { USB_SPEED_SUPER_PLUS, "10000", "Super Speed Plus(10000Mbps)" }, + { USB_SPEED_SUPER_PLUS, "20000", "Super Speed Plus(20000Mbps)" }, { 0, NULL, NULL } }; diff --git a/tools/usb/usbip/libsrc/usbip_device_driver.c b/tools/usb/usbip/libsrc/usbip_device_driver.c index 1dfbb76ab26c3..c9b3619d86f31 100644 --- a/tools/usb/usbip/libsrc/usbip_device_driver.c +++ b/tools/usb/usbip/libsrc/usbip_device_driver.c @@ -57,6 +57,10 @@ static struct { .speed = USB_SPEED_SUPER, .name = "super-speed", }, + { + .speed = USB_SPEED_SUPER_PLUS, + .name = "super-speed-plus", + }, }; static diff --git a/tools/usb/usbip/libsrc/vhci_driver.c b/tools/usb/usbip/libsrc/vhci_driver.c index 8159fd98680b4..4ca3783ee5b7c 100644 --- a/tools/usb/usbip/libsrc/vhci_driver.c +++ b/tools/usb/usbip/libsrc/vhci_driver.c @@ -338,6 +338,7 @@ int usbip_vhci_get_free_port(uint32_t speed) switch (speed) { case USB_SPEED_SUPER: + case USB_SPEED_SUPER_PLUS: if (vhci_driver->idev[i].hub != HUB_SPEED_SUPER) continue; break; diff --git a/tools/virtio/vringh_test.c b/tools/virtio/vringh_test.c index b9591223437a6..5ea6d29bc992d 100644 --- a/tools/virtio/vringh_test.c +++ b/tools/virtio/vringh_test.c @@ -159,7 +159,12 @@ static int parallel_test(u64 features, /* Parent and child use separate addresses, to check our mapping logic! */ host_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if (host_map == MAP_FAILED) + err(1, "mmap host_map"); + guest_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if (guest_map == MAP_FAILED) + err(1, "mmap guest_map"); pipe_ret = pipe(to_guest); assert(!pipe_ret); diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 0e8b8a2c5b791..93ad2ebc963f9 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -779,21 +780,18 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) return true; /* otherwise, we have to actually compare the data */ - - BUG_ON(!IS_ALIGNED((unsigned long)val, len)); - switch (len) { case 1: - _val = *(u8 *)val; + _val = get_unaligned((u8 *)val); break; case 2: - _val = *(u16 *)val; + _val = get_unaligned((u16 *)val); break; case 4: - _val = *(u32 *)val; + _val = get_unaligned((u32 *)val); break; case 8: - _val = *(u64 *)val; + _val = get_unaligned((u64 *)val); break; default: return false; diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 69c9d6d546b28..3694d5405a5e3 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -640,15 +640,16 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) } int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, loff_t offset) + unsigned int fd, uoff_t offset) { - loff_t size = slot->npages << PAGE_SHIFT; + uoff_t size = slot->npages << PAGE_SHIFT; unsigned long start, end; struct gmem_file *f; struct inode *inode; struct file *file; int r = -EINVAL; + BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset)); BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff)); file = fget(fd); @@ -664,8 +665,7 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, inode = file_inode(file); - if (offset < 0 || !PAGE_ALIGNED(offset) || - offset + size > i_size_read(inode)) + if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode)) goto err; filemap_invalidate_lock(inode->i_mapping); @@ -858,7 +858,8 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, return ret; } -long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages, +long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, + long npages, bool may_writeback_src, kvm_gmem_populate_cb post_populate, void *opaque) { struct kvm_memory_slot *slot; @@ -892,8 +893,9 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long if (src) { unsigned long uaddr = (unsigned long)src + i * PAGE_SIZE; + unsigned int flags = may_writeback_src ? FOLL_WRITE : 0; - ret = get_user_pages_fast(uaddr, 1, 0, &src_page); + ret = get_user_pages_fast(uaddr, 1, flags, &src_page); if (ret < 0) break; if (ret != 1) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 881f92d7a469e..1a529098eec98 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -6069,25 +6069,19 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr) { struct kvm_io_bus *bus; - int dev_idx, srcu_idx; - struct kvm_io_device *iodev = NULL; + int dev_idx; - srcu_idx = srcu_read_lock(&kvm->srcu); + lockdep_assert_held(&kvm->srcu); bus = kvm_get_bus_srcu(kvm, bus_idx); if (!bus) - goto out_unlock; + return NULL; dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1); if (dev_idx < 0) - goto out_unlock; - - iodev = bus->range[dev_idx].dev; - -out_unlock: - srcu_read_unlock(&kvm->srcu, srcu_idx); + return NULL; - return iodev; + return bus->range[dev_idx].dev; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_io_bus_get_dev); diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h index 9fcc5d5b7f8d0..7510ca915dd1a 100644 --- a/virt/kvm/kvm_mm.h +++ b/virt/kvm/kvm_mm.h @@ -3,6 +3,9 @@ #ifndef __KVM_MM_H__ #define __KVM_MM_H__ 1 +#include +#include + /* * Architectures can choose whether to use an rwlock or spinlock * for the mmu_lock. These macros, for use in common code @@ -72,7 +75,7 @@ int kvm_gmem_init(struct module *module); void kvm_gmem_exit(void); int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, loff_t offset); + unsigned int fd, uoff_t offset); void kvm_gmem_unbind(struct kvm_memory_slot *slot); #else static inline int kvm_gmem_init(struct module *module) @@ -82,7 +85,7 @@ static inline int kvm_gmem_init(struct module *module) static inline void kvm_gmem_exit(void) {}; static inline int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, loff_t offset) + unsigned int fd, uoff_t offset) { WARN_ON_ONCE(1); return -EIO;