diff --git a/.drone.yml b/.drone.yml index b72936ca478e..fe354341b0d1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,7 +13,7 @@ steps: from_secret: bot_api_key PD_API_KEY: from_secret: pd_api_key - TYPE: REL-4.0.0 + TYPE: REL-5.0.0 DEVICE: alioth KBUILD_BUILD_USER: UtsavTheCunt KBUILD_BUILD_HOST: CuntsSpace @@ -38,7 +38,7 @@ steps: from_secret: bot_api_key PD_API_KEY: from_secret: pd_api_key - TYPE: REL-4.0.0 + TYPE: REL-5.0.0 DEVICE: apollo KBUILD_BUILD_USER: UtsavTheCunt KBUILD_BUILD_HOST: CuntsSpace @@ -48,6 +48,31 @@ steps: - ./setup-drone --aosp-clang - ./build-kernel --aosp-clang +--- +clone: + depth: 1 +kind: pipeline +name: IMMENSITY - X | CMI +steps: + - name: building-cmi + image: debian:stable-slim + environment: + CI_CHANNEL_ID: + from_secret: ci_channel_id + BOT_API_KEY: + from_secret: bot_api_key + PD_API_KEY: + from_secret: pd_api_key + TYPE: REL-5.0.0 + DEVICE: cmi + KBUILD_BUILD_USER: UtsavTheCunt + KBUILD_BUILD_HOST: CuntsSpace + commands: + - apt-get update && apt-get install -y bison build-essential bc bison curl libssl-dev git zip python python3 flex cpio libncurses5-dev wget + - git clone https://github.com/UtsavBalar1231/Drone-scripts.git --depth=1 -b test script && cd script + - ./setup-drone --aosp-clang + - ./build-kernel --aosp-clang + --- clone: depth: 1 @@ -63,7 +88,7 @@ steps: from_secret: bot_api_key PD_API_KEY: from_secret: pd_api_key - TYPE: REL-4.0.0 + TYPE: REL-5.0.0 DEVICE: lmi KBUILD_BUILD_USER: UtsavTheCunt KBUILD_BUILD_HOST: CuntsSpace @@ -72,3 +97,28 @@ steps: - git clone https://github.com/UtsavBalar1231/Drone-scripts.git --depth=1 -b test script && cd script - ./setup-drone --aosp-clang - ./build-kernel --aosp-clang + +--- +clone: + depth: 1 +kind: pipeline +name: IMMENSITY - X | UMI +steps: + - name: building-umi + image: debian:stable-slim + environment: + CI_CHANNEL_ID: + from_secret: ci_channel_id + BOT_API_KEY: + from_secret: bot_api_key + PD_API_KEY: + from_secret: pd_api_key + TYPE: REL-5.0.0 + DEVICE: umi + KBUILD_BUILD_USER: UtsavTheCunt + KBUILD_BUILD_HOST: CuntsSpace + commands: + - apt-get update && apt-get install -y bison build-essential bc bison curl libssl-dev git zip python python3 flex cpio libncurses5-dev wget + - git clone https://github.com/UtsavBalar1231/Drone-scripts.git --depth=1 -b test script && cd script + - ./setup-drone --aosp-clang + - ./build-kernel --aosp-clang diff --git a/Documentation/ABI/testing/sysfs-ata b/Documentation/ABI/testing/sysfs-ata index 9ab0ef1dd1c7..299e0d1dc161 100644 --- a/Documentation/ABI/testing/sysfs-ata +++ b/Documentation/ABI/testing/sysfs-ata @@ -107,13 +107,14 @@ Description: described in ATA8 7.16 and 7.17. Only valid if the device is not a PM. - pio_mode: (RO) Transfer modes supported by the device when - in PIO mode. Mostly used by PATA device. + pio_mode: (RO) PIO transfer mode used by the device. + Mostly used by PATA devices. - xfer_mode: (RO) Current transfer mode + xfer_mode: (RO) Current transfer mode. Mostly used by + PATA devices. - dma_mode: (RO) Transfer modes supported by the device when - in DMA mode. Mostly used by PATA device. + dma_mode: (RO) DMA transfer mode used by the device. + Mostly used by PATA devices. class: (RO) Device class. Can be "ata" for disk, "atapi" for packet device, "pmp" for PM, or diff --git a/Documentation/asm-annotations.rst b/Documentation/asm-annotations.rst new file mode 100644 index 000000000000..29ccd6e61fe5 --- /dev/null +++ b/Documentation/asm-annotations.rst @@ -0,0 +1,216 @@ +Assembler Annotations +===================== + +Copyright (c) 2017-2019 Jiri Slaby + +This document describes the new macros for annotation of data and code in +assembly. In particular, it contains information about ``SYM_FUNC_START``, +``SYM_FUNC_END``, ``SYM_CODE_START``, and similar. + +Rationale +--------- +Some code like entries, trampolines, or boot code needs to be written in +assembly. The same as in C, such code is grouped into functions and +accompanied with data. Standard assemblers do not force users into precisely +marking these pieces as code, data, or even specifying their length. +Nevertheless, assemblers provide developers with such annotations to aid +debuggers throughout assembly. On top of that, developers also want to mark +some functions as *global* in order to be visible outside of their translation +units. + +Over time, the Linux kernel has adopted macros from various projects (like +``binutils``) to facilitate such annotations. So for historic reasons, +developers have been using ``ENTRY``, ``END``, ``ENDPROC``, and other +annotations in assembly. Due to the lack of their documentation, the macros +are used in rather wrong contexts at some locations. Clearly, ``ENTRY`` was +intended to denote the beginning of global symbols (be it data or code). +``END`` used to mark the end of data or end of special functions with +*non-standard* calling convention. In contrast, ``ENDPROC`` should annotate +only ends of *standard* functions. + +When these macros are used correctly, they help assemblers generate a nice +object with both sizes and types set correctly. For example, the result of +``arch/x86/lib/putuser.S``:: + + Num: Value Size Type Bind Vis Ndx Name + 25: 0000000000000000 33 FUNC GLOBAL DEFAULT 1 __put_user_1 + 29: 0000000000000030 37 FUNC GLOBAL DEFAULT 1 __put_user_2 + 32: 0000000000000060 36 FUNC GLOBAL DEFAULT 1 __put_user_4 + 35: 0000000000000090 37 FUNC GLOBAL DEFAULT 1 __put_user_8 + +This is not only important for debugging purposes. When there are properly +annotated objects like this, tools can be run on them to generate more useful +information. In particular, on properly annotated objects, ``objtool`` can be +run to check and fix the object if needed. Currently, ``objtool`` can report +missing frame pointer setup/destruction in functions. It can also +automatically generate annotations for :doc:`ORC unwinder ` +for most code. Both of these are especially important to support reliable +stack traces which are in turn necessary for :doc:`Kernel live patching +`. + +Caveat and Discussion +--------------------- +As one might realize, there were only three macros previously. That is indeed +insufficient to cover all the combinations of cases: + +* standard/non-standard function +* code/data +* global/local symbol + +There was a discussion_ and instead of extending the current ``ENTRY/END*`` +macros, it was decided that brand new macros should be introduced instead:: + + So how about using macro names that actually show the purpose, instead + of importing all the crappy, historic, essentially randomly chosen + debug symbol macro names from the binutils and older kernels? + +.. _discussion: https://lkml.kernel.org/r/20170217104757.28588-1-jslaby@suse.cz + +Macros Description +------------------ + +The new macros are prefixed with the ``SYM_`` prefix and can be divided into +three main groups: + +1. ``SYM_FUNC_*`` -- to annotate C-like functions. This means functions with + standard C calling conventions, i.e. the stack contains a return address at + the predefined place and a return from the function can happen in a + standard way. When frame pointers are enabled, save/restore of frame + pointer shall happen at the start/end of a function, respectively, too. + + Checking tools like ``objtool`` should ensure such marked functions conform + to these rules. The tools can also easily annotate these functions with + debugging information (like *ORC data*) automatically. + +2. ``SYM_CODE_*`` -- special functions called with special stack. Be it + interrupt handlers with special stack content, trampolines, or startup + functions. + + Checking tools mostly ignore checking of these functions. But some debug + information still can be generated automatically. For correct debug data, + this code needs hints like ``UNWIND_HINT_REGS`` provided by developers. + +3. ``SYM_DATA*`` -- obviously data belonging to ``.data`` sections and not to + ``.text``. Data do not contain instructions, so they have to be treated + specially by the tools: they should not treat the bytes as instructions, + nor assign any debug information to them. + +Instruction Macros +~~~~~~~~~~~~~~~~~~ +This section covers ``SYM_FUNC_*`` and ``SYM_CODE_*`` enumerated above. + +* ``SYM_FUNC_START`` and ``SYM_FUNC_START_LOCAL`` are supposed to be **the + most frequent markings**. They are used for functions with standard calling + conventions -- global and local. Like in C, they both align the functions to + architecture specific ``__ALIGN`` bytes. There are also ``_NOALIGN`` variants + for special cases where developers do not want this implicit alignment. + + ``SYM_FUNC_START_WEAK`` and ``SYM_FUNC_START_WEAK_NOALIGN`` markings are + also offered as an assembler counterpart to the *weak* attribute known from + C. + + All of these **shall** be coupled with ``SYM_FUNC_END``. First, it marks + the sequence of instructions as a function and computes its size to the + generated object file. Second, it also eases checking and processing such + object files as the tools can trivially find exact function boundaries. + + So in most cases, developers should write something like in the following + example, having some asm instructions in between the macros, of course:: + + SYM_FUNC_START(function_hook) + ... asm insns ... + SYM_FUNC_END(function_hook) + + In fact, this kind of annotation corresponds to the now deprecated ``ENTRY`` + and ``ENDPROC`` macros. + +* ``SYM_FUNC_START_ALIAS`` and ``SYM_FUNC_START_LOCAL_ALIAS`` serve for those + who decided to have two or more names for one function. The typical use is:: + + SYM_FUNC_START_ALIAS(__memset) + SYM_FUNC_START(memset) + ... asm insns ... + SYM_FUNC_END(memset) + SYM_FUNC_END_ALIAS(__memset) + + In this example, one can call ``__memset`` or ``memset`` with the same + result, except the debug information for the instructions is generated to + the object file only once -- for the non-``ALIAS`` case. + +* ``SYM_CODE_START`` and ``SYM_CODE_START_LOCAL`` should be used only in + special cases -- if you know what you are doing. This is used exclusively + for interrupt handlers and similar where the calling convention is not the C + one. ``_NOALIGN`` variants exist too. The use is the same as for the ``FUNC`` + category above:: + + SYM_CODE_START_LOCAL(bad_put_user) + ... asm insns ... + SYM_CODE_END(bad_put_user) + + Again, every ``SYM_CODE_START*`` **shall** be coupled by ``SYM_CODE_END``. + + To some extent, this category corresponds to deprecated ``ENTRY`` and + ``END``. Except ``END`` had several other meanings too. + +* ``SYM_INNER_LABEL*`` is used to denote a label inside some + ``SYM_{CODE,FUNC}_START`` and ``SYM_{CODE,FUNC}_END``. They are very similar + to C labels, except they can be made global. An example of use:: + + SYM_CODE_START(ftrace_caller) + /* save_mcount_regs fills in first two parameters */ + ... + + SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) + /* Load the ftrace_ops into the 3rd parameter */ + ... + + SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) + call ftrace_stub + ... + retq + SYM_CODE_END(ftrace_caller) + +Data Macros +~~~~~~~~~~~ +Similar to instructions, there is a couple of macros to describe data in the +assembly. + +* ``SYM_DATA_START`` and ``SYM_DATA_START_LOCAL`` mark the start of some data + and shall be used in conjunction with either ``SYM_DATA_END``, or + ``SYM_DATA_END_LABEL``. The latter adds also a label to the end, so that + people can use ``lstack`` and (local) ``lstack_end`` in the following + example:: + + SYM_DATA_START_LOCAL(lstack) + .skip 4096 + SYM_DATA_END_LABEL(lstack, SYM_L_LOCAL, lstack_end) + +* ``SYM_DATA`` and ``SYM_DATA_LOCAL`` are variants for simple, mostly one-line + data:: + + SYM_DATA(HEAP, .long rm_heap) + SYM_DATA(heap_end, .long rm_stack) + + In the end, they expand to ``SYM_DATA_START`` with ``SYM_DATA_END`` + internally. + +Support Macros +~~~~~~~~~~~~~~ +All the above reduce themselves to some invocation of ``SYM_START``, +``SYM_END``, or ``SYM_ENTRY`` at last. Normally, developers should avoid using +these. + +Further, in the above examples, one could see ``SYM_L_LOCAL``. There are also +``SYM_L_GLOBAL`` and ``SYM_L_WEAK``. All are intended to denote linkage of a +symbol marked by them. They are used either in ``_LABEL`` variants of the +earlier macros, or in ``SYM_START``. + + +Overriding Macros +~~~~~~~~~~~~~~~~~ +Architecture can also override any of the macros in their own +``asm/linkage.h``, including macros specifying the type of a symbol +(``SYM_T_FUNC``, ``SYM_T_OBJECT``, and ``SYM_T_NONE``). As every macro +described in this file is surrounded by ``#ifdef`` + ``#endif``, it is enough +to define the macros differently in the aforementioned architecture-dependent +header. diff --git a/Documentation/conf.py b/Documentation/conf.py index 22c1a6d96f9e..f3cf0111bbd2 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -95,7 +95,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: diff --git a/Documentation/filesystems/ext4/ondisk/directory.rst b/Documentation/filesystems/ext4/ondisk/directory.rst index 8fcba68c2884..4a9b8553643e 100644 --- a/Documentation/filesystems/ext4/ondisk/directory.rst +++ b/Documentation/filesystems/ext4/ondisk/directory.rst @@ -121,6 +121,31 @@ The directory file type is one of the following values: * - 0x7 - Symbolic link. +To support directories that are both encrypted and casefolded directories, we +must also include hash information in the directory entry. We append +``ext4_extended_dir_entry_2`` to ``ext4_dir_entry_2`` except for the entries +for dot and dotdot, which are kept the same. The structure follows immediately +after ``name`` and is included in the size listed by ``rec_len`` If a directory +entry uses this extension, it may be up to 271 bytes. + +.. list-table:: + :widths: 8 8 24 40 + :header-rows: 1 + + * - Offset + - Size + - Name + - Description + * - 0x0 + - \_\_le32 + - hash + - The hash of the directory name + * - 0x4 + - \_\_le32 + - minor\_hash + - The minor hash of the directory name + + In order to add checksums to these classic directory blocks, a phony ``struct ext4_dir_entry`` is placed at the end of each leaf block to hold the checksum. The directory entry is 12 bytes long. The inode @@ -322,6 +347,8 @@ The directory hash is one of the following values: - Half MD4, unsigned. * - 0x5 - Tea, unsigned. + * - 0x6 + - Siphash. Interior nodes of an htree are recorded as ``struct dx_node``, which is also the full length of a data block: diff --git a/Documentation/index.rst b/Documentation/index.rst index 1cdc139adb40..c1a24a503a75 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -94,6 +94,14 @@ needed). vm/index bpf/index +Architecture-agnostic documentation +----------------------------------- + +.. toctree:: + :maxdepth: 2 + + asm-annotations + Architecture-specific documentation ----------------------------------- diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst index c0917107b90a..3739c1ce686d 100644 --- a/Documentation/process/submitting-patches.rst +++ b/Documentation/process/submitting-patches.rst @@ -133,7 +133,7 @@ as you intend it to. The maintainer will thank you if you write your patch description in a form which can be easily pulled into Linux's source code management -system, ``git``, as a "commit log". See :ref:`explicit_in_reply_to`. +system, ``git``, as a "commit log". See :ref:`the_canonical_patch_format`. Solve only one problem per patch. If your description starts to get long, that's a sign that you probably need to split up your patch. diff --git a/Makefile b/Makefile index 1652b9948927..18136368443f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 4 PATCHLEVEL = 19 -SUBLEVEL = 245 +SUBLEVEL = 247 EXTRAVERSION = NAME = "People's Front" @@ -696,7 +696,6 @@ KBUILD_LDFLAGS += -Os else ifeq ($(cc-name),clang) KBUILD_CFLAGS += -O3 KBUILD_AFLAGS += -O3 -KBUILD_LDFLAGS += -O3 else KBUILD_CFLAGS += -O2 KBUILD_AFLAGS += -O2 diff --git a/README.md b/README.md index 283055f3b4c5..67aeac9da4ce 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Build Status](https://cloud.drone.io/api/badges/UtsavBalar1231/kernel_xiaomi_sm8250/status.svg?ref=refs/heads/android12-stable)](https://cloud.drone.io/UtsavBalar1231/kernel_xiaomi_sm8250) -![Commits Count](https://img.shields.io/github/commits-since/UtsavBalar1231/kernel_xiaomi_sm8250/3.0.0/android12-stable) +![Commits Count](https://img.shields.io/github/commits-since/UtsavBalar1231/kernel_xiaomi_sm8250/4.0.0/android12-stable) [![Hits](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2FUtsavBalar1231%2Fkernel_xiaomi_sm8250&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false)](https://hits.seeyoufarm.com) -![Downloads](https://img.shields.io/github/downloads/UtsavBalar1231/kernel_xiaomi_sm8250/3.0.0/total) +![Downloads](https://img.shields.io/github/downloads/UtsavBalar1231/kernel_xiaomi_sm8250/4.0.0/total) ![logo](https://github.com/UtsavBalar1231/xda-stuff/raw/master/banner.png "logo was here") diff --git a/arch/arm/boot/dts/bcm2835-rpi-b.dts b/arch/arm/boot/dts/bcm2835-rpi-b.dts index 31ff602e2cd3..6bac18d7c070 100644 --- a/arch/arm/boot/dts/bcm2835-rpi-b.dts +++ b/arch/arm/boot/dts/bcm2835-rpi-b.dts @@ -48,18 +48,17 @@ "GPIO18", "NC", /* GPIO19 */ "NC", /* GPIO20 */ - "GPIO21", + "CAM_GPIO0", "GPIO22", "GPIO23", "GPIO24", "GPIO25", "NC", /* GPIO26 */ - "CAM_GPIO0", - /* Binary number representing build/revision */ - "CONFIG0", - "CONFIG1", - "CONFIG2", - "CONFIG3", + "GPIO27", + "GPIO28", + "GPIO29", + "GPIO30", + "GPIO31", "NC", /* GPIO32 */ "NC", /* GPIO33 */ "NC", /* GPIO34 */ diff --git a/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts b/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts index 9f7145b1cc5e..3509582f92f1 100644 --- a/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts +++ b/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts @@ -77,16 +77,18 @@ "GPIO27", "SDA0", "SCL0", - "NC", /* GPIO30 */ - "NC", /* GPIO31 */ - "NC", /* GPIO32 */ - "NC", /* GPIO33 */ - "NC", /* GPIO34 */ - "NC", /* GPIO35 */ - "NC", /* GPIO36 */ - "NC", /* GPIO37 */ - "NC", /* GPIO38 */ - "NC", /* GPIO39 */ + /* Used by BT module */ + "CTS0", + "RTS0", + "TXD0", + "RXD0", + /* Used by Wifi */ + "SD1_CLK", + "SD1_CMD", + "SD1_DATA0", + "SD1_DATA1", + "SD1_DATA2", + "SD1_DATA3", "CAM_GPIO1", /* GPIO40 */ "WL_ON", /* GPIO41 */ "NC", /* GPIO42 */ diff --git a/arch/arm/boot/dts/exynos5250-smdk5250.dts b/arch/arm/boot/dts/exynos5250-smdk5250.dts index 80479ed69070..95a93ef80d97 100644 --- a/arch/arm/boot/dts/exynos5250-smdk5250.dts +++ b/arch/arm/boot/dts/exynos5250-smdk5250.dts @@ -127,7 +127,7 @@ samsung,i2c-max-bus-freq = <20000>; eeprom@50 { - compatible = "samsung,s524ad0xd1"; + compatible = "samsung,s524ad0xd1", "atmel,24c128"; reg = <0x50>; }; @@ -286,7 +286,7 @@ samsung,i2c-max-bus-freq = <20000>; eeprom@51 { - compatible = "samsung,s524ad0xd1"; + compatible = "samsung,s524ad0xd1", "atmel,24c128"; reg = <0x51>; }; diff --git a/arch/arm/boot/dts/ox820.dtsi b/arch/arm/boot/dts/ox820.dtsi index f7dddfb01f81..d629caf8b98f 100644 --- a/arch/arm/boot/dts/ox820.dtsi +++ b/arch/arm/boot/dts/ox820.dtsi @@ -286,7 +286,7 @@ clocks = <&armclk>; }; - gic: gic@1000 { + gic: interrupt-controller@1000 { compatible = "arm,arm11mp-gic"; interrupt-controller; #interrupt-cells = <3>; diff --git a/arch/arm/mach-hisi/platsmp.c b/arch/arm/mach-hisi/platsmp.c index da5689ababf7..d7fbfb6d293d 100644 --- a/arch/arm/mach-hisi/platsmp.c +++ b/arch/arm/mach-hisi/platsmp.c @@ -70,14 +70,17 @@ static void __init hi3xxx_smp_prepare_cpus(unsigned int max_cpus) } ctrl_base = of_iomap(np, 0); if (!ctrl_base) { + of_node_put(np); pr_err("failed to map address\n"); return; } if (of_property_read_u32(np, "smp-offset", &offset) < 0) { + of_node_put(np); pr_err("failed to find smp-offset property\n"); return; } ctrl_base += offset; + of_node_put(np); } } @@ -163,6 +166,7 @@ static int hip01_boot_secondary(unsigned int cpu, struct task_struct *idle) if (WARN_ON(!node)) return -1; ctrl_base = of_iomap(node, 0); + of_node_put(node); /* set the secondary core boot from DDR */ remap_reg_value = readl_relaxed(ctrl_base + REG_SC_CTRL); diff --git a/arch/arm/mach-omap1/clock.c b/arch/arm/mach-omap1/clock.c index fa512413a471..b277409f303a 100644 --- a/arch/arm/mach-omap1/clock.c +++ b/arch/arm/mach-omap1/clock.c @@ -44,7 +44,7 @@ static DEFINE_SPINLOCK(clockfw_lock); unsigned long omap1_uart_recalc(struct clk *clk) { unsigned int val = __raw_readl(clk->enable_reg); - return val & clk->enable_bit ? 48000000 : 12000000; + return val & 1 << clk->enable_bit ? 48000000 : 12000000; } unsigned long omap1_sossi_recalc(struct clk *clk) diff --git a/arch/arm/mach-vexpress/dcscb.c b/arch/arm/mach-vexpress/dcscb.c index ee2a0faafaa1..aaade91f6551 100644 --- a/arch/arm/mach-vexpress/dcscb.c +++ b/arch/arm/mach-vexpress/dcscb.c @@ -146,6 +146,7 @@ static int __init dcscb_init(void) if (!node) return -ENODEV; dcscb_base = of_iomap(node, 0); + of_node_put(node); if (!dcscb_base) return -EADDRNOTAVAIL; cfg = readl_relaxed(dcscb_base + DCS_CFG_R); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 30407622bf64..9a89be9aef8c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -78,6 +78,7 @@ config ARM64 select CLONE_BACKWARDS select COMMON_CLK select CPU_PM if (SUSPEND || CPU_IDLE) + select CRC32 select DCACHE_WORD_ACCESS select DMA_DIRECT_OPS select EDAC_SUPPORT diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi index 18226980f7c3..f48d14cd10a3 100644 --- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi @@ -490,7 +490,7 @@ clocks { sleep_clk: sleep_clk { compatible = "fixed-clock"; - clock-frequency = <32000>; + clock-frequency = <32768>; #clock-cells = <0>; }; diff --git a/arch/arm64/boot/dts/vendor/bindings/gpio/gpio-altera.txt b/arch/arm64/boot/dts/vendor/bindings/gpio/gpio-altera.txt index 146e554b3c67..2a80e272cd66 100755 --- a/arch/arm64/boot/dts/vendor/bindings/gpio/gpio-altera.txt +++ b/arch/arm64/boot/dts/vendor/bindings/gpio/gpio-altera.txt @@ -9,8 +9,9 @@ Required properties: - The second cell is reserved and is currently unused. - gpio-controller : Marks the device node as a GPIO controller. - interrupt-controller: Mark the device node as an interrupt controller -- #interrupt-cells : Should be 1. The interrupt type is fixed in the hardware. +- #interrupt-cells : Should be 2. The interrupt type is fixed in the hardware. - The first cell is the GPIO offset number within the GPIO controller. + - The second cell is the interrupt trigger type and level flags. - interrupts: Specify the interrupt. - altr,interrupt-type: Specifies the interrupt trigger type the GPIO hardware is synthesized. This field is required if the Altera GPIO controller @@ -38,6 +39,6 @@ gpio_altr: gpio@ff200000 { altr,interrupt-type = ; #gpio-cells = <2>; gpio-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; interrupt-controller; }; diff --git a/arch/arm64/boot/dts/vendor/qcom/lmi-sm8250.dtsi b/arch/arm64/boot/dts/vendor/qcom/lmi-sm8250.dtsi index 57adb5c1184f..0ff3ecb5f0db 100644 --- a/arch/arm64/boot/dts/vendor/qcom/lmi-sm8250.dtsi +++ b/arch/arm64/boot/dts/vendor/qcom/lmi-sm8250.dtsi @@ -7,26 +7,6 @@ And public attribution of xiaomi platforms(like J1 and so and) #include "xiaomi-sm8250-common.dtsi" #include "lmi-audio-overlay.dtsi" -// SLPI Memory Region -&pil_slpi_mem { - reg = <0x0 0x88c00000 0x0 0x1500000>; -}; - -// ADSP Memory Region -&pil_adsp_mem { - reg = <0x0 0x8a100000 0x0 0x2500000>; -}; - -// SPSS Memory Region -&pil_spss_mem { - reg = <0x0 0x8c600000 0x0 0x100000>; -}; - -// CDSP Memory Region -&cdsp_secure_heap { - reg = <0x0 0x8c700000 0x0 0x4600000>; -}; - &qupv3_se15_i2c { status = "ok"; nq@64 { diff --git a/arch/arm64/configs/alioth_defconfig b/arch/arm64/configs/alioth_defconfig index 0957dbfc8ea9..85f01eda19a2 100644 --- a/arch/arm64/configs/alioth_defconfig +++ b/arch/arm64/configs/alioth_defconfig @@ -304,7 +304,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -387,6 +386,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -460,6 +460,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_THERMAL_DIMMING=y @@ -701,6 +702,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_SECURITY=y diff --git a/arch/arm64/configs/apollo_defconfig b/arch/arm64/configs/apollo_defconfig index 5b8b519e881c..f5978503d1df 100644 --- a/arch/arm64/configs/apollo_defconfig +++ b/arch/arm64/configs/apollo_defconfig @@ -304,7 +304,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -391,6 +390,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -464,6 +464,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_SOUND=y @@ -704,6 +705,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/cas_defconfig b/arch/arm64/configs/cas_defconfig index 01a66e425b25..ba5d5633a3af 100644 --- a/arch/arm64/configs/cas_defconfig +++ b/arch/arm64/configs/cas_defconfig @@ -304,7 +304,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -392,6 +391,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -469,6 +469,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_THERMAL_DIMMING=y @@ -711,6 +712,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/cmi_defconfig b/arch/arm64/configs/cmi_defconfig index 1f16c5a9ef20..35feaf273fd0 100644 --- a/arch/arm64/configs/cmi_defconfig +++ b/arch/arm64/configs/cmi_defconfig @@ -305,7 +305,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -393,6 +392,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -469,6 +469,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_SOUND=y @@ -708,6 +709,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/elish_defconfig b/arch/arm64/configs/elish_defconfig index c8b56d510a0c..68e4ebadb088 100644 --- a/arch/arm64/configs/elish_defconfig +++ b/arch/arm64/configs/elish_defconfig @@ -304,7 +304,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -391,6 +390,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -466,6 +466,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_BACKLIGHT_KTZ8866=y @@ -701,6 +702,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/enuma_defconfig b/arch/arm64/configs/enuma_defconfig index 5cdb841bc8aa..d6432713e76c 100644 --- a/arch/arm64/configs/enuma_defconfig +++ b/arch/arm64/configs/enuma_defconfig @@ -304,7 +304,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -391,6 +390,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -465,6 +465,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_BACKLIGHT_KTZ8866=y @@ -705,6 +706,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/lmi_defconfig b/arch/arm64/configs/lmi_defconfig index 7b0c2b6fe33e..23b711dfd57a 100644 --- a/arch/arm64/configs/lmi_defconfig +++ b/arch/arm64/configs/lmi_defconfig @@ -305,7 +305,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -394,6 +393,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -466,6 +466,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_SOUND=y @@ -707,6 +708,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/psyche_defconfig b/arch/arm64/configs/psyche_defconfig index dbc5ea3c62f6..2a6096dff954 100644 --- a/arch/arm64/configs/psyche_defconfig +++ b/arch/arm64/configs/psyche_defconfig @@ -304,7 +304,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -393,6 +392,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -464,6 +464,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_SOUND=y @@ -704,6 +705,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/thyme_defconfig b/arch/arm64/configs/thyme_defconfig index ab2bf2cb94f5..9b58a7c295dd 100644 --- a/arch/arm64/configs/thyme_defconfig +++ b/arch/arm64/configs/thyme_defconfig @@ -304,7 +304,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -392,6 +391,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -470,6 +470,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_SOUND=y @@ -710,6 +711,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/umi_defconfig b/arch/arm64/configs/umi_defconfig index a271f559ef31..646d59293469 100644 --- a/arch/arm64/configs/umi_defconfig +++ b/arch/arm64/configs/umi_defconfig @@ -305,7 +305,6 @@ CONFIG_MTD_BLOCK=y CONFIG_MTD_OOPS=y CONFIG_MTD_BLOCK2MTD=y CONFIG_ZRAM=y -CONFIG_ZRAM_WRITEBACK=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -395,6 +394,7 @@ CONFIG_HW_RANDOM_MSM_LEGACY=y CONFIG_DIAG_CHAR=y CONFIG_MSM_ADSPRPC=y CONFIG_XLOGCHAR=y +CONFIG_LRNG=y CONFIG_OKL4_PIPE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_QCOM_GENI=y @@ -472,6 +472,7 @@ CONFIG_MSM_GLOBAL_SYNX=y CONFIG_DRM=y CONFIG_DRM_LONTIUM_LT9611UXC=y CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_QCOM_SPMI_WLED=y CONFIG_SOUND=y @@ -711,6 +712,7 @@ CONFIG_PSTORE=y CONFIG_PSTORE_CONSOLE=y CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_RAM=y +CONFIG_EROFS_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_FS_HPB=y diff --git a/arch/arm64/configs/vendor/bengal-perf_defconfig b/arch/arm64/configs/vendor/bengal-perf_defconfig index 7f30349a57b5..28bdf5519050 100644 --- a/arch/arm64/configs/vendor/bengal-perf_defconfig +++ b/arch/arm64/configs/vendor/bengal-perf_defconfig @@ -490,6 +490,7 @@ CONFIG_LEDS_CLASS=y CONFIG_LEDS_CLASS_FLASH=y CONFIG_LEDS_AW2016=y CONFIG_LEDS_QTI_FLASH=y +CONFIG_LEDS_GPIO=y CONFIG_LEDS_PWM=y CONFIG_LEDS_QTI_TRI_LED=y CONFIG_LEDS_QPNP_FLASH_V2=y diff --git a/arch/arm64/configs/vendor/bengal_defconfig b/arch/arm64/configs/vendor/bengal_defconfig index 814da30e348c..7a09c492926d 100644 --- a/arch/arm64/configs/vendor/bengal_defconfig +++ b/arch/arm64/configs/vendor/bengal_defconfig @@ -505,6 +505,7 @@ CONFIG_LEDS_CLASS=y CONFIG_LEDS_CLASS_FLASH=y CONFIG_LEDS_AW2016=y CONFIG_LEDS_QTI_FLASH=y +CONFIG_LEDS_GPIO=y CONFIG_LEDS_PWM=y CONFIG_LEDS_QTI_TRI_LED=y CONFIG_LEDS_QPNP_FLASH_V2=y diff --git a/arch/arm64/configs/vendor/xiaomi/kona-custom.config b/arch/arm64/configs/vendor/xiaomi/kona-custom.config index 9a4eb4a3f3a0..352a3bda69b8 100644 --- a/arch/arm64/configs/vendor/xiaomi/kona-custom.config +++ b/arch/arm64/configs/vendor/xiaomi/kona-custom.config @@ -5,6 +5,9 @@ # CONFIG_DVB_MPQ is not set # CONFIG_QCOM_LLCC_PERFMON is not set +# Build backlight lcd inline +CONFIG_LCD_CLASS_DEVICE=y + # Enable block device writeback throttling CONFIG_BLK_WBT=y CONFIG_BLK_WBT_SQ=y @@ -65,8 +68,8 @@ CONFIG_MPTCP_ROUNDROBIN=y CONFIG_MPTCP_REDUNDANT=y CONFIG_DEFAULT_ROUNDROBIN=y -# Enable zRAM writeback -CONFIG_ZRAM_WRITEBACK=y +# Disable zRAM writeback +# CONFIG_ZRAM_WRITEBACK is not set # Enable LZ4 zRAM CONFIG_CRYPTO_LZ4=y @@ -209,3 +212,9 @@ CONFIG_FTRACE=y # Enable "TTL" target support CONFIG_IP_NF_TARGET_TTL=y + +# Enable EROFS +CONFIG_EROFS_FS=y + +# Enable Linux Random Number Generator +CONFIG_LRNG=y diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index 6cd5d77b6b44..1877f29f6d97 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -27,4 +27,3 @@ generic-y += trace_clock.h generic-y += unaligned.h generic-y += user.h generic-y += vga.h -generic-y += xor.h diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h new file mode 100644 index 000000000000..5df500dcc627 --- /dev/null +++ b/arch/arm64/include/asm/alternative-macros.h @@ -0,0 +1,217 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_ALTERNATIVE_MACROS_H +#define __ASM_ALTERNATIVE_MACROS_H + +#include + +#define ARM64_CB_PATCH ARM64_NCAPS + +/* A64 instructions are always 32 bits. */ +#define AARCH64_INSN_SIZE 4 + +#ifndef __ASSEMBLY__ + +#include + +#define ALTINSTR_ENTRY(feature) \ + " .word 661b - .\n" /* label */ \ + " .word 663f - .\n" /* new instruction */ \ + " .hword " __stringify(feature) "\n" /* feature bit */ \ + " .byte 662b-661b\n" /* source len */ \ + " .byte 664f-663f\n" /* replacement len */ + +#define ALTINSTR_ENTRY_CB(feature, cb) \ + " .word 661b - .\n" /* label */ \ + " .word " __stringify(cb) "- .\n" /* callback */ \ + " .hword " __stringify(feature) "\n" /* feature bit */ \ + " .byte 662b-661b\n" /* source len */ \ + " .byte 664f-663f\n" /* replacement len */ + +/* + * alternative assembly primitive: + * + * If any of these .org directive fail, it means that insn1 and insn2 + * don't have the same length. This used to be written as + * + * .if ((664b-663b) != (662b-661b)) + * .error "Alternatives instruction length mismatch" + * .endif + * + * but most assemblers die if insn1 or insn2 have a .inst. This should + * be fixed in a binutils release posterior to 2.25.51.0.2 (anything + * containing commit 4e4d08cf7399b606 or c1baaddf8861). + * + * Alternatives with callbacks do not generate replacement instructions. + */ +#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \ + ".if "__stringify(cfg_enabled)" == 1\n" \ + "661:\n\t" \ + oldinstr "\n" \ + "662:\n" \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY(feature) \ + ".popsection\n" \ + ".subsection 1\n" \ + "663:\n\t" \ + newinstr "\n" \ + "664:\n\t" \ + ".org . - (664b-663b) + (662b-661b)\n\t" \ + ".org . - (662b-661b) + (664b-663b)\n\t" \ + ".previous\n" \ + ".endif\n" + +#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb) \ + ".if "__stringify(cfg_enabled)" == 1\n" \ + "661:\n\t" \ + oldinstr "\n" \ + "662:\n" \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY_CB(feature, cb) \ + ".popsection\n" \ + "663:\n\t" \ + "664:\n\t" \ + ".endif\n" + +#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ + __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) + +#define ALTERNATIVE_CB(oldinstr, cb) \ + __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb) +#else + +#include + +.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len + .word \orig_offset - . + .word \alt_offset - . + .hword \feature + .byte \orig_len + .byte \alt_len +.endm + +.macro alternative_insn insn1, insn2, cap, enable = 1 + .if \enable +661: \insn1 +662: .pushsection .altinstructions, "a" + altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f + .popsection + .subsection 1 +663: \insn2 +664: .previous + .org . - (664b-663b) + (662b-661b) + .org . - (662b-661b) + (664b-663b) + .endif +.endm + +/* + * Alternative sequences + * + * The code for the case where the capability is not present will be + * assembled and linked as normal. There are no restrictions on this + * code. + * + * The code for the case where the capability is present will be + * assembled into a special section to be used for dynamic patching. + * Code for that case must: + * + * 1. Be exactly the same length (in bytes) as the default code + * sequence. + * + * 2. Not contain a branch target that is used outside of the + * alternative sequence it is defined in (branches into an + * alternative sequence are not fixed up). + */ + +/* + * Begin an alternative code sequence. + */ +.macro alternative_if_not cap + .set .Lasm_alt_mode, 0 + .pushsection .altinstructions, "a" + altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f + .popsection +661: +.endm + +.macro alternative_if cap + .set .Lasm_alt_mode, 1 + .pushsection .altinstructions, "a" + altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f + .popsection + .subsection 1 + .align 2 /* So GAS knows label 661 is suitably aligned */ +661: +.endm + +.macro alternative_cb cb + .set .Lasm_alt_mode, 0 + .pushsection .altinstructions, "a" + altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0 + .popsection +661: +.endm + +/* + * Provide the other half of the alternative code sequence. + */ +.macro alternative_else +662: + .if .Lasm_alt_mode==0 + .subsection 1 + .else + .previous + .endif +663: +.endm + +/* + * Complete an alternative code sequence. + */ +.macro alternative_endif +664: + .if .Lasm_alt_mode==0 + .previous + .endif + .org . - (664b-663b) + (662b-661b) + .org . - (662b-661b) + (664b-663b) +.endm + +/* + * Callback-based alternative epilogue + */ +.macro alternative_cb_end +662: +.endm + +/* + * Provides a trivial alternative or default sequence consisting solely + * of NOPs. The number of NOPs is chosen automatically to match the + * previous case. + */ +.macro alternative_else_nop_endif +alternative_else + nops (662b-661b) / AARCH64_INSN_SIZE +alternative_endif +.endm + +#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ + alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) + +.macro user_alt, label, oldinstr, newinstr, cond +9999: alternative_insn "\oldinstr", "\newinstr", \cond + _asm_extable 9999b, \label +.endm + +#endif /* __ASSEMBLY__ */ + +/* + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature)); + * + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO)); + * N.B. If CONFIG_FOO is specified, but not selected, the whole block + * will be omitted, including oldinstr. + */ +#define ALTERNATIVE(oldinstr, newinstr, ...) \ + _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1) + +#endif /* __ASM_ALTERNATIVE_MACROS_H */ diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index bb616e2b060f..284dac311676 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -2,17 +2,13 @@ #ifndef __ASM_ALTERNATIVE_H #define __ASM_ALTERNATIVE_H -#include -#include - -#define ARM64_CB_PATCH ARM64_NCAPS +#include #ifndef __ASSEMBLY__ #include #include #include -#include extern int alternatives_applied; @@ -35,264 +31,5 @@ void apply_alternatives_module(void *start, size_t length); static inline void apply_alternatives_module(void *start, size_t length) { } #endif -#define ALTINSTR_ENTRY(feature) \ - " .word 661b - .\n" /* label */ \ - " .word 663f - .\n" /* new instruction */ \ - " .hword " __stringify(feature) "\n" /* feature bit */ \ - " .byte 662b-661b\n" /* source len */ \ - " .byte 664f-663f\n" /* replacement len */ - -#define ALTINSTR_ENTRY_CB(feature, cb) \ - " .word 661b - .\n" /* label */ \ - " .word " __stringify(cb) "- .\n" /* callback */ \ - " .hword " __stringify(feature) "\n" /* feature bit */ \ - " .byte 662b-661b\n" /* source len */ \ - " .byte 664f-663f\n" /* replacement len */ - -/* - * alternative assembly primitive: - * - * If any of these .org directive fail, it means that insn1 and insn2 - * don't have the same length. This used to be written as - * - * .if ((664b-663b) != (662b-661b)) - * .error "Alternatives instruction length mismatch" - * .endif - * - * but most assemblers die if insn1 or insn2 have a .inst. This should - * be fixed in a binutils release posterior to 2.25.51.0.2 (anything - * containing commit 4e4d08cf7399b606 or c1baaddf8861). - * - * Alternatives with callbacks do not generate replacement instructions. - */ -#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \ - ".if "__stringify(cfg_enabled)" == 1\n" \ - "661:\n\t" \ - oldinstr "\n" \ - "662:\n" \ - ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY(feature) \ - ".popsection\n" \ - ".pushsection .altinstr_replacement, \"a\"\n" \ - "663:\n\t" \ - newinstr "\n" \ - "664:\n\t" \ - ".popsection\n\t" \ - ".org . - (664b-663b) + (662b-661b)\n\t" \ - ".org . - (662b-661b) + (664b-663b)\n" \ - ".endif\n" - -#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb) \ - ".if "__stringify(cfg_enabled)" == 1\n" \ - "661:\n\t" \ - oldinstr "\n" \ - "662:\n" \ - ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY_CB(feature, cb) \ - ".popsection\n" \ - "663:\n\t" \ - "664:\n\t" \ - ".endif\n" - -#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ - __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) - -#define ALTERNATIVE_CB(oldinstr, cb) \ - __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb) -#else - -#include - -.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len - .word \orig_offset - . - .word \alt_offset - . - .hword \feature - .byte \orig_len - .byte \alt_len -.endm - -.macro alternative_insn insn1, insn2, cap, enable = 1 - .if \enable -661: \insn1 -662: .pushsection .altinstructions, "a" - altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f - .popsection - .pushsection .altinstr_replacement, "ax" -663: \insn2 -664: .org . - (664b-663b) + (662b-661b) - .org . - (662b-661b) + (664b-663b) - .popsection - .endif -.endm - -/* - * Alternative sequences - * - * The code for the case where the capability is not present will be - * assembled and linked as normal. There are no restrictions on this - * code. - * - * The code for the case where the capability is present will be - * assembled into a special section to be used for dynamic patching. - * Code for that case must: - * - * 1. Be exactly the same length (in bytes) as the default code - * sequence. - * - * 2. Not contain a branch target that is used outside of the - * alternative sequence it is defined in (branches into an - * alternative sequence are not fixed up). - */ - -/* - * Begin an alternative code sequence. - */ -.macro alternative_if_not cap - .set .Lasm_alt_mode, 0 - .pushsection .altinstructions, "a" - altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f - .popsection -661: -.endm - -.macro alternative_if cap - .set .Lasm_alt_mode, 1 - .pushsection .altinstructions, "a" - altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f - .popsection - .pushsection .altinstr_replacement, "ax" - .align 2 /* So GAS knows label 661 is suitably aligned */ -661: -.endm - -.macro alternative_cb cb - .set .Lasm_alt_mode, 0 - .pushsection .altinstructions, "a" - altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0 - .popsection -661: -.endm - -/* - * Provide the other half of the alternative code sequence. - */ -.macro alternative_else -662: - .if .Lasm_alt_mode==0 - .pushsection .altinstr_replacement, "ax" - .else - .popsection - .endif -663: -.endm - -/* - * Complete an alternative code sequence. - */ -.macro alternative_endif -664: - .org . - (664b-663b) + (662b-661b) - .org . - (662b-661b) + (664b-663b) - .if .Lasm_alt_mode==0 - .popsection - .endif -.endm - -/* - * Callback-based alternative epilogue - */ -.macro alternative_cb_end -662: -.endm - -/* - * Provides a trivial alternative or default sequence consisting solely - * of NOPs. The number of NOPs is chosen automatically to match the - * previous case. - */ -.macro alternative_else_nop_endif -alternative_else - nops (662b-661b) / AARCH64_INSN_SIZE -alternative_endif -.endm - -#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ - alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) - -.macro user_alt, label, oldinstr, newinstr, cond -9999: alternative_insn "\oldinstr", "\newinstr", \cond - _asm_extable 9999b, \label -.endm - -/* - * Generate the assembly for UAO alternatives with exception table entries. - * This is complicated as there is no post-increment or pair versions of the - * unprivileged instructions, and USER() only works for single instructions. - */ -#ifdef CONFIG_ARM64_UAO - .macro uao_ldp l, reg1, reg2, addr, post_inc - alternative_if_not ARM64_HAS_UAO -8888: ldp \reg1, \reg2, [\addr], \post_inc; -8889: nop; - nop; - alternative_else - ldtr \reg1, [\addr]; - ldtr \reg2, [\addr, #8]; - add \addr, \addr, \post_inc; - alternative_endif - - _asm_extable 8888b,\l; - _asm_extable 8889b,\l; - .endm - - .macro uao_stp l, reg1, reg2, addr, post_inc - alternative_if_not ARM64_HAS_UAO -8888: stp \reg1, \reg2, [\addr], \post_inc; -8889: nop; - nop; - alternative_else - sttr \reg1, [\addr]; - sttr \reg2, [\addr, #8]; - add \addr, \addr, \post_inc; - alternative_endif - - _asm_extable 8888b,\l; - _asm_extable 8889b,\l; - .endm - - .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc - alternative_if_not ARM64_HAS_UAO -8888: \inst \reg, [\addr], \post_inc; - nop; - alternative_else - \alt_inst \reg, [\addr]; - add \addr, \addr, \post_inc; - alternative_endif - - _asm_extable 8888b,\l; - .endm -#else - .macro uao_ldp l, reg1, reg2, addr, post_inc - USER(\l, ldp \reg1, \reg2, [\addr], \post_inc) - .endm - .macro uao_stp l, reg1, reg2, addr, post_inc - USER(\l, stp \reg1, \reg2, [\addr], \post_inc) - .endm - .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc - USER(\l, \inst \reg, [\addr], \post_inc) - .endm -#endif - -#endif /* __ASSEMBLY__ */ - -/* - * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature)); - * - * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO)); - * N.B. If CONFIG_FOO is specified, but not selected, the whole block - * will be omitted, including oldinstr. - */ -#define ALTERNATIVE(oldinstr, newinstr, ...) \ - _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1) - +#endif /* __ASSEMBLY__ */ #endif /* __ASM_ALTERNATIVE_H */ diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index d8b2ce576553..6e6f90bee521 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -2,7 +2,7 @@ #ifndef __ASM_ASM_UACCESS_H #define __ASM_ASM_UACCESS_H -#include +#include #include #include #include @@ -58,23 +58,6 @@ alternative_else_nop_endif .endm #endif -/* - * These macros are no-ops when UAO is present. - */ - .macro uaccess_disable_not_uao, tmp1, tmp2 - uaccess_ttbr0_disable \tmp1, \tmp2 -alternative_if ARM64_ALT_PAN_NOT_UAO - SET_PSTATE_PAN(1) -alternative_else_nop_endif - .endm - - .macro uaccess_enable_not_uao, tmp1, tmp2, tmp3 - uaccess_ttbr0_enable \tmp1, \tmp2, \tmp3 -alternative_if ARM64_ALT_PAN_NOT_UAO - SET_PSTATE_PAN(0) -alternative_else_nop_endif - .endm - /* * Remove the address tag from a virtual address, if present. */ @@ -83,4 +66,63 @@ alternative_else_nop_endif and \dst, \dst, \addr .endm +/* + * Generate the assembly for UAO alternatives with exception table entries. + * This is complicated as there is no post-increment or pair versions of the + * unprivileged instructions, and USER() only works for single instructions. + */ +#ifdef CONFIG_ARM64_UAO + .macro uao_ldp l, reg1, reg2, addr, post_inc + alternative_if_not ARM64_HAS_UAO +8888: ldp \reg1, \reg2, [\addr], \post_inc; +8889: nop; + nop; + alternative_else + ldtr \reg1, [\addr]; + ldtr \reg2, [\addr, #8]; + add \addr, \addr, \post_inc; + alternative_endif + + _asm_extable 8888b,\l; + _asm_extable 8889b,\l; + .endm + + .macro uao_stp l, reg1, reg2, addr, post_inc + alternative_if_not ARM64_HAS_UAO +8888: stp \reg1, \reg2, [\addr], \post_inc; +8889: nop; + nop; + alternative_else + sttr \reg1, [\addr]; + sttr \reg2, [\addr, #8]; + add \addr, \addr, \post_inc; + alternative_endif + + _asm_extable 8888b,\l; + _asm_extable 8889b,\l; + .endm + + .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc + alternative_if_not ARM64_HAS_UAO +8888: \inst \reg, [\addr], \post_inc; + nop; + alternative_else + \alt_inst \reg, [\addr]; + add \addr, \addr, \post_inc; + alternative_endif + + _asm_extable 8888b,\l; + .endm +#else + .macro uao_ldp l, reg1, reg2, addr, post_inc + USER(\l, ldp \reg1, \reg2, [\addr], \post_inc) + .endm + .macro uao_stp l, reg1, reg2, addr, post_inc + USER(\l, stp \reg1, \reg2, [\addr], \post_inc) + .endm + .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc + USER(\l, \inst \reg, [\addr], \post_inc) + .endm +#endif + #endif diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 0d9b99f3498d..7be91c6ac94d 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -147,6 +147,19 @@ hint #22 .endm +/* + * Speculation barrier + */ + .macro sb +alternative_if_not ARM64_HAS_SB + dsb nsh + isb +alternative_else + SB_BARRIER_INSN + nop +alternative_endif + .endm + /* * Sanitise a 64-bit bounded index wrt speculation, returning zero if out * of bounds. @@ -488,6 +501,7 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU .endm /* + * Deprecated! Use SYM_FUNC_{START,START_WEAK,END}_PI instead. * Annotate a function as position independent, i.e., safe to be called before * the kernel virtual mapping is activated. */ diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 822a9192c551..f66bb04fdf2d 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -34,6 +34,10 @@ #define psb_csync() asm volatile("hint #17" : : : "memory") #define csdb() asm volatile("hint #20" : : : "memory") +#define spec_bar() asm volatile(ALTERNATIVE("dsb nsh\nisb\n", \ + SB_BARRIER_INSN"nop\n", \ + ARM64_HAS_SB)) + #define mb() dsb(sy) #define rmb() dsb(ld) #define wmb() dsb(st) diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h index fd11e0d70e44..bb7b748e8067 100644 --- a/arch/arm64/include/asm/checksum.h +++ b/arch/arm64/include/asm/checksum.h @@ -16,7 +16,12 @@ #ifndef __ASM_CHECKSUM_H #define __ASM_CHECKSUM_H -#include +#include + +#define _HAVE_ARCH_IPV6_CSUM +__sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum sum); static inline __sum16 csum_fold(__wsum csum) { @@ -47,6 +52,9 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) } #define ip_fast_csum ip_fast_csum +extern unsigned int do_csum(const unsigned char *buff, int len); +#define do_csum do_csum + #include #endif /* __ASM_CHECKSUM_H */ diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 57b4863e5695..f10d0a97151b 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -57,8 +57,10 @@ #define ARM64_WORKAROUND_1542418 36 #define ARM64_WORKAROUND_1542419 37 #define ARM64_SPECTRE_BHB 38 +#define ARM64_HAS_CRC32 39 +#define ARM64_HAS_SB 40 -/* kabi: reserve 39 - 62 for future cpu capabilities */ +/* kabi: reserve 41 - 62 for future cpu capabilities */ #define ARM64_NCAPS 62 #endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 310e47d54d81..745fd80cff49 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -20,8 +20,7 @@ #define __ASM_INSN_H #include -/* A64 instructions are always 32 bits. */ -#define AARCH64_INSN_SIZE 4 +#include #ifndef __ASSEMBLY__ /* diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 1b266292f0be..2415aeb674fd 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -4,4 +4,28 @@ #define __ALIGN .align 2 #define __ALIGN_STR ".align 2" +/* + * Annotate a function as position independent, i.e., safe to be called before + * the kernel virtual mapping is activated. + */ +#define SYM_FUNC_START_PI(x) \ + SYM_FUNC_START_ALIAS(__pi_##x); \ + SYM_FUNC_START(x) + +#define SYM_FUNC_START_WEAK_PI(x) \ + SYM_FUNC_START_ALIAS(__pi_##x); \ + SYM_FUNC_START_WEAK(x) + +#define SYM_FUNC_START_WEAK_ALIAS_PI(x) \ + SYM_FUNC_START_ALIAS(__pi_##x); \ + SYM_START(x, SYM_L_WEAK, SYM_A_ALIGN) + +#define SYM_FUNC_END_PI(x) \ + SYM_FUNC_END(x); \ + SYM_FUNC_END_ALIAS(__pi_##x) + +#define SYM_FUNC_END_ALIAS_PI(x) \ + SYM_FUNC_END_ALIAS(x); \ + SYM_FUNC_END_ALIAS(__pi_##x) + #endif diff --git a/arch/arm64/include/asm/neon-intrinsics.h b/arch/arm64/include/asm/neon-intrinsics.h new file mode 100644 index 000000000000..2ba6c6b9541f --- /dev/null +++ b/arch/arm64/include/asm/neon-intrinsics.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2018 Linaro, Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __ASM_NEON_INTRINSICS_H +#define __ASM_NEON_INTRINSICS_H + +#include + +/* + * In the kernel, u64/s64 are [un]signed long long, not [un]signed long. + * So by redefining these macros to the former, we can force gcc-stdint.h + * to define uint64_t / in64_t in a compatible manner. + */ + +#ifdef __INT64_TYPE__ +#undef __INT64_TYPE__ +#define __INT64_TYPE__ long long +#endif + +#ifdef __UINT64_TYPE__ +#undef __UINT64_TYPE__ +#define __UINT64_TYPE__ unsigned long long +#endif + +/* + * genksyms chokes on the ARM NEON instrinsics system header, but we + * don't export anything it defines anyway, so just disregard when + * genksyms execute. + */ +#ifndef __GENKSYMS__ +#include +#endif + +#endif /* __ASM_NEON_INTRINSICS_H */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index bed5f1e4e53b..819ce0391204 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -107,6 +107,11 @@ #define SET_PSTATE_UAO(x) __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift)) #define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift)) +#define __SYS_BARRIER_INSN(CRm, op2, Rt) \ + __emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f)) + +#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 7, 31) + #define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2) #define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2) #define SYS_DC_CISW sys_insn(1, 0, 7, 14, 2) @@ -532,6 +537,7 @@ #define ID_AA64ISAR0_AES_SHIFT 4 /* id_aa64isar1 */ +#define ID_AA64ISAR1_SB_SHIFT 36 #define ID_AA64ISAR1_LRCPC_SHIFT 20 #define ID_AA64ISAR1_FCMA_SHIFT 16 #define ID_AA64ISAR1_JSCVT_SHIFT 12 diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 16969bc49bb8..e3e14484454e 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -46,8 +46,7 @@ static inline void set_fs(mm_segment_t fs) * Prevent a mispredicted conditional call to set_fs from forwarding * the wrong address limit to access_ok under speculation. */ - dsb(nsh); - isb(); + spec_bar(); /* On user-mode return, check fs is correct */ set_thread_flag(TIF_FSCHECK); @@ -409,20 +408,34 @@ do { \ extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); #define raw_copy_from_user(to, from, n) \ ({ \ - __arch_copy_from_user((to), __uaccess_mask_ptr(from), (n)); \ + unsigned long __acfu_ret; \ + uaccess_enable_not_uao(); \ + __acfu_ret = __arch_copy_from_user((to), \ + __uaccess_mask_ptr(from), (n)); \ + uaccess_disable_not_uao(); \ + __acfu_ret; \ }) extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); #define raw_copy_to_user(to, from, n) \ ({ \ - __arch_copy_to_user(__uaccess_mask_ptr(to), (from), (n)); \ + unsigned long __actu_ret; \ + uaccess_enable_not_uao(); \ + __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to), \ + (from), (n)); \ + uaccess_disable_not_uao(); \ + __actu_ret; \ }) extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n); #define raw_copy_in_user(to, from, n) \ ({ \ - __arch_copy_in_user(__uaccess_mask_ptr(to), \ - __uaccess_mask_ptr(from), (n)); \ + unsigned long __aciu_ret; \ + uaccess_enable_not_uao(); \ + __aciu_ret = __arch_copy_in_user(__uaccess_mask_ptr(to), \ + __uaccess_mask_ptr(from), (n)); \ + uaccess_disable_not_uao(); \ + __aciu_ret; \ }) #define INLINE_COPY_TO_USER @@ -431,8 +444,11 @@ extern unsigned long __must_check __arch_copy_in_user(void __user *to, const voi extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n); static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(VERIFY_WRITE, to, n)) { + uaccess_enable_not_uao(); n = __arch_clear_user(__uaccess_mask_ptr(to), n); + uaccess_disable_not_uao(); + } return n; } #define clear_user __clear_user diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h new file mode 100644 index 000000000000..856386ad076c --- /dev/null +++ b/arch/arm64/include/asm/xor.h @@ -0,0 +1,73 @@ +/* + * arch/arm64/include/asm/xor.h + * + * Authors: Jackie Liu + * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include + +#ifdef CONFIG_KERNEL_MODE_NEON + +extern struct xor_block_template const xor_block_inner_neon; + +static void +xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +{ + kernel_neon_begin(); + xor_block_inner_neon.do_2(bytes, p1, p2); + kernel_neon_end(); +} + +static void +xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, + unsigned long *p3) +{ + kernel_neon_begin(); + xor_block_inner_neon.do_3(bytes, p1, p2, p3); + kernel_neon_end(); +} + +static void +xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, + unsigned long *p3, unsigned long *p4) +{ + kernel_neon_begin(); + xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4); + kernel_neon_end(); +} + +static void +xor_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, + unsigned long *p3, unsigned long *p4, unsigned long *p5) +{ + kernel_neon_begin(); + xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5); + kernel_neon_end(); +} + +static struct xor_block_template xor_block_arm64 = { + .name = "arm64_neon", + .do_2 = xor_neon_2, + .do_3 = xor_neon_3, + .do_4 = xor_neon_4, + .do_5 = xor_neon_5 +}; +#undef XOR_TRY_TEMPLATES +#define XOR_TRY_TEMPLATES \ + do { \ + xor_speed(&xor_block_8regs); \ + xor_speed(&xor_block_32regs); \ + if (cpu_has_neon()) { \ + xor_speed(&xor_block_arm64);\ + } \ + } while (0) + +#endif /* ! CONFIG_KERNEL_MODE_NEON */ diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 2bcd6e4f3474..7784f7cba16c 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -49,5 +49,6 @@ #define HWCAP_ILRCPC (1 << 26) #define HWCAP_FLAGM (1 << 27) #define HWCAP_SSBS (1 << 28) +#define HWCAP_SB (1 << 29) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index fefd240287ac..d1375df23267 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -143,6 +143,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_SB_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0), @@ -1353,6 +1354,25 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_ssbs, }, #endif + { + .desc = "CRC32 instructions", + .capability = ARM64_HAS_CRC32, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64ISAR0_EL1, + .field_pos = ID_AA64ISAR0_CRC32_SHIFT, + .min_field_value = 1, + }, + { + .desc = "Speculation barrier (SB)", + .capability = ARM64_HAS_SB, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64ISAR1_EL1, + .field_pos = ID_AA64ISAR1_SB_SHIFT, + .sign = FTR_UNSIGNED, + .min_field_value = 1, + }, {}, }; @@ -1407,6 +1427,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA), HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC), HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ILRCPC), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SB), HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE), diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 601c89363ac0..ea41f1cd85f6 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -89,6 +89,7 @@ static const char *const hwcap_str[] = { "ilrcpc", "flagm", "ssbs", + "sb", NULL }; diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 50e28bca0041..1561275ca5c2 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -359,10 +359,6 @@ alternative_else_nop_endif ldp x24, x25, [sp, #16 * 12] ldp x26, x27, [sp, #16 * 13] ldp x28, x29, [sp, #16 * 14] - /* - * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on eret context synchronization - * when returning from IPI handler, and when returning to user-space. - */ .if \el == 0 alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 @@ -384,6 +380,7 @@ alternative_else_nop_endif add sp, sp, #S_FRAME_SIZE // restore sp eret .endif + sb .endm .macro irq_stack_entry @@ -1086,6 +1083,7 @@ alternative_endif .endif add sp, sp, #S_FRAME_SIZE // restore sp eret + sb .endm .macro generate_tramp_vector, kpti, bhb diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 4343d708ce5d..9d397f0ce99d 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -102,6 +102,7 @@ alternative_else_nop_endif // Do not touch any register after this! eret + sb ENDPROC(__guest_enter) ENTRY(__guest_exit) diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index 01e518b82c49..d52519fdc39f 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -120,6 +120,7 @@ el1_sync: // Guest trapped into EL2 do_el2_call eret + sb el1_hvc_guest: /* @@ -174,6 +175,7 @@ wa_epilogue: mov x0, xzr add sp, sp, #16 eret + sb el1_trap: get_vcpu_ptr x1, x0 @@ -209,6 +211,7 @@ el2_error: restore_caller_saved_regs_vect eret + sb ENTRY(__hyp_do_panic) mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ @@ -217,6 +220,7 @@ ENTRY(__hyp_do_panic) ldr lr, =panic msr elr_el2, lr eret + sb ENDPROC(__hyp_do_panic) ENTRY(__hyp_panic) diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 33ce9a5c324b..e1db45de71cc 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,9 +1,15 @@ # SPDX-License-Identifier: GPL-2.0 lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_in_user.o copy_page.o \ - clear_page.o memchr.o memcpy.o memmove.o memset.o \ - memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ - strchr.o strrchr.o tishift.o + clear_page.o csum.o memchr.o memcpy.o \ + memset.o memcmp.o strcmp.o strncmp.o strlen.o \ + strnlen.o strchr.o strrchr.o tishift.o + +ifeq ($(CONFIG_KERNEL_MODE_NEON), y) +obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o +CFLAGS_REMOVE_xor-neon.o += -mgeneral-regs-only +CFLAGS_xor-neon.o += -ffreestanding +endif # Tell the compiler to treat all general purpose registers (with the # exception of the IP registers, which are already handled by the caller @@ -30,3 +36,5 @@ KCOV_INSTRUMENT_atomic_ll_sc.o := n UBSAN_SANITIZE_atomic_ll_sc.o := n lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o + +obj-$(CONFIG_CRC32) += crc32.o diff --git a/arch/arm64/lib/clear_page.S b/arch/arm64/lib/clear_page.S index ef08e905e35b..a759f50e0f58 100644 --- a/arch/arm64/lib/clear_page.S +++ b/arch/arm64/lib/clear_page.S @@ -25,7 +25,7 @@ * Parameters: * x0 - dest */ -ENTRY(clear_page) +SYM_FUNC_START(clear_page) mrs x1, dczid_el0 and w1, w1, #0xf mov x2, #4 @@ -36,4 +36,4 @@ ENTRY(clear_page) tst x0, #(PAGE_SIZE - 1) b.ne 1b ret -ENDPROC(clear_page) +SYM_FUNC_END(clear_page) diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S index 4374020c824a..4377947bf8ac 100644 --- a/arch/arm64/lib/clear_user.S +++ b/arch/arm64/lib/clear_user.S @@ -1,23 +1,9 @@ /* - * Based on arch/arm/lib/clear_user.S - * - * Copyright (C) 2012 ARM Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * Copyright (C) 2021 Arm Ltd. */ -#include -#include +#include +#include .text @@ -29,34 +15,41 @@ * * Alignment fixed up by hardware. */ -ENTRY(__arch_clear_user) - uaccess_enable_not_uao x2, x3, x4 - mov x2, x1 // save the size for fixup return + + .p2align 4 + // Alignment is for the loop, but since the prologue (including BTI) + // is also 16 bytes we can keep any padding outside the function +SYM_FUNC_START(__arch_clear_user) + add x2, x0, x1 subs x1, x1, #8 b.mi 2f 1: -uao_user_alternative 9f, str, sttr, xzr, x0, 8 +USER(9f, sttr xzr, [x0]) + add x0, x0, #8 subs x1, x1, #8 - b.pl 1b -2: adds x1, x1, #4 - b.mi 3f -uao_user_alternative 9f, str, sttr, wzr, x0, 4 - sub x1, x1, #4 -3: adds x1, x1, #2 - b.mi 4f -uao_user_alternative 9f, strh, sttrh, wzr, x0, 2 - sub x1, x1, #2 -4: adds x1, x1, #1 - b.mi 5f -uao_user_alternative 9f, strb, sttrb, wzr, x0, 0 + b.hi 1b +USER(9f, sttr xzr, [x2, #-8]) + mov x0, #0 + ret + +2: tbz x1, #2, 3f +USER(9f, sttr wzr, [x0]) +USER(8f, sttr wzr, [x2, #-4]) + mov x0, #0 + ret + +3: tbz x1, #1, 4f +USER(9f, sttrh wzr, [x0]) +4: tbz x1, #0, 5f +USER(7f, sttrb wzr, [x2, #-1]) 5: mov x0, #0 - uaccess_disable_not_uao x2, x3 ret -ENDPROC(__arch_clear_user) +SYM_FUNC_END(__arch_clear_user) .section .fixup,"ax" .align 2 -9: mov x0, x2 // return the original size - uaccess_disable_not_uao x2, x3 +7: sub x0, x2, #5 // Adjust for faulting on the final byte... +8: add x0, x0, #4 // ...or the second word of the 4-7 byte case +9: sub x0, x2, x0 ret .previous diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 7cd6eeaa216c..44a3513c7793 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -30,49 +30,47 @@ * x0 - bytes not copied */ - .macro ldrb1 ptr, regB, val - uao_user_alternative 9998f, ldrb, ldtrb, \ptr, \regB, \val + .macro ldrb1 reg, ptr, val + uao_user_alternative 9998f, ldrb, ldtrb, \reg, \ptr, \val .endm - .macro strb1 ptr, regB, val - strb \ptr, [\regB], \val + .macro strb1 reg, ptr, val + strb \reg, [\ptr], \val .endm - .macro ldrh1 ptr, regB, val - uao_user_alternative 9997f, ldrh, ldtrh, \ptr, \regB, \val + .macro ldrh1 reg, ptr, val + uao_user_alternative 9997f, ldrh, ldtrh, \reg, \ptr, \val .endm - .macro strh1 ptr, regB, val - strh \ptr, [\regB], \val + .macro strh1 reg, ptr, val + strh \reg, [\ptr], \val .endm - .macro ldr1 ptr, regB, val - uao_user_alternative 9997f, ldr, ldtr, \ptr, \regB, \val + .macro ldr1 reg, ptr, val + uao_user_alternative 9997f, ldr, ldtr, \reg, \ptr, \val .endm - .macro str1 ptr, regB, val - str \ptr, [\regB], \val + .macro str1 reg, ptr, val + str \reg, [\ptr], \val .endm - .macro ldp1 ptr, regB, regC, val - uao_ldp 9997f, \ptr, \regB, \regC, \val + .macro ldp1 reg1, reg2, ptr, val + uao_ldp 9997f, \reg1, \reg2, \ptr, \val .endm - .macro stp1 ptr, regB, regC, val - stp \ptr, \regB, [\regC], \val + .macro stp1 reg1, reg2, ptr, val + stp \reg1, \reg2, [\ptr], \val .endm end .req x5 srcin .req x15 -ENTRY(__arch_copy_from_user) - uaccess_enable_not_uao x3, x4, x5 +SYM_FUNC_START(__arch_copy_from_user) add end, x0, x2 mov srcin, x1 #include "copy_template.S" - uaccess_disable_not_uao x3, x4 mov x0, #0 // Nothing to copy ret -ENDPROC(__arch_copy_from_user) +SYM_FUNC_END(__arch_copy_from_user) .section .fixup,"ax" .align 2 @@ -82,6 +80,5 @@ ENDPROC(__arch_copy_from_user) USER(9998f, ldtrb tmp1w, [srcin]) strb tmp1w, [dst], #1 9998: sub x0, end, dst // bytes not copied - uaccess_disable_not_uao x3, x4 ret .previous diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S index b20d3a0b3237..fc40f6d93ff3 100644 --- a/arch/arm64/lib/copy_in_user.S +++ b/arch/arm64/lib/copy_in_user.S @@ -31,50 +31,47 @@ * Returns: * x0 - bytes not copied */ - .macro ldrb1 ptr, regB, val - uao_user_alternative 9998f, ldrb, ldtrb, \ptr, \regB, \val + .macro ldrb1 reg, ptr, val + uao_user_alternative 9998f, ldrb, ldtrb, \reg, \ptr, \val .endm - .macro strb1 ptr, regB, val - uao_user_alternative 9998f, strb, sttrb, \ptr, \regB, \val + .macro strb1 reg, ptr, val + uao_user_alternative 9998f, strb, sttrb, \reg, \ptr, \val .endm - .macro ldrh1 ptr, regB, val - uao_user_alternative 9997f, ldrh, ldtrh, \ptr, \regB, \val + .macro ldrh1 reg, ptr, val + uao_user_alternative 9997f, ldrh, ldtrh, \reg, \ptr, \val .endm - .macro strh1 ptr, regB, val - uao_user_alternative 9997f, strh, sttrh, \ptr, \regB, \val + .macro strh1 reg, ptr, val + uao_user_alternative 9997f, strh, sttrh, \reg, \ptr, \val .endm - .macro ldr1 ptr, regB, val - uao_user_alternative 9997f, ldr, ldtr, \ptr, \regB, \val + .macro ldr1 reg, ptr, val + uao_user_alternative 9997f, ldr, ldtr, \reg, \ptr, \val .endm - .macro str1 ptr, regB, val - uao_user_alternative 9997f, str, sttr, \ptr, \regB, \val + .macro str1 reg, ptr, val + uao_user_alternative 9997f, str, sttr, \reg, \ptr, \val .endm - .macro ldp1 ptr, regB, regC, val - uao_ldp 9997f, \ptr, \regB, \regC, \val + .macro ldp1 reg1, reg2, ptr, val + uao_ldp 9997f, \reg1, \reg2, \ptr, \val .endm - .macro stp1 ptr, regB, regC, val - uao_stp 9997f, \ptr, \regB, \regC, \val + .macro stp1 reg1, reg2, ptr, val + uao_stp 9997f, \reg1, \reg2, \ptr, \val .endm end .req x5 srcin .req x15 - -ENTRY(__arch_copy_in_user) - uaccess_enable_not_uao x3, x4, x5 +SYM_FUNC_START(__arch_copy_in_user) add end, x0, x2 mov srcin, x1 #include "copy_template.S" - uaccess_disable_not_uao x3, x4 mov x0, #0 ret -ENDPROC(__arch_copy_in_user) +SYM_FUNC_END(__arch_copy_in_user) .section .fixup,"ax" .align 2 @@ -85,6 +82,5 @@ USER(9998f, ldtrb tmp1w, [srcin]) USER(9998f, sttrb tmp1w, [dst]) add dst, dst, #1 9998: sub x0, end, dst // bytes not copied - uaccess_disable_not_uao x3, x4 ret .previous diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S index 30f93161190f..1949962d524f 100644 --- a/arch/arm64/lib/copy_page.S +++ b/arch/arm64/lib/copy_page.S @@ -28,7 +28,7 @@ * x0 - dest * x1 - src */ -ENTRY(copy_page) +SYM_FUNC_START(copy_page) alternative_if ARM64_HAS_NO_HW_PREFETCH // Prefetch three cache lines ahead. prfm pldl1strm, [x1, #128] @@ -86,4 +86,4 @@ alternative_else_nop_endif stnp x16, x17, [x0, #112 - 256] ret -ENDPROC(copy_page) +SYM_FUNC_END(copy_page) diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index cfdbb1fe8d51..c08964cccac2 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -29,49 +29,47 @@ * Returns: * x0 - bytes not copied */ - .macro ldrb1 ptr, regB, val - ldrb \ptr, [\regB], \val + .macro ldrb1 reg, ptr, val + ldrb \reg, [\ptr], \val .endm - .macro strb1 ptr, regB, val - uao_user_alternative 9998f, strb, sttrb, \ptr, \regB, \val + .macro strb1 reg, ptr, val + uao_user_alternative 9998f, strb, sttrb, \reg, \ptr, \val .endm - .macro ldrh1 ptr, regB, val - ldrh \ptr, [\regB], \val + .macro ldrh1 reg, ptr, val + ldrh \reg, [\ptr], \val .endm - .macro strh1 ptr, regB, val - uao_user_alternative 9997f, strh, sttrh, \ptr, \regB, \val + .macro strh1 reg, ptr, val + uao_user_alternative 9997f, strh, sttrh, \reg, \ptr, \val .endm - .macro ldr1 ptr, regB, val - ldr \ptr, [\regB], \val + .macro ldr1 reg, ptr, val + ldr \reg, [\ptr], \val .endm - .macro str1 ptr, regB, val - uao_user_alternative 9997f, str, sttr, \ptr, \regB, \val + .macro str1 reg, ptr, val + uao_user_alternative 9997f, str, sttr, \reg, \ptr, \val .endm - .macro ldp1 ptr, regB, regC, val - ldp \ptr, \regB, [\regC], \val + .macro ldp1 reg1, reg2, ptr, val + ldp \reg1, \reg2, [\ptr], \val .endm - .macro stp1 ptr, regB, regC, val - uao_stp 9997f, \ptr, \regB, \regC, \val + .macro stp1 reg1, reg2, ptr, val + uao_stp 9997f, \reg1, \reg2, \ptr, \val .endm end .req x5 srcin .req x15 -ENTRY(__arch_copy_to_user) - uaccess_enable_not_uao x3, x4, x5 +SYM_FUNC_START(__arch_copy_to_user) add end, x0, x2 mov srcin, x1 #include "copy_template.S" - uaccess_disable_not_uao x3, x4 mov x0, #0 ret -ENDPROC(__arch_copy_to_user) +SYM_FUNC_END(__arch_copy_to_user) .section .fixup,"ax" .align 2 @@ -82,6 +80,5 @@ ENDPROC(__arch_copy_to_user) USER(9998f, sttrb tmp1w, [dst]) add dst, dst, #1 9998: sub x0, end, dst // bytes not copied - uaccess_disable_not_uao x3, x4 ret .previous diff --git a/arch/arm64/lib/crc32.S b/arch/arm64/lib/crc32.S new file mode 100644 index 000000000000..7db76f80731e --- /dev/null +++ b/arch/arm64/lib/crc32.S @@ -0,0 +1,157 @@ +/* + * Accelerated CRC32(C) using AArch64 CRC instructions + * + * Copyright (C) 2016 - 2018 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include + + .arch armv8-a+crc + + .macro byteorder, reg, be + .if \be +CPU_LE( rev \reg, \reg ) + .else +CPU_BE( rev \reg, \reg ) + .endif + .endm + + .macro byteorder16, reg, be + .if \be +CPU_LE( rev16 \reg, \reg ) + .else +CPU_BE( rev16 \reg, \reg ) + .endif + .endm + + .macro bitorder, reg, be + .if \be + rbit \reg, \reg + .endif + .endm + + .macro bitorder16, reg, be + .if \be + rbit \reg, \reg + lsr \reg, \reg, #16 + .endif + .endm + + .macro bitorder8, reg, be + .if \be + rbit \reg, \reg + lsr \reg, \reg, #24 + .endif + .endm + + .macro __crc32, c, be=0 + bitorder w0, \be + cmp x2, #16 + b.lt 8f // less than 16 bytes + + and x7, x2, #0x1f + and x2, x2, #~0x1f + cbz x7, 32f // multiple of 32 bytes + + and x8, x7, #0xf + ldp x3, x4, [x1] + add x8, x8, x1 + add x1, x1, x7 + ldp x5, x6, [x8] + byteorder x3, \be + byteorder x4, \be + byteorder x5, \be + byteorder x6, \be + bitorder x3, \be + bitorder x4, \be + bitorder x5, \be + bitorder x6, \be + + tst x7, #8 + crc32\c\()x w8, w0, x3 + csel x3, x3, x4, eq + csel w0, w0, w8, eq + tst x7, #4 + lsr x4, x3, #32 + crc32\c\()w w8, w0, w3 + csel x3, x3, x4, eq + csel w0, w0, w8, eq + tst x7, #2 + lsr w4, w3, #16 + crc32\c\()h w8, w0, w3 + csel w3, w3, w4, eq + csel w0, w0, w8, eq + tst x7, #1 + crc32\c\()b w8, w0, w3 + csel w0, w0, w8, eq + tst x7, #16 + crc32\c\()x w8, w0, x5 + crc32\c\()x w8, w8, x6 + csel w0, w0, w8, eq + cbz x2, 0f + +32: ldp x3, x4, [x1], #32 + sub x2, x2, #32 + ldp x5, x6, [x1, #-16] + byteorder x3, \be + byteorder x4, \be + byteorder x5, \be + byteorder x6, \be + bitorder x3, \be + bitorder x4, \be + bitorder x5, \be + bitorder x6, \be + crc32\c\()x w0, w0, x3 + crc32\c\()x w0, w0, x4 + crc32\c\()x w0, w0, x5 + crc32\c\()x w0, w0, x6 + cbnz x2, 32b +0: bitorder w0, \be + ret + +8: tbz x2, #3, 4f + ldr x3, [x1], #8 + byteorder x3, \be + bitorder x3, \be + crc32\c\()x w0, w0, x3 +4: tbz x2, #2, 2f + ldr w3, [x1], #4 + byteorder w3, \be + bitorder w3, \be + crc32\c\()w w0, w0, w3 +2: tbz x2, #1, 1f + ldrh w3, [x1], #2 + byteorder16 w3, \be + bitorder16 w3, \be + crc32\c\()h w0, w0, w3 +1: tbz x2, #0, 0f + ldrb w3, [x1] + bitorder8 w3, \be + crc32\c\()b w0, w0, w3 +0: bitorder w0, \be + ret + .endm + + .align 5 +SYM_FUNC_START(crc32_le) + __crc32 +SYM_FUNC_END(crc32_le) + + .align 5 +SYM_FUNC_START(__crc32c_le) + __crc32 c +SYM_FUNC_END(__crc32c_le) + + .align 5 +SYM_FUNC_START(crc32_be) +alternative_if_not ARM64_HAS_CRC32 + b crc32_be_base +alternative_else_nop_endif + __crc32 be=1 +SYM_FUNC_END(crc32_be) diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c new file mode 100644 index 000000000000..78b87a64ca0a --- /dev/null +++ b/arch/arm64/lib/csum.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2019-2020 Arm Ltd. + +#include +#include +#include + +#include + +/* Looks dumb, but generates nice-ish code */ +static u64 accumulate(u64 sum, u64 data) +{ + __uint128_t tmp = (__uint128_t)sum + data; + return tmp + (tmp >> 64); +} + +/* + * We over-read the buffer and this makes KASAN unhappy. Instead, disable + * instrumentation and call kasan explicitly. + */ +unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len) +{ + unsigned int offset, shift, sum; + const u64 *ptr; + u64 data, sum64 = 0; + + if (unlikely(len == 0)) + return 0; + + offset = (unsigned long)buff & 7; + /* + * This is to all intents and purposes safe, since rounding down cannot + * result in a different page or cache line being accessed, and @buff + * should absolutely not be pointing to anything read-sensitive. We do, + * however, have to be careful not to piss off KASAN, which means using + * unchecked reads to accommodate the head and tail, for which we'll + * compensate with an explicit check up-front. + */ + kasan_check_read(buff, len); + ptr = (u64 *)(buff - offset); + len = len + offset - 8; + + /* + * Head: zero out any excess leading bytes. Shifting back by the same + * amount should be at least as fast as any other way of handling the + * odd/even alignment, and means we can ignore it until the very end. + */ + shift = offset * 8; + data = *ptr++; +#ifdef __LITTLE_ENDIAN + data = (data >> shift) << shift; +#else + data = (data << shift) >> shift; +#endif + + /* + * Body: straightforward aligned loads from here on (the paired loads + * underlying the quadword type still only need dword alignment). The + * main loop strictly excludes the tail, so the second loop will always + * run at least once. + */ + while (unlikely(len > 64)) { + __uint128_t tmp1, tmp2, tmp3, tmp4; + + tmp1 = *(__uint128_t *)ptr; + tmp2 = *(__uint128_t *)(ptr + 2); + tmp3 = *(__uint128_t *)(ptr + 4); + tmp4 = *(__uint128_t *)(ptr + 6); + + len -= 64; + ptr += 8; + + /* This is the "don't dump the carry flag into a GPR" idiom */ + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + tmp2 += (tmp2 >> 64) | (tmp2 << 64); + tmp3 += (tmp3 >> 64) | (tmp3 << 64); + tmp4 += (tmp4 >> 64) | (tmp4 << 64); + tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64); + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64); + tmp3 += (tmp3 >> 64) | (tmp3 << 64); + tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64); + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + tmp1 = ((tmp1 >> 64) << 64) | sum64; + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + sum64 = tmp1 >> 64; + } + while (len > 8) { + __uint128_t tmp; + + sum64 = accumulate(sum64, data); + tmp = *(__uint128_t *)ptr; + + len -= 16; + ptr += 2; + +#ifdef __LITTLE_ENDIAN + data = tmp >> 64; + sum64 = accumulate(sum64, tmp); +#else + data = tmp; + sum64 = accumulate(sum64, tmp >> 64); +#endif + } + if (len > 0) { + sum64 = accumulate(sum64, data); + data = *ptr; + len -= 8; + } + /* + * Tail: zero any over-read bytes similarly to the head, again + * preserving odd/even alignment. + */ + shift = len * -8; +#ifdef __LITTLE_ENDIAN + data = (data << shift) >> shift; +#else + data = (data >> shift) << shift; +#endif + sum64 = accumulate(sum64, data); + + /* Finally, folding */ + sum64 += (sum64 >> 32) | (sum64 << 32); + sum = sum64 >> 32; + sum += (sum >> 16) | (sum << 16); + if (offset & 1) + return (u16)swab32(sum); + + return sum >> 16; +} + +__sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum csum) +{ + __uint128_t src, dst; + u64 sum = (__force u64)csum; + + src = *(const __uint128_t *)saddr->s6_addr; + dst = *(const __uint128_t *)daddr->s6_addr; + + sum += (__force u32)htonl(len); +#ifdef __LITTLE_ENDIAN + sum += (u32)proto << 24; +#else + sum += proto; +#endif + src += (src >> 64) | (src << 64); + dst += (dst >> 64) | (dst << 64); + + sum = accumulate(sum, src >> 64); + sum = accumulate(sum, dst >> 64); + + sum += ((sum >> 32) | (sum << 32)); + return csum_fold((__force __wsum)(sum >> 32)); +} +EXPORT_SYMBOL(csum_ipv6_magic); diff --git a/arch/arm64/lib/memchr.S b/arch/arm64/lib/memchr.S index 0f164a4baf52..6120c015b980 100644 --- a/arch/arm64/lib/memchr.S +++ b/arch/arm64/lib/memchr.S @@ -1,20 +1,5 @@ /* - * Based on arch/arm/lib/memchr.S - * - * Copyright (C) 1995-2000 Russell King - * Copyright (C) 2013 ARM Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * Copyright (C) 2021 Arm Ltd. */ #include @@ -30,15 +15,59 @@ * Returns: * x0 - address of first occurrence of 'c' or 0 */ -WEAK(memchr) - and w1, w1, #0xff -1: subs x2, x2, #1 - b.mi 2f - ldrb w3, [x0], #1 - cmp w3, w1 - b.ne 1b - sub x0, x0, #1 + +#define L(label) .L ## label + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f + +#define srcin x0 +#define chrin w1 +#define cntin x2 + +#define result x0 + +#define wordcnt x3 +#define rep01 x4 +#define repchr x5 +#define cur_word x6 +#define cur_byte w6 +#define tmp x7 +#define tmp2 x8 + + .p2align 4 + nop +SYM_FUNC_START_WEAK_PI(memchr) + and chrin, chrin, #0xff + lsr wordcnt, cntin, #3 + cbz wordcnt, L(byte_loop) + mov rep01, #REP8_01 + mul repchr, x1, rep01 + and cntin, cntin, #7 +L(word_loop): + ldr cur_word, [srcin], #8 + sub wordcnt, wordcnt, #1 + eor cur_word, cur_word, repchr + sub tmp, cur_word, rep01 + orr tmp2, cur_word, #REP8_7f + bics tmp, tmp, tmp2 + b.ne L(found_word) + cbnz wordcnt, L(word_loop) +L(byte_loop): + cbz cntin, L(not_found) + ldrb cur_byte, [srcin], #1 + sub cntin, cntin, #1 + cmp cur_byte, chrin + b.ne L(byte_loop) + sub srcin, srcin, #1 + ret +L(found_word): +CPU_LE( rev tmp, tmp) + clz tmp, tmp + sub tmp, tmp, #64 + add result, srcin, tmp, asr #3 ret -2: mov x0, #0 +L(not_found): + mov result, #0 ret -ENDPIPROC(memchr) +SYM_FUNC_END_PI(memchr) diff --git a/arch/arm64/lib/memcmp.S b/arch/arm64/lib/memcmp.S index fb295f52e9f8..490ceb7e8d66 100644 --- a/arch/arm64/lib/memcmp.S +++ b/arch/arm64/lib/memcmp.S @@ -1,258 +1,137 @@ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2013-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * and re-licensed under GPLv2 for the Linux kernel. The original code can - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/memcmp.S */ #include #include -/* -* compare memory areas(when two memory areas' offset are different, -* alignment handled by the hardware) -* -* Parameters: -* x0 - const memory area 1 pointer -* x1 - const memory area 2 pointer -* x2 - the maximal compare byte length -* Returns: -* x0 - a compare result, maybe less than, equal to, or greater than ZERO -*/ +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. + */ + +#define L(label) .L ## label /* Parameters and result. */ -src1 .req x0 -src2 .req x1 -limit .req x2 -result .req x0 +#define src1 x0 +#define src2 x1 +#define limit x2 +#define result w0 /* Internal variables. */ -data1 .req x3 -data1w .req w3 -data2 .req x4 -data2w .req w4 -has_nul .req x5 -diff .req x6 -endloop .req x7 -tmp1 .req x8 -tmp2 .req x9 -tmp3 .req x10 -pos .req x11 -limit_wd .req x12 -mask .req x13 - -WEAK(memcmp) - cbz limit, .Lret0 - eor tmp1, src1, src2 - tst tmp1, #7 - b.ne .Lmisaligned8 - ands tmp1, src1, #7 - b.ne .Lmutual_align - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ - lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ - /* - * The input source addresses are at alignment boundary. - * Directly compare eight bytes each time. - */ -.Lloop_aligned: - ldr data1, [src1], #8 - ldr data2, [src2], #8 -.Lstart_realigned: - subs limit_wd, limit_wd, #1 - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, cs /* Last Dword or differences. */ - cbz endloop, .Lloop_aligned - - /* Not reached the limit, must have found a diff. */ - tbz limit_wd, #63, .Lnot_limit - - /* Limit % 8 == 0 => the diff is in the last 8 bytes. */ - ands limit, limit, #7 - b.eq .Lnot_limit - /* - * The remained bytes less than 8. It is needed to extract valid data - * from last eight bytes of the intended memory range. - */ - lsl limit, limit, #3 /* bytes-> bits. */ - mov mask, #~0 -CPU_BE( lsr mask, mask, limit ) -CPU_LE( lsl mask, mask, limit ) - bic data1, data1, mask - bic data2, data2, mask - - orr diff, diff, mask - b .Lnot_limit - -.Lmutual_align: - /* - * Sources are mutually aligned, but are not currently at an - * alignment boundary. Round down the addresses and then mask off - * the bytes that precede the start point. - */ - bic src1, src1, #7 - bic src2, src2, #7 - ldr data1, [src1], #8 - ldr data2, [src2], #8 - /* - * We can not add limit with alignment offset(tmp1) here. Since the - * addition probably make the limit overflown. - */ - sub limit_wd, limit, #1/*limit != 0, so no underflow.*/ - and tmp3, limit_wd, #7 - lsr limit_wd, limit_wd, #3 - add tmp3, tmp3, tmp1 - add limit_wd, limit_wd, tmp3, lsr #3 - add limit, limit, tmp1/* Adjust the limit for the extra. */ - - lsl tmp1, tmp1, #3/* Bytes beyond alignment -> bits.*/ - neg tmp1, tmp1/* Bits to alignment -64. */ - mov tmp2, #~0 - /*mask off the non-intended bytes before the start address.*/ -CPU_BE( lsl tmp2, tmp2, tmp1 )/*Big-endian.Early bytes are at MSB*/ - /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp1 ) - - orr data1, data1, tmp2 - orr data2, data2, tmp2 - b .Lstart_realigned - - /*src1 and src2 have different alignment offset.*/ -.Lmisaligned8: - cmp limit, #8 - b.lo .Ltiny8proc /*limit < 8: compare byte by byte*/ - - and tmp1, src1, #7 - neg tmp1, tmp1 - add tmp1, tmp1, #8/*valid length in the first 8 bytes of src1*/ - and tmp2, src2, #7 - neg tmp2, tmp2 - add tmp2, tmp2, #8/*valid length in the first 8 bytes of src2*/ - subs tmp3, tmp1, tmp2 - csel pos, tmp1, tmp2, hi /*Choose the maximum.*/ - - sub limit, limit, pos - /*compare the proceeding bytes in the first 8 byte segment.*/ -.Ltinycmp: - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - subs pos, pos, #1 - ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */ - b.eq .Ltinycmp - cbnz pos, 1f /*diff occurred before the last byte.*/ - cmp data1w, data2w - b.eq .Lstart_align -1: - sub result, data1, data2 +#define data1 x3 +#define data1w w3 +#define data1h x4 +#define data2 x5 +#define data2w w5 +#define data2h x6 +#define tmp1 x7 +#define tmp2 x8 + +SYM_FUNC_START_WEAK_PI(memcmp) + subs limit, limit, 8 + b.lo L(less8) + + ldr data1, [src1], 8 + ldr data2, [src2], 8 + cmp data1, data2 + b.ne L(return) + + subs limit, limit, 8 + b.gt L(more16) + + ldr data1, [src1, limit] + ldr data2, [src2, limit] + b L(return) + +L(more16): + ldr data1, [src1], 8 + ldr data2, [src2], 8 + cmp data1, data2 + bne L(return) + + /* Jump directly to comparing the last 16 bytes for 32 byte (or less) + strings. */ + subs limit, limit, 16 + b.ls L(last_bytes) + + /* We overlap loads between 0-32 bytes at either side of SRC1 when we + try to align, so limit it only to strings larger than 128 bytes. */ + cmp limit, 96 + b.ls L(loop16) + + /* Align src1 and adjust src2 with bytes not yet done. */ + and tmp1, src1, 15 + add limit, limit, tmp1 + sub src1, src1, tmp1 + sub src2, src2, tmp1 + + /* Loop performing 16 bytes per iteration using aligned src1. + Limit is pre-decremented by 16 and must be larger than zero. + Exit if <= 16 bytes left to do or if the data is not equal. */ + .p2align 4 +L(loop16): + ldp data1, data1h, [src1], 16 + ldp data2, data2h, [src2], 16 + subs limit, limit, 16 + ccmp data1, data2, 0, hi + ccmp data1h, data2h, 0, eq + b.eq L(loop16) + + cmp data1, data2 + bne L(return) + mov data1, data1h + mov data2, data2h + cmp data1, data2 + bne L(return) + + /* Compare last 1-16 bytes using unaligned access. */ +L(last_bytes): + add src1, src1, limit + add src2, src2, limit + ldp data1, data1h, [src1] + ldp data2, data2h, [src2] + cmp data1, data2 + bne L(return) + mov data1, data1h + mov data2, data2h + cmp data1, data2 + + /* Compare data bytes and set return value to 0, -1 or 1. */ +L(return): +#ifndef __AARCH64EB__ + rev data1, data1 + rev data2, data2 +#endif + cmp data1, data2 +L(ret_eq): + cset result, ne + cneg result, result, lo ret -.Lstart_align: - lsr limit_wd, limit, #3 - cbz limit_wd, .Lremain8 - - ands xzr, src1, #7 - b.eq .Lrecal_offset - /*process more leading bytes to make src1 aligned...*/ - add src1, src1, tmp3 /*backwards src1 to alignment boundary*/ - add src2, src2, tmp3 - sub limit, limit, tmp3 - lsr limit_wd, limit, #3 - cbz limit_wd, .Lremain8 - /*load 8 bytes from aligned SRC1..*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - - subs limit_wd, limit_wd, #1 - eor diff, data1, data2 /*Non-zero if differences found.*/ - csinv endloop, diff, xzr, ne - cbnz endloop, .Lunequal_proc - /*How far is the current SRC2 from the alignment boundary...*/ - and tmp3, tmp3, #7 - -.Lrecal_offset:/*src1 is aligned now..*/ - neg pos, tmp3 -.Lloopcmp_proc: - /* - * Divide the eight bytes into two parts. First,backwards the src2 - * to an alignment boundary,load eight bytes and compare from - * the SRC2 alignment boundary. If all 8 bytes are equal,then start - * the second part's comparison. Otherwise finish the comparison. - * This special handle can garantee all the accesses are in the - * thread/task space in avoid to overrange access. - */ - ldr data1, [src1,pos] - ldr data2, [src2,pos] - eor diff, data1, data2 /* Non-zero if differences found. */ - cbnz diff, .Lnot_limit - - /*The second part process*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - eor diff, data1, data2 /* Non-zero if differences found. */ - subs limit_wd, limit_wd, #1 - csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ - cbz endloop, .Lloopcmp_proc -.Lunequal_proc: - cbz diff, .Lremain8 - -/* There is difference occurred in the latest comparison. */ -.Lnot_limit: -/* -* For little endian,reverse the low significant equal bits into MSB,then -* following CLZ can find how many equal bits exist. -*/ -CPU_LE( rev diff, diff ) -CPU_LE( rev data1, data1 ) -CPU_LE( rev data2, data2 ) - - /* - * The MS-non-zero bit of DIFF marks either the first bit - * that is different, or the end of the significant data. - * Shifting left now will bring the critical information into the - * top bits. - */ - clz pos, diff - lsl data1, data1, pos - lsl data2, data2, pos - /* - * We need to zero-extend (char is unsigned) the value and then - * perform a signed subtraction. - */ - lsr data1, data1, #56 - sub result, data1, data2, lsr #56 + .p2align 4 + /* Compare up to 8 bytes. Limit is [-8..-1]. */ +L(less8): + adds limit, limit, 4 + b.lo L(less4) + ldr data1w, [src1], 4 + ldr data2w, [src2], 4 + cmp data1w, data2w + b.ne L(return) + sub limit, limit, 4 +L(less4): + adds limit, limit, 4 + beq L(ret_eq) +L(byte_loop): + ldrb data1w, [src1], 1 + ldrb data2w, [src2], 1 + subs limit, limit, 1 + ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */ + b.eq L(byte_loop) + sub result, data1w, data2w ret -.Lremain8: - /* Limit % 8 == 0 =>. all data are equal.*/ - ands limit, limit, #7 - b.eq .Lret0 - -.Ltiny8proc: - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - subs limit, limit, #1 - - ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */ - b.eq .Ltiny8proc - sub result, data1, data2 - ret -.Lret0: - mov result, #0 - ret -ENDPIPROC(memcmp) +SYM_FUNC_END_PI(memcmp) diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S index dfedd4ab1a76..e7050b43e923 100644 --- a/arch/arm64/lib/memcpy.S +++ b/arch/arm64/lib/memcpy.S @@ -1,76 +1,247 @@ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2012-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * and re-licensed under GPLv2 for the Linux kernel. The original code can - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/memcpy.S */ #include #include -#include -/* - * Copy a buffer from src to dest (alignment handled by the hardware) +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. * - * Parameters: - * x0 - dest - * x1 - src - * x2 - n - * Returns: - * x0 - dest */ - .macro ldrb1 ptr, regB, val - ldrb \ptr, [\regB], \val - .endm - .macro strb1 ptr, regB, val - strb \ptr, [\regB], \val - .endm +#define L(label) .L ## label + +#define dstin x0 +#define src x1 +#define count x2 +#define dst x3 +#define srcend x4 +#define dstend x5 +#define A_l x6 +#define A_lw w6 +#define A_h x7 +#define B_l x8 +#define B_lw w8 +#define B_h x9 +#define C_l x10 +#define C_lw w10 +#define C_h x11 +#define D_l x12 +#define D_h x13 +#define E_l x14 +#define E_h x15 +#define F_l x16 +#define F_h x17 +#define G_l count +#define G_h dst +#define H_l src +#define H_h srcend +#define tmp1 x14 - .macro ldrh1 ptr, regB, val - ldrh \ptr, [\regB], \val - .endm +/* This implementation handles overlaps and supports both memcpy and memmove + from a single entry point. It uses unaligned accesses and branchless + sequences to keep the code small, simple and improve performance. - .macro strh1 ptr, regB, val - strh \ptr, [\regB], \val - .endm + Copies are split into 3 main cases: small copies of up to 32 bytes, medium + copies of up to 128 bytes, and large copies. The overhead of the overlap + check is negligible since it is only required for large copies. - .macro ldr1 ptr, regB, val - ldr \ptr, [\regB], \val - .endm + Large copies use a software pipelined loop processing 64 bytes per iteration. + The destination pointer is 16-byte aligned to minimize unaligned accesses. + The loop tail is handled by always copying 64 bytes from the end. +*/ - .macro str1 ptr, regB, val - str \ptr, [\regB], \val - .endm +SYM_FUNC_START_ALIAS(__memmove) +SYM_FUNC_START_WEAK_ALIAS_PI(memmove) +SYM_FUNC_START_ALIAS(__memcpy) +SYM_FUNC_START_WEAK_PI(memcpy) + add srcend, src, count + add dstend, dstin, count + cmp count, 128 + b.hi L(copy_long) + cmp count, 32 + b.hi L(copy32_128) - .macro ldp1 ptr, regB, regC, val - ldp \ptr, \regB, [\regC], \val - .endm + /* Small copies: 0..32 bytes. */ + cmp count, 16 + b.lo L(copy16) + ldp A_l, A_h, [src] + ldp D_l, D_h, [srcend, -16] + stp A_l, A_h, [dstin] + stp D_l, D_h, [dstend, -16] + ret + + /* Copy 8-15 bytes. */ +L(copy16): + tbz count, 3, L(copy8) + ldr A_l, [src] + ldr A_h, [srcend, -8] + str A_l, [dstin] + str A_h, [dstend, -8] + ret - .macro stp1 ptr, regB, regC, val - stp \ptr, \regB, [\regC], \val - .endm + .p2align 3 + /* Copy 4-7 bytes. */ +L(copy8): + tbz count, 2, L(copy4) + ldr A_lw, [src] + ldr B_lw, [srcend, -4] + str A_lw, [dstin] + str B_lw, [dstend, -4] + ret -ENTRY(__memcpy) -WEAK(memcpy) -#include "copy_template.S" + /* Copy 0..3 bytes using a branchless sequence. */ +L(copy4): + cbz count, L(copy0) + lsr tmp1, count, 1 + ldrb A_lw, [src] + ldrb C_lw, [srcend, -1] + ldrb B_lw, [src, tmp1] + strb A_lw, [dstin] + strb B_lw, [dstin, tmp1] + strb C_lw, [dstend, -1] +L(copy0): ret -ENDPIPROC(memcpy) -ENDPROC(__memcpy) + + .p2align 4 + /* Medium copies: 33..128 bytes. */ +L(copy32_128): + ldp A_l, A_h, [src] + ldp B_l, B_h, [src, 16] + ldp C_l, C_h, [srcend, -32] + ldp D_l, D_h, [srcend, -16] + cmp count, 64 + b.hi L(copy128) + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy 65..128 bytes. */ +L(copy128): + ldp E_l, E_h, [src, 32] + ldp F_l, F_h, [src, 48] + cmp count, 96 + b.ls L(copy96) + ldp G_l, G_h, [srcend, -64] + ldp H_l, H_h, [srcend, -48] + stp G_l, G_h, [dstend, -64] + stp H_l, H_h, [dstend, -48] +L(copy96): + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp E_l, E_h, [dstin, 32] + stp F_l, F_h, [dstin, 48] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy more than 128 bytes. */ +L(copy_long): + /* Use backwards copy if there is an overlap. */ + sub tmp1, dstin, src + cbz tmp1, L(copy0) + cmp tmp1, count + b.lo L(copy_long_backwards) + + /* Copy 16 bytes and then align dst to 16-byte alignment. */ + + ldp D_l, D_h, [src] + and tmp1, dstin, 15 + bic dst, dstin, 15 + sub src, src, tmp1 + add count, count, tmp1 /* Count is now 16 too large. */ + ldp A_l, A_h, [src, 16] + stp D_l, D_h, [dstin] + ldp B_l, B_h, [src, 32] + ldp C_l, C_h, [src, 48] + ldp D_l, D_h, [src, 64]! + subs count, count, 128 + 16 /* Test and readjust count. */ + b.ls L(copy64_from_end) + +L(loop64): + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [src, 16] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [src, 32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [src, 48] + stp D_l, D_h, [dst, 64]! + ldp D_l, D_h, [src, 64]! + subs count, count, 64 + b.hi L(loop64) + + /* Write the last iteration and copy 64 bytes from the end. */ +L(copy64_from_end): + ldp E_l, E_h, [srcend, -64] + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [srcend, -48] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [srcend, -32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [srcend, -16] + stp D_l, D_h, [dst, 64] + stp E_l, E_h, [dstend, -64] + stp A_l, A_h, [dstend, -48] + stp B_l, B_h, [dstend, -32] + stp C_l, C_h, [dstend, -16] + ret + + .p2align 4 + + /* Large backwards copy for overlapping copies. + Copy 16 bytes and then align dst to 16-byte alignment. */ +L(copy_long_backwards): + ldp D_l, D_h, [srcend, -16] + and tmp1, dstend, 15 + sub srcend, srcend, tmp1 + sub count, count, tmp1 + ldp A_l, A_h, [srcend, -16] + stp D_l, D_h, [dstend, -16] + ldp B_l, B_h, [srcend, -32] + ldp C_l, C_h, [srcend, -48] + ldp D_l, D_h, [srcend, -64]! + sub dstend, dstend, tmp1 + subs count, count, 128 + b.ls L(copy64_from_start) + +L(loop64_backwards): + stp A_l, A_h, [dstend, -16] + ldp A_l, A_h, [srcend, -16] + stp B_l, B_h, [dstend, -32] + ldp B_l, B_h, [srcend, -32] + stp C_l, C_h, [dstend, -48] + ldp C_l, C_h, [srcend, -48] + stp D_l, D_h, [dstend, -64]! + ldp D_l, D_h, [srcend, -64]! + subs count, count, 64 + b.hi L(loop64_backwards) + + /* Write the last iteration and copy 64 bytes from the start. */ +L(copy64_from_start): + ldp G_l, G_h, [src, 48] + stp A_l, A_h, [dstend, -16] + ldp A_l, A_h, [src, 32] + stp B_l, B_h, [dstend, -32] + ldp B_l, B_h, [src, 16] + stp C_l, C_h, [dstend, -48] + ldp C_l, C_h, [src] + stp D_l, D_h, [dstend, -64] + stp G_l, G_h, [dstin, 48] + stp A_l, A_h, [dstin, 32] + stp B_l, B_h, [dstin, 16] + stp C_l, C_h, [dstin] + ret + +SYM_FUNC_END_PI(memcpy) +SYM_FUNC_END_ALIAS(__memcpy) +SYM_FUNC_END_ALIAS_PI(memmove) +SYM_FUNC_END_ALIAS(__memmove) diff --git a/arch/arm64/lib/memmove.S b/arch/arm64/lib/memmove.S deleted file mode 100644 index e3de8f05c21a..000000000000 --- a/arch/arm64/lib/memmove.S +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. - * - * This code is based on glibc cortex strings work originally authored by Linaro - * and re-licensed under GPLv2 for the Linux kernel. The original code can - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include -#include -#include - -/* - * Move a buffer from src to test (alignment handled by the hardware). - * If dest <= src, call memcpy, otherwise copy in reverse order. - * - * Parameters: - * x0 - dest - * x1 - src - * x2 - n - * Returns: - * x0 - dest - */ -dstin .req x0 -src .req x1 -count .req x2 -tmp1 .req x3 -tmp1w .req w3 -tmp2 .req x4 -tmp2w .req w4 -tmp3 .req x5 -tmp3w .req w5 -dst .req x6 - -A_l .req x7 -A_h .req x8 -B_l .req x9 -B_h .req x10 -C_l .req x11 -C_h .req x12 -D_l .req x13 -D_h .req x14 - -ENTRY(__memmove) -WEAK(memmove) - cmp dstin, src - b.lo __memcpy - add tmp1, src, count - cmp dstin, tmp1 - b.hs __memcpy /* No overlap. */ - - add dst, dstin, count - add src, src, count - cmp count, #16 - b.lo .Ltail15 /*probably non-alignment accesses.*/ - - ands tmp2, src, #15 /* Bytes to reach alignment. */ - b.eq .LSrcAligned - sub count, count, tmp2 - /* - * process the aligned offset length to make the src aligned firstly. - * those extra instructions' cost is acceptable. It also make the - * coming accesses are based on aligned address. - */ - tbz tmp2, #0, 1f - ldrb tmp1w, [src, #-1]! - strb tmp1w, [dst, #-1]! -1: - tbz tmp2, #1, 2f - ldrh tmp1w, [src, #-2]! - strh tmp1w, [dst, #-2]! -2: - tbz tmp2, #2, 3f - ldr tmp1w, [src, #-4]! - str tmp1w, [dst, #-4]! -3: - tbz tmp2, #3, .LSrcAligned - ldr tmp1, [src, #-8]! - str tmp1, [dst, #-8]! - -.LSrcAligned: - cmp count, #64 - b.ge .Lcpy_over64 - - /* - * Deal with small copies quickly by dropping straight into the - * exit block. - */ -.Ltail63: - /* - * Copy up to 48 bytes of data. At this point we only need the - * bottom 6 bits of count to be accurate. - */ - ands tmp1, count, #0x30 - b.eq .Ltail15 - cmp tmp1w, #0x20 - b.eq 1f - b.lt 2f - ldp A_l, A_h, [src, #-16]! - stp A_l, A_h, [dst, #-16]! -1: - ldp A_l, A_h, [src, #-16]! - stp A_l, A_h, [dst, #-16]! -2: - ldp A_l, A_h, [src, #-16]! - stp A_l, A_h, [dst, #-16]! - -.Ltail15: - tbz count, #3, 1f - ldr tmp1, [src, #-8]! - str tmp1, [dst, #-8]! -1: - tbz count, #2, 2f - ldr tmp1w, [src, #-4]! - str tmp1w, [dst, #-4]! -2: - tbz count, #1, 3f - ldrh tmp1w, [src, #-2]! - strh tmp1w, [dst, #-2]! -3: - tbz count, #0, .Lexitfunc - ldrb tmp1w, [src, #-1] - strb tmp1w, [dst, #-1] - -.Lexitfunc: - ret - -.Lcpy_over64: - subs count, count, #128 - b.ge .Lcpy_body_large - /* - * Less than 128 bytes to copy, so handle 64 bytes here and then jump - * to the tail. - */ - ldp A_l, A_h, [src, #-16] - stp A_l, A_h, [dst, #-16] - ldp B_l, B_h, [src, #-32] - ldp C_l, C_h, [src, #-48] - stp B_l, B_h, [dst, #-32] - stp C_l, C_h, [dst, #-48] - ldp D_l, D_h, [src, #-64]! - stp D_l, D_h, [dst, #-64]! - - tst count, #0x3f - b.ne .Ltail63 - ret - - /* - * Critical loop. Start at a new cache line boundary. Assuming - * 64 bytes per line this ensures the entire loop is in one line. - */ - .p2align L1_CACHE_SHIFT -.Lcpy_body_large: - /* pre-load 64 bytes data. */ - ldp A_l, A_h, [src, #-16] - ldp B_l, B_h, [src, #-32] - ldp C_l, C_h, [src, #-48] - ldp D_l, D_h, [src, #-64]! -1: - /* - * interlace the load of next 64 bytes data block with store of the last - * loaded 64 bytes data. - */ - stp A_l, A_h, [dst, #-16] - ldp A_l, A_h, [src, #-16] - stp B_l, B_h, [dst, #-32] - ldp B_l, B_h, [src, #-32] - stp C_l, C_h, [dst, #-48] - ldp C_l, C_h, [src, #-48] - stp D_l, D_h, [dst, #-64]! - ldp D_l, D_h, [src, #-64]! - subs count, count, #64 - b.ge 1b - stp A_l, A_h, [dst, #-16] - stp B_l, B_h, [dst, #-32] - stp C_l, C_h, [dst, #-48] - stp D_l, D_h, [dst, #-64]! - - tst count, #0x3f - b.ne .Ltail63 - ret -ENDPIPROC(memmove) -ENDPROC(__memmove) diff --git a/arch/arm64/lib/memset.S b/arch/arm64/lib/memset.S index 316263c47c00..599d16d8c85a 100644 --- a/arch/arm64/lib/memset.S +++ b/arch/arm64/lib/memset.S @@ -54,8 +54,8 @@ dst .req x8 tmp3w .req w9 tmp3 .req x9 -ENTRY(__memset) -WEAK(memset) +SYM_FUNC_START_ALIAS(__memset) +SYM_FUNC_START_WEAK_PI(memset) mov dst, dstin /* Preserve return value. */ and A_lw, val, #255 orr A_lw, A_lw, A_lw, lsl #8 @@ -214,5 +214,5 @@ WEAK(memset) ands count, count, zva_bits_x b.ne .Ltail_maybe_long ret -ENDPIPROC(memset) -ENDPROC(__memset) +SYM_FUNC_END_PI(memset) +SYM_FUNC_END_ALIAS(__memset) diff --git a/arch/arm64/lib/strchr.S b/arch/arm64/lib/strchr.S index 7c83091d1bcd..8fb7c1cfcc4c 100644 --- a/arch/arm64/lib/strchr.S +++ b/arch/arm64/lib/strchr.S @@ -29,7 +29,7 @@ * Returns: * x0 - address of first occurrence of 'c' or 0 */ -WEAK(strchr) +SYM_FUNC_START_WEAK(strchr) and w1, w1, #0xff 1: ldrb w2, [x0], #1 cmp w2, w1 @@ -39,4 +39,4 @@ WEAK(strchr) cmp w2, w1 csel x0, x0, xzr, eq ret -ENDPROC(strchr) +SYM_FUNC_END(strchr) diff --git a/arch/arm64/lib/strcmp.S b/arch/arm64/lib/strcmp.S index 7d5d15398bfb..a03875d463a0 100644 --- a/arch/arm64/lib/strcmp.S +++ b/arch/arm64/lib/strcmp.S @@ -1,234 +1,188 @@ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2012-2022, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * and re-licensed under GPLv2 for the Linux kernel. The original code can - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strcmp.S */ #include #include -/* - * compare two strings +/* Assumptions: * - * Parameters: - * x0 - const string 1 pointer - * x1 - const string 2 pointer - * Returns: - * x0 - an integer less than, equal to, or greater than zero - * if s1 is found, respectively, to be less than, to match, - * or be greater than s2. + * ARMv8-a, AArch64. + * MTE compatible. */ +#define L(label) .L ## label + #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f -#define REP8_80 0x8080808080808080 - -/* Parameters and result. */ -src1 .req x0 -src2 .req x1 -result .req x0 - -/* Internal variables. */ -data1 .req x2 -data1w .req w2 -data2 .req x3 -data2w .req w3 -has_nul .req x4 -diff .req x5 -syndrome .req x6 -tmp1 .req x7 -tmp2 .req x8 -tmp3 .req x9 -zeroones .req x10 -pos .req x11 - -WEAK(strcmp) - eor tmp1, src1, src2 - mov zeroones, #REP8_01 - tst tmp1, #7 - b.ne .Lmisaligned8 - ands tmp1, src1, #7 - b.ne .Lmutual_align - - /* - * NUL detection works on the principle that (X - 1) & (~X) & 0x80 - * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - * can be done in parallel across the entire word. - */ -.Lloop_aligned: - ldr data1, [src1], #8 - ldr data2, [src2], #8 -.Lstart_realigned: - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + +#define src1 x0 +#define src2 x1 +#define result x0 + +#define data1 x2 +#define data1w w2 +#define data2 x3 +#define data2w w3 +#define has_nul x4 +#define diff x5 +#define off1 x5 +#define syndrome x6 +#define tmp x6 +#define data3 x7 +#define zeroones x8 +#define shift x9 +#define off2 x10 + +/* On big-endian early bytes are at MSB and on little-endian LSB. + LS_FW means shifting towards early bytes. */ +#ifdef __AARCH64EB__ +# define LS_FW lsl +#else +# define LS_FW lsr +#endif + +/* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. + Since carry propagation makes 0x1 bytes before a NUL byte appear + NUL too in big-endian, byte-reverse the data before the NUL check. */ + + +SYM_FUNC_START_WEAK_PI(strcmp) + sub off2, src2, src1 + mov zeroones, REP8_01 + and tmp, src1, 7 + tst off2, 7 + b.ne L(misaligned8) + cbnz tmp, L(mutual_align) + + .p2align 4 + +L(loop_aligned): + ldr data2, [src1, off2] + ldr data1, [src1], 8 +L(start_realigned): +#ifdef __AARCH64EB__ + rev tmp, data1 + sub has_nul, tmp, zeroones + orr tmp, tmp, REP8_7f +#else + sub has_nul, data1, zeroones + orr tmp, data1, REP8_7f +#endif + bics has_nul, has_nul, tmp /* Non-zero if NUL terminator. */ + ccmp data1, data2, 0, eq + b.eq L(loop_aligned) +#ifdef __AARCH64EB__ + rev has_nul, has_nul +#endif + eor diff, data1, data2 orr syndrome, diff, has_nul - cbz syndrome, .Lloop_aligned - b .Lcal_cmpresult - -.Lmutual_align: - /* - * Sources are mutually aligned, but are not currently at an - * alignment boundary. Round down the addresses and then mask off - * the bytes that preceed the start point. - */ - bic src1, src1, #7 - bic src2, src2, #7 - lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ - ldr data1, [src1], #8 - neg tmp1, tmp1 /* Bits to alignment -64. */ - ldr data2, [src2], #8 - mov tmp2, #~0 - /* Big-endian. Early bytes are at MSB. */ -CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ - /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ - - orr data1, data1, tmp2 - orr data2, data2, tmp2 - b .Lstart_realigned - -.Lmisaligned8: - /* - * Get the align offset length to compare per byte first. - * After this process, one string's address will be aligned. - */ - and tmp1, src1, #7 - neg tmp1, tmp1 - add tmp1, tmp1, #8 - and tmp2, src2, #7 - neg tmp2, tmp2 - add tmp2, tmp2, #8 - subs tmp3, tmp1, tmp2 - csel pos, tmp1, tmp2, hi /*Choose the maximum. */ -.Ltinycmp: - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - subs pos, pos, #1 - ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ - ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ - b.eq .Ltinycmp - cbnz pos, 1f /*find the null or unequal...*/ - cmp data1w, #1 - ccmp data1w, data2w, #0, cs - b.eq .Lstart_align /*the last bytes are equal....*/ -1: - sub result, data1, data2 +L(end): +#ifndef __AARCH64EB__ + rev syndrome, syndrome + rev data1, data1 + rev data2, data2 +#endif + clz shift, syndrome + /* The most-significant-non-zero bit of the syndrome marks either the + first bit that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ + lsl data1, data1, shift + lsl data2, data2, shift + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, 56 + sub result, data1, data2, lsr 56 ret -.Lstart_align: - ands xzr, src1, #7 - b.eq .Lrecal_offset - /*process more leading bytes to make str1 aligned...*/ - add src1, src1, tmp3 - add src2, src2, tmp3 - /*load 8 bytes from aligned str1 and non-aligned str2..*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul, tmp1, tmp2 - eor diff, data1, data2 /* Non-zero if differences found. */ - orr syndrome, diff, has_nul - cbnz syndrome, .Lcal_cmpresult - /*How far is the current str2 from the alignment boundary...*/ - and tmp3, tmp3, #7 -.Lrecal_offset: - neg pos, tmp3 -.Lloopcmp_proc: - /* - * Divide the eight bytes into two parts. First,backwards the src2 - * to an alignment boundary,load eight bytes from the SRC2 alignment - * boundary,then compare with the relative bytes from SRC1. - * If all 8 bytes are equal,then start the second part's comparison. - * Otherwise finish the comparison. - * This special handle can garantee all the accesses are in the - * thread/task space in avoid to overrange access. - */ - ldr data1, [src1,pos] - ldr data2, [src2,pos] - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul, tmp1, tmp2 - eor diff, data1, data2 /* Non-zero if differences found. */ - orr syndrome, diff, has_nul - cbnz syndrome, .Lcal_cmpresult - - /*The second part process*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul, tmp1, tmp2 - eor diff, data1, data2 /* Non-zero if differences found. */ + .p2align 4 + +L(mutual_align): + /* Sources are mutually aligned, but are not currently at an + alignment boundary. Round down the addresses and then mask off + the bytes that precede the start point. */ + bic src1, src1, 7 + ldr data2, [src1, off2] + ldr data1, [src1], 8 + neg shift, src2, lsl 3 /* Bits to alignment -64. */ + mov tmp, -1 + LS_FW tmp, tmp, shift + orr data1, data1, tmp + orr data2, data2, tmp + b L(start_realigned) + +L(misaligned8): + /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always + checking to make sure that we don't access beyond the end of SRC2. */ + cbz tmp, L(src1_aligned) +L(do_misaligned): + ldrb data1w, [src1], 1 + ldrb data2w, [src2], 1 + cmp data1w, 0 + ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */ + b.ne L(done) + tst src1, 7 + b.ne L(do_misaligned) + +L(src1_aligned): + neg shift, src2, lsl 3 + bic src2, src2, 7 + ldr data3, [src2], 8 +#ifdef __AARCH64EB__ + rev data3, data3 +#endif + lsr tmp, zeroones, shift + orr data3, data3, tmp + sub has_nul, data3, zeroones + orr tmp, data3, REP8_7f + bics has_nul, has_nul, tmp + b.ne L(tail) + + sub off1, src2, src1 + + .p2align 4 + +L(loop_unaligned): + ldr data3, [src1, off1] + ldr data2, [src1, off2] +#ifdef __AARCH64EB__ + rev data3, data3 +#endif + sub has_nul, data3, zeroones + orr tmp, data3, REP8_7f + ldr data1, [src1], 8 + bics has_nul, has_nul, tmp + ccmp data1, data2, 0, eq + b.eq L(loop_unaligned) + + lsl tmp, has_nul, shift +#ifdef __AARCH64EB__ + rev tmp, tmp +#endif + eor diff, data1, data2 + orr syndrome, diff, tmp + cbnz syndrome, L(end) +L(tail): + ldr data1, [src1] + neg shift, shift + lsr data2, data3, shift + lsr has_nul, has_nul, shift +#ifdef __AARCH64EB__ + rev data2, data2 + rev has_nul, has_nul +#endif + eor diff, data1, data2 orr syndrome, diff, has_nul - cbz syndrome, .Lloopcmp_proc - -.Lcal_cmpresult: - /* - * reversed the byte-order as big-endian,then CLZ can find the most - * significant zero bits. - */ -CPU_LE( rev syndrome, syndrome ) -CPU_LE( rev data1, data1 ) -CPU_LE( rev data2, data2 ) - - /* - * For big-endian we cannot use the trick with the syndrome value - * as carry-propagation can corrupt the upper bits if the trailing - * bytes in the string contain 0x01. - * However, if there is no NUL byte in the dword, we can generate - * the result directly. We ca not just subtract the bytes as the - * MSB might be significant. - */ -CPU_BE( cbnz has_nul, 1f ) -CPU_BE( cmp data1, data2 ) -CPU_BE( cset result, ne ) -CPU_BE( cneg result, result, lo ) -CPU_BE( ret ) -CPU_BE( 1: ) - /*Re-compute the NUL-byte detection, using a byte-reversed value. */ -CPU_BE( rev tmp3, data1 ) -CPU_BE( sub tmp1, tmp3, zeroones ) -CPU_BE( orr tmp2, tmp3, #REP8_7f ) -CPU_BE( bic has_nul, tmp1, tmp2 ) -CPU_BE( rev has_nul, has_nul ) -CPU_BE( orr syndrome, diff, has_nul ) - - clz pos, syndrome - /* - * The MS-non-zero bit of the syndrome marks either the first bit - * that is different, or the top bit of the first zero byte. - * Shifting left now will bring the critical information into the - * top bits. - */ - lsl data1, data1, pos - lsl data2, data2, pos - /* - * But we need to zero-extend (char is unsigned) the value and then - * perform a signed 32-bit subtraction. - */ - lsr data1, data1, #56 - sub result, data1, data2, lsr #56 + b L(end) + +L(done): + sub result, data1, data2 ret -ENDPIPROC(strcmp) + +SYM_FUNC_END_PI(strcmp) diff --git a/arch/arm64/lib/strlen.S b/arch/arm64/lib/strlen.S index 8e0b14205dcb..8f46c3858217 100644 --- a/arch/arm64/lib/strlen.S +++ b/arch/arm64/lib/strlen.S @@ -1,126 +1,201 @@ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2013-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * and re-licensed under GPLv2 for the Linux kernel. The original code can - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/98e4d6a5c13c8e54/string/aarch64/strlen.S */ #include #include -/* - * calculate the length of a string +/* Assumptions: * - * Parameters: - * x0 - const string pointer - * Returns: - * x0 - the return length of specific string + * ARMv8-a, AArch64, unaligned accesses, min page size 4k. */ +#define L(label) .L ## label + /* Arguments and results. */ -srcin .req x0 -len .req x0 +#define srcin x0 +#define len x0 /* Locals and temporaries. */ -src .req x1 -data1 .req x2 -data2 .req x3 -data2a .req x4 -has_nul1 .req x5 -has_nul2 .req x6 -tmp1 .req x7 -tmp2 .req x8 -tmp3 .req x9 -tmp4 .req x10 -zeroones .req x11 -pos .req x12 +#define src x1 +#define data1 x2 +#define data2 x3 +#define has_nul1 x4 +#define has_nul2 x5 +#define tmp1 x4 +#define tmp2 x5 +#define tmp3 x6 +#define tmp4 x7 +#define zeroones x8 + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. A faster check + (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives + false hits for characters 129..255. */ #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 -WEAK(strlen) - mov zeroones, #REP8_01 - bic src, srcin, #15 - ands tmp1, srcin, #15 - b.ne .Lmisaligned - /* - * NUL detection works on the principle that (X - 1) & (~X) & 0x80 - * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - * can be done in parallel across the entire word. - */ - /* - * The inner loop deals with two Dwords at a time. This has a - * slightly higher start-up cost, but we should win quite quickly, - * especially on cores with a high number of issue slots per - * cycle, as we get much better parallelism out of the operations. - */ -.Lloop: - ldp data1, data2, [src], #16 -.Lrealigned: +#define MIN_PAGE_SIZE 4096 + + /* Since strings are short on average, we check the first 16 bytes + of the string for a NUL character. In order to do an unaligned ldp + safely we have to do a page cross check first. If there is a NUL + byte we calculate the length from the 2 8-byte words using + conditional select to reduce branch mispredictions (it is unlikely + strlen will be repeatedly called on strings with the same length). + + If the string is longer than 16 bytes, we align src so don't need + further page cross checks, and process 32 bytes per iteration + using the fast NUL check. If we encounter non-ASCII characters, + fallback to a second loop using the full NUL check. + + If the page cross check fails, we read 16 bytes from an aligned + address, remove any characters before the string, and continue + in the main loop using aligned loads. Since strings crossing a + page in the first 16 bytes are rare (probability of + 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized. + + AArch64 systems have a minimum page size of 4k. We don't bother + checking for larger page sizes - the cost of setting up the correct + page size is just not worth the extra gain from a small reduction in + the cases taking the slow path. Note that we only care about + whether the first fetch, which may be misaligned, crosses a page + boundary. */ + +SYM_FUNC_START_WEAK_PI(strlen) + and tmp1, srcin, MIN_PAGE_SIZE - 1 + mov zeroones, REP8_01 + cmp tmp1, MIN_PAGE_SIZE - 16 + b.gt L(page_cross) + ldp data1, data2, [srcin] +#ifdef __AARCH64EB__ + /* For big-endian, carry propagation (if the final byte in the + string is 0x01) means we cannot use has_nul1/2 directly. + Since we expect strings to be small and early-exit, + byte-swap the data now so has_null1/2 will be correct. */ + rev data1, data1 + rev data2, data2 +#endif sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f + orr tmp2, data1, REP8_7f sub tmp3, data2, zeroones - orr tmp4, data2, #REP8_7f - bic has_nul1, tmp1, tmp2 - bics has_nul2, tmp3, tmp4 - ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ - b.eq .Lloop + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(main_loop_entry) + + /* Enter with C = has_nul1 == 0. */ + csel has_nul1, has_nul1, has_nul2, cc + mov len, 8 + rev has_nul1, has_nul1 + clz tmp1, has_nul1 + csel len, xzr, len, cc + add len, len, tmp1, lsr 3 + ret + /* The inner loop processes 32 bytes per iteration and uses the fast + NUL check. If we encounter non-ASCII characters, use a second + loop with the accurate NUL check. */ + .p2align 4 +L(main_loop_entry): + bic src, srcin, 15 + sub src, src, 16 +L(main_loop): + ldp data1, data2, [src, 32]! +L(page_cross_entry): + sub tmp1, data1, zeroones + sub tmp3, data2, zeroones + orr tmp2, tmp1, tmp3 + tst tmp2, zeroones, lsl 7 + bne 1f + ldp data1, data2, [src, 16] + sub tmp1, data1, zeroones + sub tmp3, data2, zeroones + orr tmp2, tmp1, tmp3 + tst tmp2, zeroones, lsl 7 + beq L(main_loop) + add src, src, 16 +1: + /* The fast check failed, so do the slower, accurate NUL check. */ + orr tmp2, data1, REP8_7f + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(nonascii_loop) + + /* Enter with C = has_nul1 == 0. */ +L(tail): +#ifdef __AARCH64EB__ + /* For big-endian, carry propagation (if the final byte in the + string is 0x01) means we cannot use has_nul1/2 directly. The + easiest way to get the correct byte is to byte-swap the data + and calculate the syndrome a second time. */ + csel data1, data1, data2, cc + rev data1, data1 + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + bic has_nul1, tmp1, tmp2 +#else + csel has_nul1, has_nul1, has_nul2, cc +#endif sub len, src, srcin - cbz has_nul1, .Lnul_in_data2 -CPU_BE( mov data2, data1 ) /*prepare data to re-calculate the syndrome*/ - sub len, len, #8 - mov has_nul2, has_nul1 -.Lnul_in_data2: - /* - * For big-endian, carry propagation (if the final byte in the - * string is 0x01) means we cannot use has_nul directly. The - * easiest way to get the correct byte is to byte-swap the data - * and calculate the syndrome a second time. - */ -CPU_BE( rev data2, data2 ) -CPU_BE( sub tmp1, data2, zeroones ) -CPU_BE( orr tmp2, data2, #REP8_7f ) -CPU_BE( bic has_nul2, tmp1, tmp2 ) - - sub len, len, #8 - rev has_nul2, has_nul2 - clz pos, has_nul2 - add len, len, pos, lsr #3 /* Bits to bytes. */ + rev has_nul1, has_nul1 + add tmp2, len, 8 + clz tmp1, has_nul1 + csel len, len, tmp2, cc + add len, len, tmp1, lsr 3 ret -.Lmisaligned: - cmp tmp1, #8 - neg tmp1, tmp1 - ldp data1, data2, [src], #16 - lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ - mov tmp2, #~0 - /* Big-endian. Early bytes are at MSB. */ -CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ +L(nonascii_loop): + ldp data1, data2, [src, 16]! + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + bne L(tail) + ldp data1, data2, [src, 16]! + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(nonascii_loop) + b L(tail) + + /* Load 16 bytes from [srcin & ~15] and force the bytes that precede + srcin to 0x7f, so we ignore any NUL bytes before the string. + Then continue in the aligned loop. */ +L(page_cross): + bic src, srcin, 15 + ldp data1, data2, [src] + lsl tmp1, srcin, 3 + mov tmp4, -1 +#ifdef __AARCH64EB__ + /* Big-endian. Early bytes are at MSB. */ + lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ +#else /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ - - orr data1, data1, tmp2 - orr data2a, data2, tmp2 - csinv data1, data1, xzr, le - csel data2, data2, data2a, le - b .Lrealigned -ENDPIPROC(strlen) + lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ +#endif + orr tmp1, tmp1, REP8_80 + orn data1, data1, tmp1 + orn tmp2, data2, tmp1 + tst srcin, 8 + csel data1, data1, tmp4, eq + csel data2, data2, tmp2, eq + b L(page_cross_entry) + +SYM_FUNC_END_PI(strlen) diff --git a/arch/arm64/lib/strncmp.S b/arch/arm64/lib/strncmp.S index 66bd145935d9..c82de5715cc1 100644 --- a/arch/arm64/lib/strncmp.S +++ b/arch/arm64/lib/strncmp.S @@ -1,310 +1,307 @@ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2013-2022, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * and re-licensed under GPLv2 for the Linux kernel. The original code can - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strncmp.S */ #include #include -/* - * compare two strings +/* Assumptions: * - * Parameters: - * x0 - const string 1 pointer - * x1 - const string 2 pointer - * x2 - the maximal length to be compared - * Returns: - * x0 - an integer less than, equal to, or greater than zero if s1 is found, - * respectively, to be less than, to match, or be greater than s2. + * ARMv8-a, AArch64. + * MTE compatible. */ +#define L(label) .L ## label + #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f -#define REP8_80 0x8080808080808080 /* Parameters and result. */ -src1 .req x0 -src2 .req x1 -limit .req x2 -result .req x0 +#define src1 x0 +#define src2 x1 +#define limit x2 +#define result x0 /* Internal variables. */ -data1 .req x3 -data1w .req w3 -data2 .req x4 -data2w .req w4 -has_nul .req x5 -diff .req x6 -syndrome .req x7 -tmp1 .req x8 -tmp2 .req x9 -tmp3 .req x10 -zeroones .req x11 -pos .req x12 -limit_wd .req x13 -mask .req x14 -endloop .req x15 +#define data1 x3 +#define data1w w3 +#define data2 x4 +#define data2w w4 +#define has_nul x5 +#define diff x6 +#define syndrome x7 +#define tmp1 x8 +#define tmp2 x9 +#define tmp3 x10 +#define zeroones x11 +#define pos x12 +#define mask x13 +#define endloop x14 +#define count mask +#define offset pos +#define neg_offset x15 -WEAK(strncmp) - cbz limit, .Lret0 +/* Define endian dependent shift operations. + On big-endian early bytes are at MSB and on little-endian LSB. + LS_FW means shifting towards early bytes. + LS_BK means shifting towards later bytes. + */ +#ifdef __AARCH64EB__ +#define LS_FW lsl +#define LS_BK lsr +#else +#define LS_FW lsr +#define LS_BK lsl +#endif + +SYM_FUNC_START_WEAK_PI(strncmp) + cbz limit, L(ret0) eor tmp1, src1, src2 mov zeroones, #REP8_01 tst tmp1, #7 - b.ne .Lmisaligned8 - ands tmp1, src1, #7 - b.ne .Lmutual_align - /* Calculate the number of full and partial words -1. */ - /* - * when limit is mulitply of 8, if not sub 1, - * the judgement of last dword will wrong. - */ - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ - lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ + and count, src1, #7 + b.ne L(misaligned8) + cbnz count, L(mutual_align) - /* - * NUL detection works on the principle that (X - 1) & (~X) & 0x80 - * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - * can be done in parallel across the entire word. - */ -.Lloop_aligned: + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ + .p2align 4 +L(loop_aligned): ldr data1, [src1], #8 ldr data2, [src2], #8 -.Lstart_realigned: - subs limit_wd, limit_wd, #1 +L(start_realigned): + subs limit, limit, #8 sub tmp1, data1, zeroones orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, pl /* Last Dword or differences.*/ - bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + eor diff, data1, data2 /* Non-zero if differences found. */ + csinv endloop, diff, xzr, hi /* Last Dword or differences. */ + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ ccmp endloop, #0, #0, eq - b.eq .Lloop_aligned - - /*Not reached the limit, must have found the end or a diff. */ - tbz limit_wd, #63, .Lnot_limit + b.eq L(loop_aligned) + /* End of main loop */ - /* Limit % 8 == 0 => all bytes significant. */ - ands limit, limit, #7 - b.eq .Lnot_limit +L(full_check): +#ifndef __AARCH64EB__ + orr syndrome, diff, has_nul + add limit, limit, 8 /* Rewind limit to before last subs. */ +L(syndrome_check): + /* Limit was reached. Check if the NUL byte or the difference + is before the limit. */ + rev syndrome, syndrome + rev data1, data1 + clz pos, syndrome + rev data2, data2 + lsl data1, data1, pos + cmp limit, pos, lsr #3 + lsl data2, data2, pos + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, #56 + sub result, data1, data2, lsr #56 + csel result, result, xzr, hi + ret +#else + /* Not reached the limit, must have found the end or a diff. */ + tbz limit, #63, L(not_limit) + add tmp1, limit, 8 + cbz limit, L(not_limit) - lsl limit, limit, #3 /* Bits -> bytes. */ + lsl limit, tmp1, #3 /* Bits -> bytes. */ mov mask, #~0 -CPU_BE( lsr mask, mask, limit ) -CPU_LE( lsl mask, mask, limit ) + lsr mask, mask, limit bic data1, data1, mask bic data2, data2, mask /* Make sure that the NUL byte is marked in the syndrome. */ orr has_nul, has_nul, mask -.Lnot_limit: +L(not_limit): + /* For big-endian we cannot use the trick with the syndrome value + as carry-propagation can corrupt the upper bits if the trailing + bytes in the string contain 0x01. */ + /* However, if there is no NUL byte in the dword, we can generate + the result directly. We can't just subtract the bytes as the + MSB might be significant. */ + cbnz has_nul, 1f + cmp data1, data2 + cset result, ne + cneg result, result, lo + ret +1: + /* Re-compute the NUL-byte detection, using a byte-reversed value. */ + rev tmp3, data1 + sub tmp1, tmp3, zeroones + orr tmp2, tmp3, #REP8_7f + bic has_nul, tmp1, tmp2 + rev has_nul, has_nul orr syndrome, diff, has_nul - b .Lcal_cmpresult + clz pos, syndrome + /* The most-significant-non-zero bit of the syndrome marks either the + first bit that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ +L(end_quick): + lsl data1, data1, pos + lsl data2, data2, pos + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, #56 + sub result, data1, data2, lsr #56 + ret +#endif -.Lmutual_align: - /* - * Sources are mutually aligned, but are not currently at an - * alignment boundary. Round down the addresses and then mask off - * the bytes that precede the start point. - * We also need to adjust the limit calculations, but without - * overflowing if the limit is near ULONG_MAX. - */ +L(mutual_align): + /* Sources are mutually aligned, but are not currently at an + alignment boundary. Round down the addresses and then mask off + the bytes that precede the start point. + We also need to adjust the limit calculations, but without + overflowing if the limit is near ULONG_MAX. */ bic src1, src1, #7 bic src2, src2, #7 ldr data1, [src1], #8 - neg tmp3, tmp1, lsl #3 /* 64 - bits(bytes beyond align). */ + neg tmp3, count, lsl #3 /* 64 - bits(bytes beyond align). */ ldr data2, [src2], #8 mov tmp2, #~0 - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ - /* Big-endian. Early bytes are at MSB. */ -CPU_BE( lsl tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */ - /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */ - - and tmp3, limit_wd, #7 - lsr limit_wd, limit_wd, #3 - /* Adjust the limit. Only low 3 bits used, so overflow irrelevant.*/ - add limit, limit, tmp1 - add tmp3, tmp3, tmp1 + LS_FW tmp2, tmp2, tmp3 /* Shift (count & 63). */ + /* Adjust the limit and ensure it doesn't overflow. */ + adds limit, limit, count + csinv limit, limit, xzr, lo orr data1, data1, tmp2 orr data2, data2, tmp2 - add limit_wd, limit_wd, tmp3, lsr #3 - b .Lstart_realigned + b L(start_realigned) -/*when src1 offset is not equal to src2 offset...*/ -.Lmisaligned8: - cmp limit, #8 - b.lo .Ltiny8proc /*limit < 8... */ - /* - * Get the align offset length to compare per byte first. - * After this process, one string's address will be aligned.*/ - and tmp1, src1, #7 - neg tmp1, tmp1 - add tmp1, tmp1, #8 - and tmp2, src2, #7 - neg tmp2, tmp2 - add tmp2, tmp2, #8 - subs tmp3, tmp1, tmp2 - csel pos, tmp1, tmp2, hi /*Choose the maximum. */ - /* - * Here, limit is not less than 8, so directly run .Ltinycmp - * without checking the limit.*/ - sub limit, limit, pos -.Ltinycmp: + .p2align 4 + /* Don't bother with dwords for up to 16 bytes. */ +L(misaligned8): + cmp limit, #16 + b.hs L(try_misaligned_words) + +L(byte_loop): + /* Perhaps we can do better than this. */ ldrb data1w, [src1], #1 ldrb data2w, [src2], #1 - subs pos, pos, #1 - ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ - ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ - b.eq .Ltinycmp - cbnz pos, 1f /*find the null or unequal...*/ - cmp data1w, #1 - ccmp data1w, data2w, #0, cs - b.eq .Lstart_align /*the last bytes are equal....*/ -1: + subs limit, limit, #1 + ccmp data1w, #1, #0, hi /* NZCV = 0b0000. */ + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ + b.eq L(byte_loop) +L(done): sub result, data1, data2 ret + /* Align the SRC1 to a dword by doing a bytewise compare and then do + the dword loop. */ +L(try_misaligned_words): + cbz count, L(src1_aligned) -.Lstart_align: - lsr limit_wd, limit, #3 - cbz limit_wd, .Lremain8 - /*process more leading bytes to make str1 aligned...*/ - ands xzr, src1, #7 - b.eq .Lrecal_offset - add src1, src1, tmp3 /*tmp3 is positive in this branch.*/ - add src2, src2, tmp3 - ldr data1, [src1], #8 - ldr data2, [src2], #8 + neg count, count + and count, count, #7 + sub limit, limit, count - sub limit, limit, tmp3 - lsr limit_wd, limit, #3 - subs limit_wd, limit_wd, #1 +L(page_end_loop): + ldrb data1w, [src1], #1 + ldrb data2w, [src2], #1 + cmp data1w, #1 + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ + b.ne L(done) + subs count, count, #1 + b.hi L(page_end_loop) - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ - bics has_nul, tmp1, tmp2 - ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/ - b.ne .Lunequal_proc - /*How far is the current str2 from the alignment boundary...*/ - and tmp3, tmp3, #7 -.Lrecal_offset: - neg pos, tmp3 -.Lloopcmp_proc: - /* - * Divide the eight bytes into two parts. First,backwards the src2 - * to an alignment boundary,load eight bytes from the SRC2 alignment - * boundary,then compare with the relative bytes from SRC1. - * If all 8 bytes are equal,then start the second part's comparison. - * Otherwise finish the comparison. - * This special handle can garantee all the accesses are in the - * thread/task space in avoid to overrange access. - */ - ldr data1, [src1,pos] - ldr data2, [src2,pos] - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, eq - cbnz endloop, .Lunequal_proc + /* The following diagram explains the comparison of misaligned strings. + The bytes are shown in natural order. For little-endian, it is + reversed in the registers. The "x" bytes are before the string. + The "|" separates data that is loaded at one time. + src1 | a a a a a a a a | b b b c c c c c | . . . + src2 | x x x x x a a a a a a a a b b b | c c c c c . . . + + After shifting in each step, the data looks like this: + STEP_A STEP_B STEP_C + data1 a a a a a a a a b b b c c c c c b b b c c c c c + data2 a a a a a a a a b b b 0 0 0 0 0 0 0 0 c c c c c - /*The second part process*/ + The bytes with "0" are eliminated from the syndrome via mask. + + Align SRC2 down to 16 bytes. This way we can read 16 bytes at a + time from SRC2. The comparison happens in 3 steps. After each step + the loop can exit, or read from SRC1 or SRC2. */ +L(src1_aligned): + /* Calculate offset from 8 byte alignment to string start in bits. No + need to mask offset since shifts are ignoring upper bits. */ + lsl offset, src2, #3 + bic src2, src2, #0xf + mov mask, -1 + neg neg_offset, offset ldr data1, [src1], #8 - ldr data2, [src2], #8 - subs limit_wd, limit_wd, #1 - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ - bics has_nul, tmp1, tmp2 - ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/ - b.eq .Lloopcmp_proc + ldp tmp1, tmp2, [src2], #16 + LS_BK mask, mask, neg_offset + and neg_offset, neg_offset, #63 /* Need actual value for cmp later. */ + /* Skip the first compare if data in tmp1 is irrelevant. */ + tbnz offset, 6, L(misaligned_mid_loop) -.Lunequal_proc: +L(loop_misaligned): + /* STEP_A: Compare full 8 bytes when there is enough data from SRC2.*/ + LS_FW data2, tmp1, offset + LS_BK tmp1, tmp2, neg_offset + subs limit, limit, #8 + orr data2, data2, tmp1 /* 8 bytes from SRC2 combined from two regs.*/ + sub has_nul, data1, zeroones + eor diff, data1, data2 /* Non-zero if differences found. */ + orr tmp3, data1, #REP8_7f + csinv endloop, diff, xzr, hi /* If limit, set to all ones. */ + bic has_nul, has_nul, tmp3 /* Non-zero if NUL byte found in SRC1. */ + orr tmp3, endloop, has_nul + cbnz tmp3, L(full_check) + + ldr data1, [src1], #8 +L(misaligned_mid_loop): + /* STEP_B: Compare first part of data1 to second part of tmp2. */ + LS_FW data2, tmp2, offset +#ifdef __AARCH64EB__ + /* For big-endian we do a byte reverse to avoid carry-propagation + problem described above. This way we can reuse the has_nul in the + next step and also use syndrome value trick at the end. */ + rev tmp3, data1 + #define data1_fixed tmp3 +#else + #define data1_fixed data1 +#endif + sub has_nul, data1_fixed, zeroones + orr tmp3, data1_fixed, #REP8_7f + eor diff, data2, data1 /* Non-zero if differences found. */ + bic has_nul, has_nul, tmp3 /* Non-zero if NUL terminator. */ +#ifdef __AARCH64EB__ + rev has_nul, has_nul +#endif + cmp limit, neg_offset, lsr #3 orr syndrome, diff, has_nul - cbz syndrome, .Lremain8 -.Lcal_cmpresult: - /* - * reversed the byte-order as big-endian,then CLZ can find the most - * significant zero bits. - */ -CPU_LE( rev syndrome, syndrome ) -CPU_LE( rev data1, data1 ) -CPU_LE( rev data2, data2 ) - /* - * For big-endian we cannot use the trick with the syndrome value - * as carry-propagation can corrupt the upper bits if the trailing - * bytes in the string contain 0x01. - * However, if there is no NUL byte in the dword, we can generate - * the result directly. We can't just subtract the bytes as the - * MSB might be significant. - */ -CPU_BE( cbnz has_nul, 1f ) -CPU_BE( cmp data1, data2 ) -CPU_BE( cset result, ne ) -CPU_BE( cneg result, result, lo ) -CPU_BE( ret ) -CPU_BE( 1: ) - /* Re-compute the NUL-byte detection, using a byte-reversed value.*/ -CPU_BE( rev tmp3, data1 ) -CPU_BE( sub tmp1, tmp3, zeroones ) -CPU_BE( orr tmp2, tmp3, #REP8_7f ) -CPU_BE( bic has_nul, tmp1, tmp2 ) -CPU_BE( rev has_nul, has_nul ) -CPU_BE( orr syndrome, diff, has_nul ) - /* - * The MS-non-zero bit of the syndrome marks either the first bit - * that is different, or the top bit of the first zero byte. - * Shifting left now will bring the critical information into the - * top bits. - */ - clz pos, syndrome - lsl data1, data1, pos - lsl data2, data2, pos - /* - * But we need to zero-extend (char is unsigned) the value and then - * perform a signed 32-bit subtraction. - */ - lsr data1, data1, #56 - sub result, data1, data2, lsr #56 - ret + bic syndrome, syndrome, mask /* Ignore later bytes. */ + csinv tmp3, syndrome, xzr, hi /* If limit, set to all ones. */ + cbnz tmp3, L(syndrome_check) -.Lremain8: - /* Limit % 8 == 0 => all bytes significant. */ - ands limit, limit, #7 - b.eq .Lret0 -.Ltiny8proc: - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - subs limit, limit, #1 + /* STEP_C: Compare second part of data1 to first part of tmp1. */ + ldp tmp1, tmp2, [src2], #16 + cmp limit, #8 + LS_BK data2, tmp1, neg_offset + eor diff, data2, data1 /* Non-zero if differences found. */ + orr syndrome, diff, has_nul + and syndrome, syndrome, mask /* Ignore earlier bytes. */ + csinv tmp3, syndrome, xzr, hi /* If limit, set to all ones. */ + cbnz tmp3, L(syndrome_check) - ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ - ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ - b.eq .Ltiny8proc - sub result, data1, data2 - ret + ldr data1, [src1], #8 + sub limit, limit, #8 + b L(loop_misaligned) + +#ifdef __AARCH64EB__ +L(syndrome_check): + clz pos, syndrome + cmp pos, limit, lsl #3 + b.lo L(end_quick) +#endif -.Lret0: +L(ret0): mov result, #0 ret -ENDPIPROC(strncmp) +SYM_FUNC_END_PI(strncmp) diff --git a/arch/arm64/lib/strnlen.S b/arch/arm64/lib/strnlen.S index 355be04441fe..634a44c2195d 100644 --- a/arch/arm64/lib/strnlen.S +++ b/arch/arm64/lib/strnlen.S @@ -59,7 +59,7 @@ limit_wd .req x14 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 -WEAK(strnlen) +SYM_FUNC_START_WEAK_PI(strnlen) cbz limit, .Lhit_limit mov zeroones, #REP8_01 bic src, srcin, #15 @@ -168,4 +168,4 @@ CPU_LE( lsr tmp2, tmp2, tmp4 ) /* Shift (tmp1 & 63). */ .Lhit_limit: mov len, limit ret -ENDPIPROC(strnlen) +SYM_FUNC_END_PI(strnlen) diff --git a/arch/arm64/lib/strrchr.S b/arch/arm64/lib/strrchr.S index ea84924d5990..1dc85d45fb09 100644 --- a/arch/arm64/lib/strrchr.S +++ b/arch/arm64/lib/strrchr.S @@ -29,7 +29,7 @@ * Returns: * x0 - address of last occurrence of 'c' or 0 */ -WEAK(strrchr) +SYM_FUNC_START_WEAK_PI(strrchr) mov x3, #0 and w1, w1, #0xff 1: ldrb w2, [x0], #1 @@ -40,4 +40,4 @@ WEAK(strrchr) b 1b 2: mov x0, x3 ret -ENDPIPROC(strrchr) +SYM_FUNC_END_PI(strrchr) diff --git a/arch/arm64/lib/tishift.S b/arch/arm64/lib/tishift.S index 0fdff97794de..820d1d0b4115 100644 --- a/arch/arm64/lib/tishift.S +++ b/arch/arm64/lib/tishift.S @@ -5,7 +5,7 @@ #include -ENTRY(__ashlti3) +SYM_FUNC_START(__ashlti3) cbz x2, 1f mov x3, #64 sub x3, x3, x2 @@ -24,9 +24,9 @@ ENTRY(__ashlti3) lsl x1, x0, x1 mov x0, x2 ret -ENDPROC(__ashlti3) +SYM_FUNC_END(__ashlti3) -ENTRY(__ashrti3) +SYM_FUNC_START(__ashrti3) cbz x2, 1f mov x3, #64 sub x3, x3, x2 @@ -45,9 +45,9 @@ ENTRY(__ashrti3) asr x0, x1, x0 mov x1, x2 ret -ENDPROC(__ashrti3) +SYM_FUNC_END(__ashrti3) -ENTRY(__lshrti3) +SYM_FUNC_START(__lshrti3) cbz x2, 1f mov x3, #64 sub x3, x3, x2 @@ -66,4 +66,4 @@ ENTRY(__lshrti3) lsr x0, x1, x0 mov x1, x2 ret -ENDPROC(__lshrti3) +SYM_FUNC_END(__lshrti3) diff --git a/arch/arm64/lib/uaccess_flushcache.c b/arch/arm64/lib/uaccess_flushcache.c index b6ceafdb8b72..64ed9296c5a7 100644 --- a/arch/arm64/lib/uaccess_flushcache.c +++ b/arch/arm64/lib/uaccess_flushcache.c @@ -39,7 +39,11 @@ void memcpy_page_flushcache(char *to, struct page *page, size_t offset, unsigned long __copy_user_flushcache(void *to, const void __user *from, unsigned long n) { - unsigned long rc = __arch_copy_from_user(to, from, n); + unsigned long rc; + + uaccess_enable_not_uao(); + rc = __arch_copy_from_user(to, from, n); + uaccess_disable_not_uao(); /* See above */ __clean_dcache_area_pop(to, n - rc); diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c new file mode 100644 index 000000000000..131c60c27dff --- /dev/null +++ b/arch/arm64/lib/xor-neon.c @@ -0,0 +1,184 @@ +/* + * arch/arm64/lib/xor-neon.c + * + * Authors: Jackie Liu + * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include + +void xor_arm64_neon_2(unsigned long bytes, unsigned long *p1, + unsigned long *p2) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + } while (--lines > 0); +} + +void xor_arm64_neon_3(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* p1 ^= p3 */ + v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + } while (--lines > 0); +} + +void xor_arm64_neon_4(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, unsigned long *p4) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* p1 ^= p3 */ + v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + } while (--lines > 0); +} + +void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4, unsigned long *p5) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + uint64_t *dp5 = (uint64_t *)p5; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* p1 ^= p3 */ + v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* p1 ^= p5 */ + v0 = veorq_u64(v0, vld1q_u64(dp5 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp5 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp5 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp5 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + dp5 += 8; + } while (--lines > 0); +} + +struct xor_block_template const xor_block_inner_neon = { + .name = "__inner_neon__", + .do_2 = xor_arm64_neon_2, + .do_3 = xor_arm64_neon_3, + .do_4 = xor_arm64_neon_4, + .do_5 = xor_arm64_neon_5, +}; +EXPORT_SYMBOL(xor_block_inner_neon); + +MODULE_AUTHOR("Jackie Liu "); +MODULE_DESCRIPTION("ARMv8 XOR Extensions"); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 000fb4465414..c183dbec6082 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -108,7 +108,7 @@ ENDPROC(flush_cache_all) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(__flush_icache_range) +SYM_FUNC_START(__flush_icache_range) /* FALLTHROUGH */ /* @@ -121,7 +121,7 @@ ENTRY(__flush_icache_range) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(__flush_cache_user_range) +SYM_FUNC_START(__flush_cache_user_range) uaccess_ttbr0_enable x2, x3, x4 alternative_if ARM64_HAS_CACHE_IDC dsb ishst @@ -150,8 +150,8 @@ alternative_else_nop_endif 9: mov x0, #-EFAULT b 1b -ENDPROC(__flush_icache_range) -ENDPROC(__flush_cache_user_range) +SYM_FUNC_END(__flush_icache_range) +SYM_FUNC_END(__flush_cache_user_range) /* * invalidate_icache_range(start,end) @@ -161,7 +161,7 @@ ENDPROC(__flush_cache_user_range) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(invalidate_icache_range) +SYM_FUNC_START(invalidate_icache_range) alternative_if ARM64_HAS_CACHE_DIC mov x0, xzr isb @@ -178,7 +178,7 @@ alternative_else_nop_endif 2: mov x0, #-EFAULT b 1b -ENDPROC(invalidate_icache_range) +SYM_FUNC_END(invalidate_icache_range) /* * __flush_dcache_area(kaddr, size) @@ -189,10 +189,10 @@ ENDPROC(invalidate_icache_range) * - kaddr - kernel address * - size - size in question */ -ENTRY(__flush_dcache_area) +SYM_FUNC_START_PI(__flush_dcache_area) dcache_by_line_op civac, sy, x0, x1, x2, x3 ret -ENDPIPROC(__flush_dcache_area) +SYM_FUNC_END_PI(__flush_dcache_area) /* * __clean_dcache_area_pou(kaddr, size) @@ -203,14 +203,14 @@ ENDPIPROC(__flush_dcache_area) * - kaddr - kernel address * - size - size in question */ -ENTRY(__clean_dcache_area_pou) +SYM_FUNC_START(__clean_dcache_area_pou) alternative_if ARM64_HAS_CACHE_IDC dsb ishst ret alternative_else_nop_endif dcache_by_line_op cvau, ish, x0, x1, x2, x3 ret -ENDPROC(__clean_dcache_area_pou) +SYM_FUNC_END(__clean_dcache_area_pou) /* * __inval_dcache_area(kaddr, size) @@ -222,7 +222,8 @@ ENDPROC(__clean_dcache_area_pou) * - kaddr - kernel address * - size - size in question */ -ENTRY(__inval_dcache_area) +ENTRY(__dma_inv_area) +SYM_FUNC_START_PI(__inval_dcache_area) /* FALLTHROUGH */ /* @@ -230,7 +231,6 @@ ENTRY(__inval_dcache_area) * - start - virtual start address of region * - size - size in question */ -ENTRY(__dma_inv_area) add x1, x1, x0 dcache_line_size x2, x3 sub x3, x2, #1 @@ -249,8 +249,8 @@ ENTRY(__dma_inv_area) b.lo 2b dsb sy ret -ENDPIPROC(__inval_dcache_area) -ENDPROC(__dma_inv_area) +SYM_FUNC_END_PI(__inval_dcache_area) +SYM_FUNC_END(__dma_inv_area) /* * __clean_dcache_area_poc(kaddr, size) @@ -261,7 +261,8 @@ ENDPROC(__dma_inv_area) * - kaddr - kernel address * - size - size in question */ -ENTRY(__clean_dcache_area_poc) +ENTRY(__dma_clean_area) +SYM_FUNC_START_PI(__clean_dcache_area_poc) /* FALLTHROUGH */ /* @@ -269,11 +270,10 @@ ENTRY(__clean_dcache_area_poc) * - start - virtual start address of region * - size - size in question */ -ENTRY(__dma_clean_area) dcache_by_line_op cvac, sy, x0, x1, x2, x3 ret -ENDPIPROC(__clean_dcache_area_poc) -ENDPROC(__dma_clean_area) +SYM_FUNC_END_PI(__clean_dcache_area_poc) +SYM_FUNC_END(__dma_clean_area) /* * __clean_dcache_area_pop(kaddr, size) @@ -284,13 +284,13 @@ ENDPROC(__dma_clean_area) * - kaddr - kernel address * - size - size in question */ -ENTRY(__clean_dcache_area_pop) +SYM_FUNC_START_PI(__clean_dcache_area_pop) alternative_if_not ARM64_HAS_DCPOP b __clean_dcache_area_poc alternative_else_nop_endif dcache_by_line_op cvap, sy, x0, x1, x2, x3 ret -ENDPIPROC(__clean_dcache_area_pop) +SYM_FUNC_END_PI(__clean_dcache_area_pop) /* * __dma_flush_area(start, size) @@ -300,10 +300,10 @@ ENDPIPROC(__clean_dcache_area_pop) * - start - virtual start address of region * - size - size in question */ -ENTRY(__dma_flush_area) +SYM_FUNC_START_PI(__dma_flush_area) dcache_by_line_op civac, sy, x0, x1, x2, x3 ret -ENDPIPROC(__dma_flush_area) +SYM_FUNC_END_PI(__dma_flush_area) /* * __dma_map_area(start, size, dir) @@ -311,11 +311,11 @@ ENDPIPROC(__dma_flush_area) * - size - size of region * - dir - DMA direction */ -ENTRY(__dma_map_area) +SYM_FUNC_START_PI(__dma_map_area) cmp w2, #DMA_FROM_DEVICE b.eq __dma_inv_area b __dma_clean_area -ENDPIPROC(__dma_map_area) +SYM_FUNC_END_PI(__dma_map_area) /* * __dma_unmap_area(start, size, dir) @@ -323,8 +323,8 @@ ENDPIPROC(__dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(__dma_unmap_area) +SYM_FUNC_START_PI(__dma_unmap_area) cmp w2, #DMA_TO_DEVICE b.ne __dma_inv_area ret -ENDPIPROC(__dma_unmap_area) +SYM_FUNC_END_PI(__dma_unmap_area) diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 1d7656761316..1695b40024da 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -431,6 +431,27 @@ void huge_ptep_clear_flush(struct vm_area_struct *vma, clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); } +static void __init add_huge_page_size(unsigned long size) +{ + if (size_to_hstate(size)) + return; + + hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); +} + +static int __init hugetlbpage_init(void) +{ +#ifdef CONFIG_ARM64_4K_PAGES + add_huge_page_size(PUD_SIZE); +#endif + add_huge_page_size(PMD_SIZE * CONT_PMDS); + add_huge_page_size(PMD_SIZE); + add_huge_page_size(PAGE_SIZE * CONT_PTES); + + return 0; +} +arch_initcall(hugetlbpage_init); + static __init int setup_hugepagesz(char *opt) { unsigned long ps = memparse(opt, &opt); @@ -442,7 +463,7 @@ static __init int setup_hugepagesz(char *opt) case PMD_SIZE * CONT_PMDS: case PMD_SIZE: case PAGE_SIZE * CONT_PTES: - hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); + add_huge_page_size(ps); return 1; } @@ -451,13 +472,3 @@ static __init int setup_hugepagesz(char *opt) return 0; } __setup("hugepagesz=", setup_hugepagesz); - -#ifdef CONFIG_ARM64_64K_PAGES -static __init int add_default_hugepagesz(void) -{ - if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) - hugetlb_add_hstate(CONT_PTE_SHIFT); - return 0; -} -arch_initcall(add_default_hugepagesz); -#endif diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index d40c37bf27ae..5d8f44903283 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -120,7 +120,7 @@ ENDPROC(cpu_do_idle) * * This must be kept in sync with struct cpu_suspend_ctx in . */ -ENTRY(cpu_do_suspend) +SYM_FUNC_START(cpu_do_suspend) mrs x2, tpidr_el0 mrs x3, tpidrro_el0 mrs x4, contextidr_el1 @@ -149,7 +149,7 @@ alternative_endif */ str x18, [x0, #96] ret -ENDPROC(cpu_do_suspend) +SYM_FUNC_END(cpu_do_suspend) /** * cpu_do_resume - restore CPU register context @@ -157,7 +157,7 @@ ENDPROC(cpu_do_suspend) * x0: Address of context pointer */ .pushsection ".idmap.text", "awx" -ENTRY(cpu_do_resume) +SYM_FUNC_START(cpu_do_resume) ldp x2, x3, [x0] ldp x4, x5, [x0, #16] ldp x6, x8, [x0, #32] @@ -213,7 +213,7 @@ alternative_else_nop_endif isb ret -ENDPROC(cpu_do_resume) +SYM_FUNC_END(cpu_do_resume) .popsection #endif @@ -224,7 +224,7 @@ ENDPROC(cpu_do_resume) * * - pgd_phys - physical address of new TTB */ -ENTRY(cpu_do_switch_mm) +SYM_FUNC_START(cpu_do_switch_mm) mrs x2, ttbr1_el1 mmid x1, x1 // get mm->context.id phys_to_ttbr x3, x0 @@ -237,7 +237,7 @@ ENTRY(cpu_do_switch_mm) msr ttbr0_el1, x3 // now update TTBR0 isb b post_ttbr_update_workaround // Back to C code... -ENDPROC(cpu_do_switch_mm) +SYM_FUNC_END(cpu_do_switch_mm) .pushsection ".idmap.text", "awx" @@ -257,7 +257,7 @@ ENDPROC(cpu_do_switch_mm) * This is the low-level counterpart to cpu_replace_ttbr1, and should not be * called by anything else. It can only be executed from a TTBR0 mapping. */ -ENTRY(idmap_cpu_replace_ttbr1) +SYM_FUNC_START(idmap_cpu_replace_ttbr1) save_and_disable_daif flags=x2 __idmap_cpu_set_reserved_ttbr1 x1, x3 @@ -269,7 +269,7 @@ ENTRY(idmap_cpu_replace_ttbr1) restore_daif x2 ret -ENDPROC(idmap_cpu_replace_ttbr1) +SYM_FUNC_END(idmap_cpu_replace_ttbr1) .popsection #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 @@ -297,7 +297,7 @@ ENDPROC(idmap_cpu_replace_ttbr1) */ __idmap_kpti_flag: .long 1 -ENTRY(idmap_kpti_install_ng_mappings) +SYM_FUNC_START(idmap_kpti_install_ng_mappings) cpu .req w0 num_cpus .req w1 swapper_pa .req x2 @@ -467,7 +467,7 @@ __idmap_kpti_secondary: .unreq swapper_ttb .unreq flag_ptr -ENDPROC(idmap_kpti_install_ng_mappings) +SYM_FUNC_END(idmap_kpti_install_ng_mappings) .popsection #endif @@ -478,7 +478,7 @@ ENDPROC(idmap_kpti_install_ng_mappings) * value of the SCTLR_EL1 register. */ .pushsection ".idmap.text", "awx" -ENTRY(__cpu_setup) +SYM_FUNC_START(__cpu_setup) tlbi vmalle1 // Invalidate local TLB dsb nsh @@ -539,4 +539,4 @@ ENTRY(__cpu_setup) #endif /* CONFIG_ARM64_HW_AFDBM */ msr tcr_el1, x10 ret // return to head.S -ENDPROC(__cpu_setup) +SYM_FUNC_END(__cpu_setup) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 5ae657d99f99..fe622cb354a9 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -938,6 +938,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bpf_jit_binary_free(header); prog->bpf_func = NULL; prog->jited = 0; + prog->jited_len = 0; goto out_off; } bpf_jit_binary_lock_ro(header); diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index 21f00349af52..b2fd1e2fbd89 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -308,7 +308,7 @@ comment "Processor Specific Options" config M68KFPU_EMU bool "Math emulation support" - depends on MMU + depends on M68KCLASSIC && FPU help At some point in the future, this will cause floating-point math instructions to be emulated by the kernel on machines that lack a diff --git a/arch/m68k/Kconfig.machine b/arch/m68k/Kconfig.machine index 0c451081432a..d07f3009a4a0 100644 --- a/arch/m68k/Kconfig.machine +++ b/arch/m68k/Kconfig.machine @@ -315,6 +315,7 @@ comment "Machine Options" config UBOOT bool "Support for U-Boot command line parameters" + depends on COLDFIRE help If you say Y here kernel will try to collect command line parameters from the initial u-boot stack. diff --git a/arch/m68k/include/asm/pgtable_no.h b/arch/m68k/include/asm/pgtable_no.h index fc3a96c77bd8..12f673707d4b 100644 --- a/arch/m68k/include/asm/pgtable_no.h +++ b/arch/m68k/include/asm/pgtable_no.h @@ -42,7 +42,8 @@ extern void paging_init(void); * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ -#define ZERO_PAGE(vaddr) (virt_to_page(0)) +extern void *empty_zero_page; +#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) /* * No page table caches to initialise. diff --git a/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h index 136d6d464e32..93c69fc7bbd8 100644 --- a/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h @@ -28,7 +28,6 @@ #define cpu_has_6k_cache 0 #define cpu_has_8k_cache 0 #define cpu_has_tx39_cache 0 -#define cpu_has_fpu 1 #define cpu_has_nofpuex 0 #define cpu_has_32fpr 1 #define cpu_has_counter 1 diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c index fcf9af492d60..cf46502c605e 100644 --- a/arch/mips/kernel/mips-cpc.c +++ b/arch/mips/kernel/mips-cpc.c @@ -31,6 +31,7 @@ phys_addr_t __weak mips_cpc_default_phys_base(void) cpc_node = of_find_compatible_node(of_root, NULL, "mti,mips-cpc"); if (cpc_node) { err = of_address_to_resource(cpc_node, 0, &res); + of_node_put(cpc_node); if (!err) return res.start; } diff --git a/arch/openrisc/include/asm/timex.h b/arch/openrisc/include/asm/timex.h index 9935cad1b9b9..34d015bf0462 100644 --- a/arch/openrisc/include/asm/timex.h +++ b/arch/openrisc/include/asm/timex.h @@ -27,6 +27,7 @@ static inline cycles_t get_cycles(void) { return mfspr(SPR_TTCR); } +#define get_cycles get_cycles /* This isn't really used any more */ #define CLOCK_TICK_RATE 1000 diff --git a/arch/openrisc/kernel/head.S b/arch/openrisc/kernel/head.S index 31ed257ff061..d7e49b947164 100644 --- a/arch/openrisc/kernel/head.S +++ b/arch/openrisc/kernel/head.S @@ -525,6 +525,15 @@ _start: l.ori r3,r0,0x1 l.mtspr r0,r3,SPR_SR + /* + * Start the TTCR as early as possible, so that the RNG can make use of + * measurements of boot time from the earliest opportunity. Especially + * important is that the TTCR does not return zero by the time we reach + * rand_initialize(). + */ + l.movhi r3,hi(SPR_TTMR_CR) + l.mtspr r0,r3,SPR_TTMR + CLEAR_GPR(r1) CLEAR_GPR(r2) CLEAR_GPR(r3) diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c index d7216c9abda1..ca79aacfeda2 100644 --- a/arch/powerpc/kernel/idle.c +++ b/arch/powerpc/kernel/idle.c @@ -41,7 +41,7 @@ static int __init powersave_off(char *arg) { ppc_md.power_save = NULL; cpuidle_disable = IDLE_POWERSAVE_OFF; - return 0; + return 1; } __setup("powersave=off", powersave_off); diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index e08b32ccf1d9..073117ba75fe 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -3007,8 +3007,13 @@ long arch_ptrace(struct task_struct *child, long request, flush_fp_to_thread(child); if (fpidx < (PT_FPSCR - PT_FPR0)) - memcpy(&tmp, &child->thread.TS_FPR(fpidx), - sizeof(long)); + if (IS_ENABLED(CONFIG_PPC32)) { + // On 32-bit the index we are passed refers to 32-bit words + tmp = ((u32 *)child->thread.fp_state.fpr)[fpidx]; + } else { + memcpy(&tmp, &child->thread.TS_FPR(fpidx), + sizeof(long)); + } else tmp = child->thread.fp_state.fpscr; } @@ -3040,8 +3045,13 @@ long arch_ptrace(struct task_struct *child, long request, flush_fp_to_thread(child); if (fpidx < (PT_FPSCR - PT_FPR0)) - memcpy(&child->thread.TS_FPR(fpidx), &data, - sizeof(long)); + if (IS_ENABLED(CONFIG_PPC32)) { + // On 32-bit the index we are passed refers to 32-bit words + ((u32 *)child->thread.fp_state.fpr)[fpidx] = data; + } else { + memcpy(&child->thread.TS_FPR(fpidx), &data, + sizeof(long)); + } else child->thread.fp_state.fpscr = data; ret = 0; diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index a1ff4142cc6a..08f2227761f6 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -322,7 +322,8 @@ int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp) if (event_is_threshold(event) && is_thresh_cmp_valid(event)) { mask |= CNST_THRESH_MASK; value |= CNST_THRESH_VAL(event >> EVENT_THRESH_SHIFT); - } + } else if (event_is_threshold(event)) + return -1; } else { /* * Special case for PM_MRK_FAB_RSP_MATCH and PM_MRK_FAB_RSP_MATCH_CYC, diff --git a/arch/powerpc/platforms/4xx/cpm.c b/arch/powerpc/platforms/4xx/cpm.c index 53ff81ca8a3c..6400ae376216 100644 --- a/arch/powerpc/platforms/4xx/cpm.c +++ b/arch/powerpc/platforms/4xx/cpm.c @@ -341,6 +341,6 @@ late_initcall(cpm_init); static int __init cpm_powersave_off(char *arg) { cpm.powersave_off = 1; - return 0; + return 1; } __setup("powersave=off", cpm_powersave_off); diff --git a/arch/powerpc/sysdev/cpm1.c b/arch/powerpc/sysdev/cpm1.c index 4f8dcf124828..f5b1ea2d6524 100644 --- a/arch/powerpc/sysdev/cpm1.c +++ b/arch/powerpc/sysdev/cpm1.c @@ -290,6 +290,7 @@ cpm_setbrg(uint brg, uint rate) out_be32(bp, (((BRG_UART_CLK_DIV16 / rate) - 1) << 1) | CPM_BRG_EN | CPM_BRG_DIV16); } +EXPORT_SYMBOL(cpm_setbrg); struct cpm_ioport16 { __be16 dir, par, odr_sor, dat, intr; diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c index 5011ffea4e4b..c48ebe677962 100644 --- a/arch/powerpc/sysdev/fsl_rio.c +++ b/arch/powerpc/sysdev/fsl_rio.c @@ -509,8 +509,10 @@ int fsl_rio_setup(struct platform_device *dev) if (rc) { dev_err(&dev->dev, "Can't get %pOF property 'reg'\n", rmu_node); + of_node_put(rmu_node); goto err_rmu; } + of_node_put(rmu_node); rmu_regs_win = ioremap(rmu_regs.start, resource_size(&rmu_regs)); if (!rmu_regs_win) { dev_err(&dev->dev, "Unable to map rmu register window\n"); diff --git a/arch/powerpc/sysdev/xics/icp-opal.c b/arch/powerpc/sysdev/xics/icp-opal.c index e3e52cf035a9..672d8aedae12 100644 --- a/arch/powerpc/sysdev/xics/icp-opal.c +++ b/arch/powerpc/sysdev/xics/icp-opal.c @@ -199,6 +199,7 @@ int icp_opal_init(void) printk("XICS: Using OPAL ICP fallbacks\n"); + of_node_put(np); return 0; } diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 2bc189187ed4..c663caf37ba4 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -861,7 +861,7 @@ static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw, unsigned int nbytes) { gw->walk_bytes_remain -= nbytes; - scatterwalk_unmap(&gw->walk); + scatterwalk_unmap(gw->walk_ptr); scatterwalk_advance(&gw->walk, nbytes); scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain); gw->walk_ptr = NULL; @@ -936,7 +936,7 @@ static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) goto out; } - scatterwalk_unmap(&gw->walk); + scatterwalk_unmap(gw->walk_ptr); gw->walk_ptr = NULL; gw->ptr = gw->buf; diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 23a14d187fb1..1aebf09fbcd8 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -50,10 +50,17 @@ static inline bool test_preempt_need_resched(void) static inline void __preempt_count_add(int val) { - if (__builtin_constant_p(val) && (val >= -128) && (val <= 127)) - __atomic_add_const(val, &S390_lowcore.preempt_count); - else - __atomic_add(val, &S390_lowcore.preempt_count); + /* + * With some obscure config options and CONFIG_PROFILE_ALL_BRANCHES + * enabled, gcc 12 fails to handle __builtin_constant_p(). + */ + if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES)) { + if (__builtin_constant_p(val) && (val >= -128) && (val <= 127)) { + __atomic_add_const(val, &S390_lowcore.preempt_count); + return; + } + } + __atomic_add(val, &S390_lowcore.preempt_count); } static inline void __preempt_count_sub(int val) diff --git a/arch/um/drivers/chan_user.c b/arch/um/drivers/chan_user.c index 669124d5290b..b3f320a129e3 100644 --- a/arch/um/drivers/chan_user.c +++ b/arch/um/drivers/chan_user.c @@ -220,7 +220,7 @@ static int winch_tramp(int fd, struct tty_port *port, int *fd_out, unsigned long *stack_out) { struct winch_data data; - int fds[2], n, err; + int fds[2], n, err, pid; char c; err = os_pipe(fds, 1, 1); @@ -238,8 +238,9 @@ static int winch_tramp(int fd, struct tty_port *port, int *fd_out, * problem with /dev/net/tun, which if held open by this * thread, prevents the TUN/TAP device from being reused. */ - err = run_helper_thread(winch_thread, &data, CLONE_FILES, stack_out); - if (err < 0) { + pid = run_helper_thread(winch_thread, &data, CLONE_FILES, stack_out); + if (pid < 0) { + err = pid; printk(UM_KERN_ERR "fork of winch_thread failed - errno = %d\n", -err); goto out_close; @@ -263,7 +264,7 @@ static int winch_tramp(int fd, struct tty_port *port, int *fd_out, goto out_close; } - return err; + return pid; out_close: close(fds[1]); diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index 3f9d43f26f63..47817937fd6e 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -329,7 +329,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) static __init int vdso_setup(char *s) { vdso64_enabled = simple_strtoul(s, NULL, 0); - return 0; + return 1; } __setup("vdso=", vdso_setup); diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 2d9e1372b070..d157d0adef06 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -324,6 +324,16 @@ static int perf_ibs_init(struct perf_event *event) hwc->config_base = perf_ibs->msr; hwc->config = config; + /* + * rip recorded by IbsOpRip will not be consistent with rsp and rbp + * recorded as part of interrupt regs. Thus we need to use rip from + * interrupt regs while unwinding call stack. Setting _EARLY flag + * makes sure we unwind call-stack before perf sample rip is set to + * IbsOpRip. + */ + if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) + event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; + return 0; } @@ -693,6 +703,14 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) data.raw = &raw; } + /* + * rip recorded by IbsOpRip will not be consistent with rsp and rbp + * recorded as part of interrupt regs. Thus we need to use rip from + * interrupt regs while unwinding call stack. + */ + if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) + data.callchain = perf_callchain(event, iregs); + throttle = perf_event_overflow(event, &data, ®s); out: if (throttle) { diff --git a/arch/x86/include/asm/acenv.h b/arch/x86/include/asm/acenv.h index 1b010a859b8b..6de59a4f723c 100644 --- a/arch/x86/include/asm/acenv.h +++ b/arch/x86/include/asm/acenv.h @@ -16,7 +16,19 @@ /* Asm macros */ -#define ACPI_FLUSH_CPU_CACHE() wbinvd() +/* + * ACPI_FLUSH_CPU_CACHE() flushes caches on entering sleep states. + * It is required to prevent data loss. + * + * While running inside virtual machine, the kernel can bypass cache flushing. + * Changing sleep state in a virtual machine doesn't affect the host system + * sleep state and cannot lead to data loss. + */ +#define ACPI_FLUSH_CPU_CACHE() \ +do { \ + if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) \ + wbinvd(); \ +} while (0) int __acpi_acquire_global_lock(unsigned int *lock); int __acpi_release_global_lock(unsigned int *lock); diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index 14caa9d9fb7f..e07188e8d763 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -13,9 +13,13 @@ #ifdef __ASSEMBLY__ -#define GLOBAL(name) \ - .globl name; \ - name: +/* + * GLOBAL is DEPRECATED + * + * use SYM_DATA_START, SYM_FUNC_START, SYM_INNER_LABEL, SYM_CODE_START, or + * similar + */ +#define GLOBAL(name) SYM_ENTRY(name, SYM_L_GLOBAL, SYM_A_NONE) #if defined(CONFIG_X86_64) || defined(CONFIG_X86_ALIGNMENT_16) #define __ALIGN .p2align 4, 0x90 diff --git a/arch/x86/include/asm/suspend_32.h b/arch/x86/include/asm/suspend_32.h index 8be6afb58471..32662cbaa27e 100644 --- a/arch/x86/include/asm/suspend_32.h +++ b/arch/x86/include/asm/suspend_32.h @@ -21,7 +21,6 @@ struct saved_context { #endif unsigned long cr0, cr2, cr3, cr4; u64 misc_enable; - bool misc_enable_saved; struct saved_msrs saved_msrs; struct desc_ptr gdt_desc; struct desc_ptr idt; @@ -30,6 +29,7 @@ struct saved_context { unsigned long tr; unsigned long safety; unsigned long return_address; + bool misc_enable_saved; } __attribute__((packed)); #endif /* _ASM_X86_SUSPEND_32_H */ diff --git a/arch/x86/include/asm/suspend_64.h b/arch/x86/include/asm/suspend_64.h index a7af9f53c0cb..b2861400c6a2 100644 --- a/arch/x86/include/asm/suspend_64.h +++ b/arch/x86/include/asm/suspend_64.h @@ -14,9 +14,13 @@ * Image of the saved processor state, used by the low level ACPI suspend to * RAM code and by the low level hibernation code. * - * If you modify it, fix arch/x86/kernel/acpi/wakeup_64.S and make sure that - * __save/__restore_processor_state(), defined in arch/x86/kernel/suspend_64.c, - * still work as required. + * If you modify it, check how it is used in arch/x86/kernel/acpi/wakeup_64.S + * and make sure that __save/__restore_processor_state(), defined in + * arch/x86/power/cpu.c, still work as required. + * + * Because the structure is packed, make sure to avoid unaligned members. For + * optimisation purposes but also because tools like kmemleak only search for + * pointers that are aligned. */ struct saved_context { struct pt_regs regs; @@ -36,7 +40,6 @@ struct saved_context { unsigned long cr0, cr2, cr3, cr4, cr8; u64 misc_enable; - bool misc_enable_saved; struct saved_msrs saved_msrs; unsigned long efer; u16 gdt_pad; /* Unused */ @@ -48,6 +51,7 @@ struct saved_context { unsigned long tr; unsigned long safety; unsigned long return_address; + bool misc_enable_saved; } __attribute__((packed)); #define loaddebug(thread,register) \ diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 9791828f3fcd..926939978c1c 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -166,7 +166,7 @@ static __init int setup_apicpmtimer(char *s) { apic_calibrate_pmtmr = 1; notsc_setup(NULL); - return 0; + return 1; } __setup("apicpmtimer", setup_apicpmtimer); #endif diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index a5287b18a63f..76692590eff8 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -71,7 +71,7 @@ static bool ring3mwait_disabled __read_mostly; static int __init ring3mwait_disable(char *__unused) { ring3mwait_disabled = true; - return 0; + return 1; } __setup("ring3mwait=disable", ring3mwait_disable); diff --git a/arch/x86/kernel/step.c b/arch/x86/kernel/step.c index 60d2c3798ba2..2f97d1a1032f 100644 --- a/arch/x86/kernel/step.c +++ b/arch/x86/kernel/step.c @@ -175,8 +175,7 @@ void set_task_blockstep(struct task_struct *task, bool on) * * NOTE: this means that set/clear TIF_BLOCKSTEP is only safe if * task is current or it can't be running, otherwise we can race - * with __switch_to_xtra(). We rely on ptrace_freeze_traced() but - * PTRACE_KILL is not safe. + * with __switch_to_xtra(). We rely on ptrace_freeze_traced(). */ local_irq_disable(); debugctl = get_debugctlmsr(); diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index 6a78d4b36a79..9fb266c797fb 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -70,9 +70,6 @@ static int __init control_va_addr_alignment(char *str) if (*str == 0) return 1; - if (*str == '=') - str++; - if (!strcmp(str, "32")) va_align.flags = ALIGN_VA_32; else if (!strcmp(str, "64")) @@ -82,11 +79,11 @@ static int __init control_va_addr_alignment(char *str) else if (!strcmp(str, "on")) va_align.flags = ALIGN_VA_32 | ALIGN_VA_64; else - return 0; + pr_warn("invalid option value: 'align_va_addr=%s'\n", str); return 1; } -__setup("align_va_addr", control_va_addr_alignment); +__setup("align_va_addr=", control_va_addr_alignment); SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c index 614c2c6b1959..68ca883abfdb 100644 --- a/arch/x86/lib/delay.c +++ b/arch/x86/lib/delay.c @@ -43,8 +43,8 @@ static void delay_loop(unsigned long loops) " jnz 2b \n" "3: dec %0 \n" - : /* we don't need output */ - :"a" (loops) + : "+a" (loops) + : ); } diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c index 324e26d0607b..b33304c0042a 100644 --- a/arch/x86/mm/pat.c +++ b/arch/x86/mm/pat.c @@ -74,7 +74,7 @@ int pat_debug_enable; static int __init pat_debug_setup(char *str) { pat_debug_enable = 1; - return 0; + return 1; } __setup("debugpat", pat_debug_setup); diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 22da9bfd8a45..bacf8d988f65 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -441,6 +441,11 @@ void __init xen_msi_init(void) x86_msi.setup_msi_irqs = xen_hvm_setup_msi_irqs; x86_msi.teardown_msi_irq = xen_teardown_msi_irq; + /* + * With XEN PIRQ/Eventchannels in use PCI/MSI[-X] masking is solely + * controlled by the hypervisor. + */ + pci_msi_ignore_mask = 1; } #endif diff --git a/arch/x86/um/ldt.c b/arch/x86/um/ldt.c index 3ee234b6234d..255a44dd415a 100644 --- a/arch/x86/um/ldt.c +++ b/arch/x86/um/ldt.c @@ -23,9 +23,11 @@ static long write_ldt_entry(struct mm_id *mm_idp, int func, { long res; void *stub_addr; + + BUILD_BUG_ON(sizeof(*desc) % sizeof(long)); + res = syscall_stub_data(mm_idp, (unsigned long *)desc, - (sizeof(*desc) + sizeof(long) - 1) & - ~(sizeof(long) - 1), + sizeof(*desc) / sizeof(long), addr, &stub_addr); if (!res) { unsigned long args[] = { func, diff --git a/arch/xtensa/kernel/ptrace.c b/arch/xtensa/kernel/ptrace.c index d9541be0605a..3cfcbdc456ae 100644 --- a/arch/xtensa/kernel/ptrace.c +++ b/arch/xtensa/kernel/ptrace.c @@ -35,12 +35,12 @@ void user_enable_single_step(struct task_struct *child) { - child->ptrace |= PT_SINGLESTEP; + set_tsk_thread_flag(child, TIF_SINGLESTEP); } void user_disable_single_step(struct task_struct *child) { - child->ptrace &= ~PT_SINGLESTEP; + clear_tsk_thread_flag(child, TIF_SINGLESTEP); } /* diff --git a/arch/xtensa/kernel/signal.c b/arch/xtensa/kernel/signal.c index f88e7a0b232c..72039bd61b72 100644 --- a/arch/xtensa/kernel/signal.c +++ b/arch/xtensa/kernel/signal.c @@ -455,7 +455,7 @@ static void do_signal(struct pt_regs *regs) /* Set up the stack frame */ ret = setup_frame(&ksig, sigmask_to_save(), regs); signal_setup_done(ret, &ksig, 0); - if (current->ptrace & PT_SINGLESTEP) + if (test_thread_flag(TIF_SINGLESTEP)) task_pt_regs(current)->icountlevel = 1; return; @@ -481,7 +481,7 @@ static void do_signal(struct pt_regs *regs) /* If there's no signal to deliver, we just restore the saved mask. */ restore_saved_sigmask(); - if (current->ptrace & PT_SINGLESTEP) + if (test_thread_flag(TIF_SINGLESTEP)) task_pt_regs(current)->icountlevel = 1; return; } diff --git a/block/bio.c b/block/bio.c index c6c92d207032..14a9bea974b7 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1533,7 +1533,7 @@ struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len, if (bytes > len) bytes = len; - page = alloc_page(q->bounce_gfp | gfp_mask); + page = alloc_page(q->bounce_gfp | __GFP_ZERO | gfp_mask); if (!page) goto cleanup; diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index ebc2bdcb1ec1..dda22231c4b0 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -85,7 +85,17 @@ struct iolatency_grp; struct blk_iolatency { struct rq_qos rqos; struct timer_list timer; - atomic_t enabled; + + /* + * ->enabled is the master enable switch gating the throttling logic and + * inflight tracking. The number of cgroups which have iolat enabled is + * tracked in ->enable_cnt, and ->enable is flipped on/off accordingly + * from ->enable_work with the request_queue frozen. For details, See + * blkiolatency_enable_work_fn(). + */ + bool enabled; + atomic_t enable_cnt; + struct work_struct enable_work; }; static inline struct blk_iolatency *BLKIOLATENCY(struct rq_qos *rqos) @@ -93,11 +103,6 @@ static inline struct blk_iolatency *BLKIOLATENCY(struct rq_qos *rqos) return container_of(rqos, struct blk_iolatency, rqos); } -static inline bool blk_iolatency_enabled(struct blk_iolatency *blkiolat) -{ - return atomic_read(&blkiolat->enabled) > 0; -} - struct child_latency_info { spinlock_t lock; @@ -402,7 +407,7 @@ static void blkcg_iolatency_throttle(struct rq_qos *rqos, struct bio *bio, struct request_queue *q = rqos->q; bool issue_as_root = bio_issue_as_root_blkg(bio); - if (!blk_iolatency_enabled(blkiolat)) + if (!blkiolat->enabled) return; rcu_read_lock(); @@ -561,7 +566,6 @@ static void blkcg_iolatency_done_bio(struct rq_qos *rqos, struct bio *bio) u64 window_start; u64 now = ktime_to_ns(ktime_get()); bool issue_as_root = bio_issue_as_root_blkg(bio); - bool enabled = false; int inflight = 0; blkg = bio->bi_blkg; @@ -572,8 +576,7 @@ static void blkcg_iolatency_done_bio(struct rq_qos *rqos, struct bio *bio) if (!iolat) return; - enabled = blk_iolatency_enabled(iolat->blkiolat); - if (!enabled) + if (!iolat->blkiolat->enabled) return; while (blkg && blkg->parent) { @@ -611,6 +614,7 @@ static void blkcg_iolatency_exit(struct rq_qos *rqos) struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos); del_timer_sync(&blkiolat->timer); + flush_work(&blkiolat->enable_work); blkcg_deactivate_policy(rqos->q, &blkcg_policy_iolatency); kfree(blkiolat); } @@ -682,6 +686,44 @@ static void blkiolatency_timer_fn(struct timer_list *t) rcu_read_unlock(); } +/** + * blkiolatency_enable_work_fn - Enable or disable iolatency on the device + * @work: enable_work of the blk_iolatency of interest + * + * iolatency needs to keep track of the number of in-flight IOs per cgroup. This + * is relatively expensive as it involves walking up the hierarchy twice for + * every IO. Thus, if iolatency is not enabled in any cgroup for the device, we + * want to disable the in-flight tracking. + * + * We have to make sure that the counting is balanced - we don't want to leak + * the in-flight counts by disabling accounting in the completion path while IOs + * are in flight. This is achieved by ensuring that no IO is in flight by + * freezing the queue while flipping ->enabled. As this requires a sleepable + * context, ->enabled flipping is punted to this work function. + */ +static void blkiolatency_enable_work_fn(struct work_struct *work) +{ + struct blk_iolatency *blkiolat = container_of(work, struct blk_iolatency, + enable_work); + bool enabled; + + /* + * There can only be one instance of this function running for @blkiolat + * and it's guaranteed to be executed at least once after the latest + * ->enabled_cnt modification. Acting on the latest ->enable_cnt is + * sufficient. + * + * Also, we know @blkiolat is safe to access as ->enable_work is flushed + * in blkcg_iolatency_exit(). + */ + enabled = atomic_read(&blkiolat->enable_cnt); + if (enabled != blkiolat->enabled) { + blk_mq_freeze_queue(blkiolat->rqos.q); + blkiolat->enabled = enabled; + blk_mq_unfreeze_queue(blkiolat->rqos.q); + } +} + int blk_iolatency_init(struct request_queue *q) { struct blk_iolatency *blkiolat; @@ -707,17 +749,15 @@ int blk_iolatency_init(struct request_queue *q) } timer_setup(&blkiolat->timer, blkiolatency_timer_fn, 0); + INIT_WORK(&blkiolat->enable_work, blkiolatency_enable_work_fn); return 0; } -/* - * return 1 for enabling iolatency, return -1 for disabling iolatency, otherwise - * return 0. - */ -static int iolatency_set_min_lat_nsec(struct blkcg_gq *blkg, u64 val) +static void iolatency_set_min_lat_nsec(struct blkcg_gq *blkg, u64 val) { struct iolatency_grp *iolat = blkg_to_lat(blkg); + struct blk_iolatency *blkiolat = iolat->blkiolat; u64 oldval = iolat->min_lat_nsec; iolat->min_lat_nsec = val; @@ -725,13 +765,15 @@ static int iolatency_set_min_lat_nsec(struct blkcg_gq *blkg, u64 val) iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec, BLKIOLATENCY_MAX_WIN_SIZE); - if (!oldval && val) - return 1; + if (!oldval && val) { + if (atomic_inc_return(&blkiolat->enable_cnt) == 1) + schedule_work(&blkiolat->enable_work); + } if (oldval && !val) { blkcg_clear_delay(blkg); - return -1; + if (atomic_dec_return(&blkiolat->enable_cnt) == 0) + schedule_work(&blkiolat->enable_work); } - return 0; } static void iolatency_clear_scaling(struct blkcg_gq *blkg) @@ -764,7 +806,6 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, u64 lat_val = 0; u64 oldval; int ret; - int enable = 0; ret = blkg_conf_prep(blkcg, &blkcg_policy_iolatency, buf, &ctx); if (ret) @@ -800,41 +841,12 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, blkg = ctx.blkg; oldval = iolat->min_lat_nsec; - enable = iolatency_set_min_lat_nsec(blkg, lat_val); - if (enable) { - if (!blk_get_queue(blkg->q)) { - ret = -ENODEV; - goto out; - } - - blkg_get(blkg); - } - - if (oldval != iolat->min_lat_nsec) { + iolatency_set_min_lat_nsec(blkg, lat_val); + if (oldval != iolat->min_lat_nsec) iolatency_clear_scaling(blkg); - } - ret = 0; out: blkg_conf_finish(&ctx); - if (ret == 0 && enable) { - struct iolatency_grp *tmp = blkg_to_lat(blkg); - struct blk_iolatency *blkiolat = tmp->blkiolat; - - blk_mq_freeze_queue(blkg->q); - - if (enable == 1) - atomic_inc(&blkiolat->enabled); - else if (enable == -1) - atomic_dec(&blkiolat->enabled); - else - WARN_ON_ONCE(1); - - blk_mq_unfreeze_queue(blkg->q); - - blkg_put(blkg); - blk_put_queue(blkg->q); - } return ret ?: nbytes; } @@ -934,14 +946,8 @@ static void iolatency_pd_offline(struct blkg_policy_data *pd) { struct iolatency_grp *iolat = pd_to_lat(pd); struct blkcg_gq *blkg = lat_to_blkg(iolat); - struct blk_iolatency *blkiolat = iolat->blkiolat; - int ret; - ret = iolatency_set_min_lat_nsec(blkg, 0); - if (ret == 1) - atomic_inc(&blkiolat->enabled); - if (ret == -1) - atomic_dec(&blkiolat->enabled); + iolatency_set_min_lat_nsec(blkg, 0); iolatency_clear_scaling(blkg); } diff --git a/crypto/drbg.c b/crypto/drbg.c index bc52d9562611..f490c332f066 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -113,7 +113,7 @@ * the SHA256 / AES 256 over other ciphers. Thus, the favored * DRBGs are the latest entries in this array. */ -static const struct drbg_core drbg_cores[] = { +const struct drbg_core drbg_cores[] = { #ifdef CONFIG_CRYPTO_DRBG_CTR { .flags = DRBG_CTR | DRBG_STRENGTH128, @@ -190,6 +190,7 @@ static const struct drbg_core drbg_cores[] = { }, #endif /* CONFIG_CRYPTO_DRBG_HMAC */ }; +EXPORT_SYMBOL(drbg_cores); static int drbg_uninstantiate(struct drbg_state *drbg); @@ -205,7 +206,7 @@ static int drbg_uninstantiate(struct drbg_state *drbg); * Return: normalized strength in *bytes* value or 32 as default * to counter programming errors */ -static inline unsigned short drbg_sec_strength(drbg_flag_t flags) +unsigned short drbg_sec_strength(drbg_flag_t flags) { switch (flags & DRBG_STRENGTH_MASK) { case DRBG_STRENGTH128: @@ -218,6 +219,7 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) return 32; } } +EXPORT_SYMBOL(drbg_sec_strength); /* * Convert an integer into a byte representation of this integer. @@ -1127,7 +1129,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, } /* Free all substructures in a DRBG state without the DRBG state structure */ -static inline void drbg_dealloc_state(struct drbg_state *drbg) +void drbg_dealloc_state(struct drbg_state *drbg) { if (!drbg) return; @@ -1143,12 +1145,13 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) drbg->d_ops = NULL; drbg->core = NULL; } +EXPORT_SYMBOL(drbg_dealloc_state); /* * Allocate all sub-structures for a DRBG state. * The DRBG state structure must already be allocated. */ -static inline int drbg_alloc_state(struct drbg_state *drbg) +int drbg_alloc_state(struct drbg_state *drbg) { int ret = -ENOMEM; unsigned int sb_size = 0; @@ -1219,6 +1222,7 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) drbg_dealloc_state(drbg); return ret; } +EXPORT_SYMBOL(drbg_alloc_state); /************************************************************************* * DRBG interface functions @@ -1787,8 +1791,7 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, * * return: flags */ -static inline void drbg_convert_tfm_core(const char *cra_driver_name, - int *coreref, bool *pr) +void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, bool *pr) { int i = 0; size_t start = 0; @@ -1815,6 +1818,7 @@ static inline void drbg_convert_tfm_core(const char *cra_driver_name, } } } +EXPORT_SYMBOL(drbg_convert_tfm_core); static int drbg_kcapi_init(struct crypto_tfm *tfm) { diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 787dccca3715..ed21fab13da6 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -43,24 +43,12 @@ #include #include #include - -struct rand_data; -int jent_read_entropy(struct rand_data *ec, unsigned char *data, - unsigned int len); -int jent_entropy_init(void); -struct rand_data *jent_entropy_collector_alloc(unsigned int osr, - unsigned int flags); -void jent_entropy_collector_free(struct rand_data *entropy_collector); +#include /*************************************************************************** * Helper function ***************************************************************************/ -__u64 jent_rol64(__u64 word, unsigned int shift) -{ - return rol64(word, shift); -} - void *jent_zalloc(unsigned int len) { return kzalloc(len, GFP_KERNEL); @@ -119,6 +107,7 @@ void jent_get_nstime(__u64 *out) struct jitterentropy { spinlock_t jent_lock; struct rand_data *entropy_collector; + unsigned int reset_cnt; }; static int jent_kcapi_init(struct crypto_tfm *tfm) @@ -153,7 +142,33 @@ static int jent_kcapi_random(struct crypto_rng *tfm, int ret = 0; spin_lock(&rng->jent_lock); + + /* Return a permanent error in case we had too many resets in a row. */ + if (rng->reset_cnt > (1<<10)) { + ret = -EFAULT; + goto out; + } + ret = jent_read_entropy(rng->entropy_collector, rdata, dlen); + + /* Reset RNG in case of health failures */ + if (ret < -1) { + pr_warn_ratelimited("Reset Jitter RNG due to health test failure: %s failure\n", + (ret == -2) ? "Repetition Count Test" : + "Adaptive Proportion Test"); + + rng->reset_cnt++; + + ret = -EAGAIN; + } else { + rng->reset_cnt = 0; + + /* Convert the Jitter RNG error into a usable error code */ + if (ret == -1) + ret = -EINVAL; + } + +out: spin_unlock(&rng->jent_lock); return ret; diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index acf44b2d2d1d..51813fa519fe 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -2,7 +2,7 @@ * Non-physical true random number generator based on timing jitter -- * Jitter RNG standalone code. * - * Copyright Stephan Mueller , 2015 + * Copyright Stephan Mueller , 2015 - 2020 * * Design * ====== @@ -47,7 +47,7 @@ /* * This Jitterentropy RNG is based on the jitterentropy library - * version 1.1.0 provided at http://www.chronox.de/jent.html + * version 2.2.0 provided at http://www.chronox.de/jent.html */ #ifdef __OPTIMIZE__ @@ -71,10 +71,7 @@ struct rand_data { #define DATA_SIZE_BITS ((sizeof(__u64)) * 8) __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ - unsigned int stuck:1; /* Time measurement stuck */ unsigned int osr; /* Oversample rate */ - unsigned int stir:1; /* Post-processing stirring */ - unsigned int disable_unbias:1; /* Deactivate Von-Neuman unbias */ #define JENT_MEMORY_BLOCKS 64 #define JENT_MEMORY_BLOCKSIZE 32 #define JENT_MEMORY_ACCESSLOOPS 128 @@ -86,11 +83,25 @@ struct rand_data { unsigned int memblocksize; /* Size of one memory block in bytes */ unsigned int memaccessloops; /* Number of memory accesses per random * bit generation */ + + /* Repetition Count Test */ + int rct_count; /* Number of stuck values */ + + /* Adaptive Proportion Test for a significance level of 2^-30 */ +#define JENT_APT_CUTOFF 325 /* Taken from SP800-90B sec 4.4.2 */ +#define JENT_APT_WINDOW_SIZE 512 /* Data window size */ + /* LSB of time stamp to process */ +#define JENT_APT_LSB 16 +#define JENT_APT_WORD_MASK (JENT_APT_LSB - 1) + unsigned int apt_observations; /* Number of collected observations */ + unsigned int apt_count; /* APT counter */ + unsigned int apt_base; /* APT base reference */ + unsigned int apt_base_set:1; /* APT base reference set? */ + + unsigned int health_failure:1; /* Permanent health failure */ }; /* Flags that can be used to initialize the RNG */ -#define JENT_DISABLE_STIR (1<<0) /* Disable stirring the entropy pool */ -#define JENT_DISABLE_UNBIAS (1<<1) /* Disable the Von-Neuman Unbiaser */ #define JENT_DISABLE_MEMORY_ACCESS (1<<2) /* Disable memory access for more * entropy, saves MEMORY_SIZE RAM for * entropy collector */ @@ -99,24 +110,205 @@ struct rand_data { #define JENT_ENOTIME 1 /* Timer service not available */ #define JENT_ECOARSETIME 2 /* Timer too coarse for RNG */ #define JENT_ENOMONOTONIC 3 /* Timer is not monotonic increasing */ -#define JENT_EMINVARIATION 4 /* Timer variations too small for RNG */ #define JENT_EVARVAR 5 /* Timer does not produce variations of * variations (2nd derivation of time is * zero). */ -#define JENT_EMINVARVAR 6 /* Timer variations of variations is tooi - * small. */ +#define JENT_ESTUCK 8 /* Too many stuck results during init. */ +#define JENT_EHEALTH 9 /* Health test failed during initialization */ +#define JENT_ERCT 10 /* RCT failed during initialization */ + +#include + +/*************************************************************************** + * Adaptive Proportion Test + * + * This test complies with SP800-90B section 4.4.2. + ***************************************************************************/ + +/** + * Reset the APT counter + * + * @ec [in] Reference to entropy collector + */ +static void jent_apt_reset(struct rand_data *ec, unsigned int delta_masked) +{ + /* Reset APT counter */ + ec->apt_count = 0; + ec->apt_base = delta_masked; + ec->apt_observations = 0; +} + +/** + * Insert a new entropy event into APT + * + * @ec [in] Reference to entropy collector + * @delta_masked [in] Masked time delta to process + */ +static void jent_apt_insert(struct rand_data *ec, unsigned int delta_masked) +{ + /* Initialize the base reference */ + if (!ec->apt_base_set) { + ec->apt_base = delta_masked; + ec->apt_base_set = 1; + return; + } + + if (delta_masked == ec->apt_base) { + ec->apt_count++; + + if (ec->apt_count >= JENT_APT_CUTOFF) + ec->health_failure = 1; + } + + ec->apt_observations++; + + if (ec->apt_observations >= JENT_APT_WINDOW_SIZE) + jent_apt_reset(ec, delta_masked); +} /*************************************************************************** - * Helper functions + * Stuck Test and its use as Repetition Count Test + * + * The Jitter RNG uses an enhanced version of the Repetition Count Test + * (RCT) specified in SP800-90B section 4.4.1. Instead of counting identical + * back-to-back values, the input to the RCT is the counting of the stuck + * values during the generation of one Jitter RNG output block. + * + * The RCT is applied with an alpha of 2^{-30} compliant to FIPS 140-2 IG 9.8. + * + * During the counting operation, the Jitter RNG always calculates the RCT + * cut-off value of C. If that value exceeds the allowed cut-off value, + * the Jitter RNG output block will be calculated completely but discarded at + * the end. The caller of the Jitter RNG is informed with an error code. ***************************************************************************/ -void jent_get_nstime(__u64 *out); -__u64 jent_rol64(__u64 word, unsigned int shift); -void *jent_zalloc(unsigned int len); -void jent_zfree(void *ptr); -int jent_fips_enabled(void); -void jent_panic(char *s); -void jent_memcpy(void *dest, const void *src, unsigned int n); +/** + * Repetition Count Test as defined in SP800-90B section 4.4.1 + * + * @ec [in] Reference to entropy collector + * @stuck [in] Indicator whether the value is stuck + */ +static void jent_rct_insert(struct rand_data *ec, int stuck) +{ + /* + * If we have a count less than zero, a previous RCT round identified + * a failure. We will not overwrite it. + */ + if (ec->rct_count < 0) + return; + + if (stuck) { + ec->rct_count++; + + /* + * The cutoff value is based on the following consideration: + * alpha = 2^-30 as recommended in FIPS 140-2 IG 9.8. + * In addition, we require an entropy value H of 1/OSR as this + * is the minimum entropy required to provide full entropy. + * Note, we collect 64 * OSR deltas for inserting them into + * the entropy pool which should then have (close to) 64 bits + * of entropy. + * + * Note, ec->rct_count (which equals to value B in the pseudo + * code of SP800-90B section 4.4.1) starts with zero. Hence + * we need to subtract one from the cutoff value as calculated + * following SP800-90B. + */ + if ((unsigned int)ec->rct_count >= (31 * ec->osr)) { + ec->rct_count = -1; + ec->health_failure = 1; + } + } else { + ec->rct_count = 0; + } +} + +/** + * Is there an RCT health test failure? + * + * @ec [in] Reference to entropy collector + * + * @return + * 0 No health test failure + * 1 Permanent health test failure + */ +static int jent_rct_failure(struct rand_data *ec) +{ + if (ec->rct_count < 0) + return 1; + return 0; +} + +static inline __u64 jent_delta(__u64 prev, __u64 next) +{ +#define JENT_UINT64_MAX (__u64)(~((__u64) 0)) + return (prev < next) ? (next - prev) : + (JENT_UINT64_MAX - prev + 1 + next); +} + +/** + * Stuck test by checking the: + * 1st derivative of the jitter measurement (time delta) + * 2nd derivative of the jitter measurement (delta of time deltas) + * 3rd derivative of the jitter measurement (delta of delta of time deltas) + * + * All values must always be non-zero. + * + * @ec [in] Reference to entropy collector + * @current_delta [in] Jitter time delta + * + * @return + * 0 jitter measurement not stuck (good bit) + * 1 jitter measurement stuck (reject bit) + */ +static int jent_stuck(struct rand_data *ec, __u64 current_delta) +{ + __u64 delta2 = jent_delta(ec->last_delta, current_delta); + __u64 delta3 = jent_delta(ec->last_delta2, delta2); + unsigned int delta_masked = current_delta & JENT_APT_WORD_MASK; + + ec->last_delta = current_delta; + ec->last_delta2 = delta2; + + /* + * Insert the result of the comparison of two back-to-back time + * deltas. + */ + jent_apt_insert(ec, delta_masked); + + if (!current_delta || !delta2 || !delta3) { + /* RCT with a stuck bit */ + jent_rct_insert(ec, 1); + return 1; + } + + /* RCT with a non-stuck bit */ + jent_rct_insert(ec, 0); + + return 0; +} + +/** + * Report any health test failures + * + * @ec [in] Reference to entropy collector + * + * @return + * 0 No health test failure + * 1 Permanent health test failure + */ +static int jent_health_failure(struct rand_data *ec) +{ + /* Test is only enabled in FIPS mode */ + if (!jent_fips_enabled()) + return 0; + + return ec->health_failure; +} + +/*************************************************************************** + * Noise sources + ***************************************************************************/ /** * Update of the loop count used for the next round of @@ -140,16 +332,16 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, jent_get_nstime(&time); /* - * mix the current state of the random number into the shuffle - * calculation to balance that shuffle a bit more + * Mix the current state of the random number into the shuffle + * calculation to balance that shuffle a bit more. */ if (ec) time ^= ec->data; /* - * we fold the time value as much as possible to ensure that as many - * bits of the time stamp are included as possible + * We fold the time value as much as possible to ensure that as many + * bits of the time stamp are included as possible. */ - for (i = 0; (DATA_SIZE_BITS / bits) > i; i++) { + for (i = 0; ((DATA_SIZE_BITS + bits - 1) / bits) > i; i++) { shuffle ^= time & mask; time = time >> bits; } @@ -161,46 +353,33 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, return (shuffle + (1<data * * @return Number of loops the folding operation is performed */ -static __u64 jent_fold_time(struct rand_data *ec, __u64 time, - __u64 *folded, __u64 loop_cnt) +static void jent_lfsr_time(struct rand_data *ec, __u64 time, __u64 loop_cnt, + int stuck) { unsigned int i; __u64 j = 0; @@ -217,16 +396,43 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, if (loop_cnt) fold_loop_cnt = loop_cnt; for (j = 0; j < fold_loop_cnt; j++) { - new = 0; + new = ec->data; for (i = 1; (DATA_SIZE_BITS) >= i; i++) { __u64 tmp = time << (DATA_SIZE_BITS - i); tmp = tmp >> (DATA_SIZE_BITS - 1); + + /* + * Fibonacci LSFR with polynomial of + * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is + * primitive according to + * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf + * (the shift values are the polynomial values minus one + * due to counting bits from 0 to 63). As the current + * position is always the LSB, the polynomial only needs + * to shift data in from the left without wrap. + */ + tmp ^= ((new >> 63) & 1); + tmp ^= ((new >> 60) & 1); + tmp ^= ((new >> 55) & 1); + tmp ^= ((new >> 30) & 1); + tmp ^= ((new >> 27) & 1); + tmp ^= ((new >> 22) & 1); + new <<= 1; new ^= tmp; } } - *folded = new; - return fold_loop_cnt; + + /* + * If the time stamp is stuck, do not finally insert the value into + * the entropy pool. Although this operation should not do any harm + * even when the time stamp has no entropy, SP800-90B requires that + * any conditioning operation (SP800-90B considers the LFSR to be a + * conditioning operation) to have an identical amount of input + * data according to section 3.1.5. + */ + if (!stuck) + ec->data = new; } /** @@ -247,18 +453,14 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, * to reliably access either L3 or memory, the ec->mem memory must be quite * large which is usually not desirable. * - * Input: - * @ec Reference to the entropy collector with the memory access data -- if - * the reference to the memory block to be accessed is NULL, this noise - * source is disabled - * @loop_cnt if a value not equal to 0 is set, use the given value as number of - * loops to perform the folding - * - * @return Number of memory access operations + * @ec [in] Reference to the entropy collector with the memory access data -- if + * the reference to the memory block to be accessed is NULL, this noise + * source is disabled + * @loop_cnt [in] if a value not equal to 0 is set, use the given value + * number of loops to perform the LFSR */ -static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) +static void jent_memaccess(struct rand_data *ec, __u64 loop_cnt) { - unsigned char *tmpval = NULL; unsigned int wrap = 0; __u64 i = 0; #define MAX_ACC_LOOP_BIT 7 @@ -267,7 +469,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) jent_loop_shuffle(ec, MAX_ACC_LOOP_BIT, MIN_ACC_LOOP_BIT); if (NULL == ec || NULL == ec->mem) - return 0; + return; wrap = ec->memblocksize * ec->memblocks; /* @@ -278,7 +480,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) acc_loop_cnt = loop_cnt; for (i = 0; i < (ec->memaccessloops + acc_loop_cnt); i++) { - tmpval = ec->mem + ec->memlocation; + unsigned char *tmpval = ec->mem + ec->memlocation; /* * memory access: just add 1 to one byte, * wrap at 255 -- memory access implies read @@ -293,61 +495,29 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) ec->memlocation = ec->memlocation + ec->memblocksize - 1; ec->memlocation = ec->memlocation % wrap; } - return i; } /*************************************************************************** * Start of entropy processing logic ***************************************************************************/ - -/** - * Stuck test by checking the: - * 1st derivation of the jitter measurement (time delta) - * 2nd derivation of the jitter measurement (delta of time deltas) - * 3rd derivation of the jitter measurement (delta of delta of time deltas) - * - * All values must always be non-zero. - * - * Input: - * @ec Reference to entropy collector - * @current_delta Jitter time delta - * - * @return - * 0 jitter measurement not stuck (good bit) - * 1 jitter measurement stuck (reject bit) - */ -static void jent_stuck(struct rand_data *ec, __u64 current_delta) -{ - __s64 delta2 = ec->last_delta - current_delta; - __s64 delta3 = delta2 - ec->last_delta2; - - ec->last_delta = current_delta; - ec->last_delta2 = delta2; - - if (!current_delta || !delta2 || !delta3) - ec->stuck = 1; -} - /** * This is the heart of the entropy generation: calculate time deltas and - * use the CPU jitter in the time deltas. The jitter is folded into one - * bit. You can call this function the "random bit generator" as it - * produces one random bit per invocation. + * use the CPU jitter in the time deltas. The jitter is injected into the + * entropy pool. * * WARNING: ensure that ->prev_time is primed before using the output * of this function! This can be done by calling this function * and not using its result. * - * Input: - * @entropy_collector Reference to entropy collector + * @ec [in] Reference to entropy collector * - * @return One random bit + * @return result of stuck test */ -static __u64 jent_measure_jitter(struct rand_data *ec) +static int jent_measure_jitter(struct rand_data *ec) { __u64 time = 0; - __u64 data = 0; __u64 current_delta = 0; + int stuck; /* Invoke one noise source before time measurement to add variations */ jent_memaccess(ec, 0); @@ -357,120 +527,23 @@ static __u64 jent_measure_jitter(struct rand_data *ec) * invocation to measure the timing variations */ jent_get_nstime(&time); - current_delta = time - ec->prev_time; + current_delta = jent_delta(ec->prev_time, time); ec->prev_time = time; - /* Now call the next noise sources which also folds the data */ - jent_fold_time(ec, current_delta, &data, 0); - - /* - * Check whether we have a stuck measurement. The enforcement - * is performed after the stuck value has been mixed into the - * entropy pool. - */ - jent_stuck(ec, current_delta); - - return data; -} - -/** - * Von Neuman unbias as explained in RFC 4086 section 4.2. As shown in the - * documentation of that RNG, the bits from jent_measure_jitter are considered - * independent which implies that the Von Neuman unbias operation is applicable. - * A proof of the Von-Neumann unbias operation to remove skews is given in the - * document "A proposal for: Functionality classes for random number - * generators", version 2.0 by Werner Schindler, section 5.4.1. - * - * Input: - * @entropy_collector Reference to entropy collector - * - * @return One random bit - */ -static __u64 jent_unbiased_bit(struct rand_data *entropy_collector) -{ - do { - __u64 a = jent_measure_jitter(entropy_collector); - __u64 b = jent_measure_jitter(entropy_collector); - - if (a == b) - continue; - if (1 == a) - return 1; - else - return 0; - } while (1); -} - -/** - * Shuffle the pool a bit by mixing some value with a bijective function (XOR) - * into the pool. - * - * The function generates a mixer value that depends on the bits set and the - * location of the set bits in the random number generated by the entropy - * source. Therefore, based on the generated random number, this mixer value - * can have 2**64 different values. That mixer value is initialized with the - * first two SHA-1 constants. After obtaining the mixer value, it is XORed into - * the random number. - * - * The mixer value is not assumed to contain any entropy. But due to the XOR - * operation, it can also not destroy any entropy present in the entropy pool. - * - * Input: - * @entropy_collector Reference to entropy collector - */ -static void jent_stir_pool(struct rand_data *entropy_collector) -{ - /* - * to shut up GCC on 32 bit, we have to initialize the 64 variable - * with two 32 bit variables - */ - union c { - __u64 u64; - __u32 u32[2]; - }; - /* - * This constant is derived from the first two 32 bit initialization - * vectors of SHA-1 as defined in FIPS 180-4 section 5.3.1 - */ - union c constant; - /* - * The start value of the mixer variable is derived from the third - * and fourth 32 bit initialization vector of SHA-1 as defined in - * FIPS 180-4 section 5.3.1 - */ - union c mixer; - unsigned int i = 0; + /* Check whether we have a stuck measurement. */ + stuck = jent_stuck(ec, current_delta); - /* - * Store the SHA-1 constants in reverse order to make up the 64 bit - * value -- this applies to a little endian system, on a big endian - * system, it reverses as expected. But this really does not matter - * as we do not rely on the specific numbers. We just pick the SHA-1 - * constants as they have a good mix of bit set and unset. - */ - constant.u32[1] = 0x67452301; - constant.u32[0] = 0xefcdab89; - mixer.u32[1] = 0x98badcfe; - mixer.u32[0] = 0x10325476; + /* Now call the next noise sources which also injects the data */ + jent_lfsr_time(ec, current_delta, 0, stuck); - for (i = 0; i < DATA_SIZE_BITS; i++) { - /* - * get the i-th bit of the input random number and only XOR - * the constant into the mixer value when that bit is set - */ - if ((entropy_collector->data >> i) & 1) - mixer.u64 ^= constant.u64; - mixer.u64 = jent_rol64(mixer.u64, 1); - } - entropy_collector->data ^= mixer.u64; + return stuck; } /** * Generator of one 64 bit random number * Function fills rand_data->data * - * Input: - * @ec Reference to entropy collector + * @ec [in] Reference to entropy collector */ static void jent_gen_entropy(struct rand_data *ec) { @@ -480,48 +553,9 @@ static void jent_gen_entropy(struct rand_data *ec) jent_measure_jitter(ec); while (1) { - __u64 data = 0; - - if (ec->disable_unbias == 1) - data = jent_measure_jitter(ec); - else - data = jent_unbiased_bit(ec); - - /* enforcement of the jent_stuck test */ - if (ec->stuck) { - /* - * We only mix in the bit considered not appropriate - * without the LSFR. The reason is that if we apply - * the LSFR and we do not rotate, the 2nd bit with LSFR - * will cancel out the first LSFR application on the - * bad bit. - * - * And we do not rotate as we apply the next bit to the - * current bit location again. - */ - ec->data ^= data; - ec->stuck = 0; + /* If a stuck measurement is received, repeat measurement */ + if (jent_measure_jitter(ec)) continue; - } - - /* - * Fibonacci LSFR with polynom of - * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is - * primitive according to - * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf - * (the shift values are the polynom values minus one - * due to counting bits from 0 to 63). As the current - * position is always the LSB, the polynom only needs - * to shift data in from the left without wrap. - */ - ec->data ^= data; - ec->data ^= ((ec->data >> 63) & 1); - ec->data ^= ((ec->data >> 60) & 1); - ec->data ^= ((ec->data >> 55) & 1); - ec->data ^= ((ec->data >> 30) & 1); - ec->data ^= ((ec->data >> 27) & 1); - ec->data ^= ((ec->data >> 22) & 1); - ec->data = jent_rol64(ec->data, 1); /* * We multiply the loop value with ->osr to obtain the @@ -530,33 +564,6 @@ static void jent_gen_entropy(struct rand_data *ec) if (++k >= (DATA_SIZE_BITS * ec->osr)) break; } - if (ec->stir) - jent_stir_pool(ec); -} - -/** - * The continuous test required by FIPS 140-2 -- the function automatically - * primes the test if needed. - * - * Return: - * 0 if FIPS test passed - * < 0 if FIPS test failed - */ -static void jent_fips_test(struct rand_data *ec) -{ - if (!jent_fips_enabled()) - return; - - /* prime the FIPS test */ - if (!ec->old_data) { - ec->old_data = ec->data; - jent_gen_entropy(ec); - } - - if (ec->data == ec->old_data) - jent_panic("jitterentropy: Duplicate output detected\n"); - - ec->old_data = ec->data; } /** @@ -569,17 +576,18 @@ static void jent_fips_test(struct rand_data *ec) * This function truncates the last 64 bit entropy value output to the exact * size specified by the caller. * - * Input: - * @ec Reference to entropy collector - * @data pointer to buffer for storing random data -- buffer must already - * exist - * @len size of the buffer, specifying also the requested number of random - * in bytes + * @ec [in] Reference to entropy collector + * @data [in] pointer to buffer for storing random data -- buffer must already + * exist + * @len [in] size of the buffer, specifying also the requested number of random + * in bytes * * @return 0 when request is fulfilled or an error * * The following error codes can occur: * -1 entropy_collector is NULL + * -2 RCT failed + * -3 APT test failed */ int jent_read_entropy(struct rand_data *ec, unsigned char *data, unsigned int len) @@ -589,11 +597,46 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, if (!ec) return -1; - while (0 < len) { + while (len > 0) { unsigned int tocopy; jent_gen_entropy(ec); - jent_fips_test(ec); + + if (jent_health_failure(ec)) { + int ret; + + if (jent_rct_failure(ec)) + ret = -2; + else + ret = -3; + + /* + * Re-initialize the noise source + * + * If the health test fails, the Jitter RNG remains + * in failure state and will return a health failure + * during next invocation. + */ + if (jent_entropy_init()) + return ret; + + /* Set APT to initial state */ + jent_apt_reset(ec, 0); + ec->apt_base_set = 0; + + /* Set RCT to initial state */ + ec->rct_count = 0; + + /* Re-enable Jitter RNG */ + ec->health_failure = 0; + + /* + * Return the health test failure status to the + * caller as the generated value is not appropriate. + */ + return ret; + } + if ((DATA_SIZE_BITS / 8) < len) tocopy = (DATA_SIZE_BITS / 8); else @@ -635,16 +678,10 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr, } /* verify and set the oversampling rate */ - if (0 == osr) + if (osr == 0) osr = 1; /* minimum sampling rate is 1 */ entropy_collector->osr = osr; - entropy_collector->stir = 1; - if (flags & JENT_DISABLE_STIR) - entropy_collector->stir = 0; - if (flags & JENT_DISABLE_UNBIAS) - entropy_collector->disable_unbias = 1; - /* fill the data pad with non-zero values */ jent_gen_entropy(entropy_collector); @@ -656,7 +693,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector) jent_zfree(entropy_collector->mem); entropy_collector->mem = NULL; jent_zfree(entropy_collector); - entropy_collector = NULL; } int jent_entropy_init(void) @@ -664,9 +700,14 @@ int jent_entropy_init(void) int i; __u64 delta_sum = 0; __u64 old_delta = 0; + unsigned int nonstuck = 0; int time_backwards = 0; - int count_var = 0; int count_mod = 0; + int count_stuck = 0; + struct rand_data ec = { 0 }; + + /* Required for RCT */ + ec.osr = 1; /* We could perform statistical tests here, but the problem is * that we only have a few loop counts to do testing. These @@ -689,24 +730,28 @@ int jent_entropy_init(void) /* * TESTLOOPCOUNT needs some loops to identify edge systems. 100 is * definitely too little. + * + * SP800-90B requires at least 1024 initial test cycles. */ -#define TESTLOOPCOUNT 300 +#define TESTLOOPCOUNT 1024 #define CLEARCACHE 100 for (i = 0; (TESTLOOPCOUNT + CLEARCACHE) > i; i++) { __u64 time = 0; __u64 time2 = 0; - __u64 folded = 0; __u64 delta = 0; unsigned int lowdelta = 0; + int stuck; + /* Invoke core entropy collection logic */ jent_get_nstime(&time); - jent_fold_time(NULL, time, &folded, 1< i) + if (i < CLEARCACHE) continue; + if (stuck) + count_stuck++; + else { + nonstuck++; + + /* + * Ensure that the APT succeeded. + * + * With the check below that count_stuck must be less + * than 10% of the overall generated raw entropy values + * it is guaranteed that the APT is invoked at + * floor((TESTLOOPCOUNT * 0.9) / 64) == 14 times. + */ + if ((nonstuck % JENT_APT_WINDOW_SIZE) == 0) { + jent_apt_reset(&ec, + delta & JENT_APT_WORD_MASK); + if (jent_health_failure(&ec)) + return JENT_EHEALTH; + } + } + + /* Validate RCT */ + if (jent_rct_failure(&ec)) + return JENT_ERCT; + /* test whether we have an increasing timer */ if (!(time2 > time)) time_backwards++; - /* - * Avoid modulo of 64 bit integer to allow code to compile - * on 32 bit architectures. - */ + /* use 32 bit value to ensure compilation on 32 bit arches */ lowdelta = time2 - time; if (!(lowdelta % 100)) count_mod++; @@ -743,14 +812,10 @@ int jent_entropy_init(void) * only after the first loop is executed as we need to prime * the old_data value */ - if (i) { - if (delta != old_delta) - count_var++; - if (delta > old_delta) - delta_sum += (delta - old_delta); - else - delta_sum += (old_delta - delta); - } + if (delta > old_delta) + delta_sum += (delta - old_delta); + else + delta_sum += (old_delta - delta); old_delta = delta; } @@ -761,27 +826,31 @@ int jent_entropy_init(void) * should not fail. The value of 3 should cover the NTP case being * performed during our test run. */ - if (3 < time_backwards) + if (time_backwards > 3) return JENT_ENOMONOTONIC; - /* Error if the time variances are always identical */ - if (!delta_sum) - return JENT_EVARVAR; /* * Variations of deltas of time must on average be larger * than 1 to ensure the entropy estimation * implied with 1 is preserved */ - if (delta_sum <= 1) - return JENT_EMINVARVAR; + if ((delta_sum) <= 1) + return JENT_EVARVAR; /* * Ensure that we have variations in the time stamp below 10 for at - * least 10% of all checks -- on some platforms, the counter - * increments in multiples of 100, but not always + * least 10% of all checks -- on some platforms, the counter increments + * in multiples of 100, but not always */ if ((TESTLOOPCOUNT/10 * 9) < count_mod) return JENT_ECOARSETIME; + /* + * If we have more than 90% stuck results, then this Jitter RNG is + * likely to not work well. + */ + if ((TESTLOOPCOUNT/10 * 9) < count_stuck) + return JENT_ESTUCK; + return 0; } diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c index 39ee0ca636aa..0b6489fd5d0d 100644 --- a/drivers/acpi/sysfs.c +++ b/drivers/acpi/sysfs.c @@ -439,18 +439,29 @@ static ssize_t acpi_data_show(struct file *filp, struct kobject *kobj, { struct acpi_data_attr *data_attr; void __iomem *base; - ssize_t rc; + ssize_t size; data_attr = container_of(bin_attr, struct acpi_data_attr, attr); + size = data_attr->attr.size; + + if (offset < 0) + return -EINVAL; + + if (offset >= size) + return 0; - base = acpi_os_map_memory(data_attr->addr, data_attr->attr.size); + if (count > size - offset) + count = size - offset; + + base = acpi_os_map_iomem(data_attr->addr, size); if (!base) return -ENOMEM; - rc = memory_read_from_buffer(buf, count, &offset, base, - data_attr->attr.size); - acpi_os_unmap_memory(base, data_attr->attr.size); - return rc; + memcpy_fromio(buf, base + offset, count); + + acpi_os_unmap_iomem(base, size); + + return count; } static int acpi_bert_data_init(void *th, struct acpi_data_attr *data_attr) diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index a0b0b4d986f2..43a91495ee67 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -196,7 +196,7 @@ static struct { { XFER_PIO_0, "XFER_PIO_0" }, { XFER_PIO_SLOW, "XFER_PIO_SLOW" } }; -ata_bitfield_name_match(xfer,ata_xfer_names) +ata_bitfield_name_search(xfer, ata_xfer_names) /* * ATA Port attributes diff --git a/drivers/ata/pata_octeon_cf.c b/drivers/ata/pata_octeon_cf.c index ac3b1fda820f..c240d8cbfd41 100644 --- a/drivers/ata/pata_octeon_cf.c +++ b/drivers/ata/pata_octeon_cf.c @@ -888,12 +888,14 @@ static int octeon_cf_probe(struct platform_device *pdev) int i; res_dma = platform_get_resource(dma_dev, IORESOURCE_MEM, 0); if (!res_dma) { + put_device(&dma_dev->dev); of_node_put(dma_node); return -EINVAL; } cf_port->dma_base = (u64)devm_ioremap_nocache(&pdev->dev, res_dma->start, resource_size(res_dma)); if (!cf_port->dma_base) { + put_device(&dma_dev->dev); of_node_put(dma_node); return -EINVAL; } @@ -903,6 +905,7 @@ static int octeon_cf_probe(struct platform_device *pdev) irq = i; irq_handler = octeon_cf_interrupt; } + put_device(&dma_dev->dev); } of_node_put(dma_node); } diff --git a/drivers/base/node.c b/drivers/base/node.c index d2f133feaa9b..d5dfcfbcd018 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -350,6 +350,7 @@ static int register_node(struct node *node, int num) */ void unregister_node(struct node *node) { + compaction_unregister_node(node); hugetlb_unregister_node(node); /* no-op, if memoryless node */ device_unregister(&node->dev); diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 81b955670b12..2ef7eec6461c 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1288,7 +1288,7 @@ static int nbd_start_device_ioctl(struct nbd_device *nbd, struct block_device *b static void nbd_clear_sock_ioctl(struct nbd_device *nbd, struct block_device *bdev) { - sock_shutdown(nbd); + nbd_clear_sock(nbd); __invalidate_device(bdev, true); nbd_bdev_reset(bdev); if (test_and_clear_bit(NBD_HAS_CONFIG_REF, @@ -1395,15 +1395,20 @@ static struct nbd_config *nbd_alloc_config(void) { struct nbd_config *config; + if (!try_module_get(THIS_MODULE)) + return ERR_PTR(-ENODEV); + config = kzalloc(sizeof(struct nbd_config), GFP_NOFS); - if (!config) - return NULL; + if (!config) { + module_put(THIS_MODULE); + return ERR_PTR(-ENOMEM); + } + atomic_set(&config->recv_threads, 0); init_waitqueue_head(&config->recv_wq); init_waitqueue_head(&config->conn_wait); config->blksize = NBD_DEF_BLKSIZE; atomic_set(&config->live_connections, 0); - try_module_get(THIS_MODULE); return config; } @@ -1430,12 +1435,13 @@ static int nbd_open(struct block_device *bdev, fmode_t mode) mutex_unlock(&nbd->config_lock); goto out; } - config = nbd->config = nbd_alloc_config(); - if (!config) { - ret = -ENOMEM; + config = nbd_alloc_config(); + if (IS_ERR(config)) { + ret = PTR_ERR(config); mutex_unlock(&nbd->config_lock); goto out; } + nbd->config = config; refcount_set(&nbd->config_refs, 1); refcount_inc(&nbd->refs); mutex_unlock(&nbd->config_lock); @@ -1820,13 +1826,14 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) nbd_put(nbd); return -EINVAL; } - config = nbd->config = nbd_alloc_config(); - if (!nbd->config) { + config = nbd_alloc_config(); + if (IS_ERR(config)) { mutex_unlock(&nbd->config_lock); nbd_put(nbd); printk(KERN_ERR "nbd: couldn't allocate config\n"); - return -ENOMEM; + return PTR_ERR(config); } + nbd->config = config; refcount_set(&nbd->config_refs, 1); set_bit(NBD_BOUND, &config->runtime_flags); @@ -2337,6 +2344,12 @@ static void __exit nbd_cleanup(void) struct nbd_device *nbd; LIST_HEAD(del_list); + /* + * Unregister netlink interface prior to waiting + * for the completion of netlink commands. + */ + genl_unregister_family(&nbd_genl_family); + nbd_dbg_close(); mutex_lock(&nbd_index_mutex); @@ -2346,13 +2359,15 @@ static void __exit nbd_cleanup(void) while (!list_empty(&del_list)) { nbd = list_first_entry(&del_list, struct nbd_device, list); list_del_init(&nbd->list); + if (refcount_read(&nbd->config_refs)) + printk(KERN_ERR "nbd: possibly leaking nbd_config (ref %d)\n", + refcount_read(&nbd->config_refs)); if (refcount_read(&nbd->refs) != 1) printk(KERN_ERR "nbd: possibly leaking a device\n"); nbd_put(nbd); } idr_destroy(&nbd_index_idr); - genl_unregister_family(&nbd_genl_family); unregister_blkdev(NBD_MAJOR, "nbd"); } diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index b6a278183d82..bc274ddb9767 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -1801,7 +1801,9 @@ static int sysc_remove(struct platform_device *pdev) struct sysc *ddata = platform_get_drvdata(pdev); int error; - cancel_delayed_work_sync(&ddata->idle_work); + /* Device can still be enabled, see deferred idle quirk in probe */ + if (cancel_delayed_work_sync(&ddata->idle_work)) + ti_sysc_idle(&ddata->idle_work.work); error = pm_runtime_get_sync(ddata->dev); if (error < 0) { diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index a6c48a4882ea..7b4ff92d0f79 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -628,6 +628,9 @@ config XLOGCHAR tristate "xlogchar driver" help Implements a high performance log driver + +source "drivers/char/lrng/Kconfig" + endmenu config RANDOM_TRUST_CPU diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 5d633d50b363..f2dd86dbef6e 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -3,7 +3,9 @@ # Makefile for the kernel character device drivers. # -obj-y += mem.o random.o +obj-y += mem.o +obj-$(CONFIG_RANDOM_DEFAULT_IMPL) += random.o +obj-$(CONFIG_LRNG) += lrng/ obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o obj-y += misc.o obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o diff --git a/drivers/char/adsprpc_compat.c b/drivers/char/adsprpc_compat.c index 1086088ed240..f006a5237c6c 100644 --- a/drivers/char/adsprpc_compat.c +++ b/drivers/char/adsprpc_compat.c @@ -156,7 +156,7 @@ struct compat_fastrpc_ioctl_dsp_capabilities { static int compat_get_fastrpc_ioctl_invoke( struct compat_fastrpc_ioctl_invoke_crc __user *inv32, struct fastrpc_ioctl_invoke_crc __user **inva, - unsigned int cmd, unsigned int sc) + unsigned int cmd, compat_uint_t sc) { compat_uint_t u; compat_size_t s; @@ -520,14 +520,14 @@ long compat_fastrpc_device_ioctl(struct file *filp, unsigned int cmd, case COMPAT_FASTRPC_IOCTL_INVOKE_ATTRS: case COMPAT_FASTRPC_IOCTL_INVOKE_CRC: { - struct compat_fastrpc_ioctl_invoke_crc __user *inv32; - struct fastrpc_ioctl_invoke_crc __user *inv; + struct compat_fastrpc_ioctl_invoke_crc __user *inv32 = NULL; + struct fastrpc_ioctl_invoke_crc __user *inv = NULL; + inv32 = compat_ptr(arg); err = get_user(sc, &inv32->inv.sc); if (err) return err; - inv32 = compat_ptr(arg); VERIFY(err, 0 == compat_get_fastrpc_ioctl_invoke(inv32, &inv, cmd, sc)); if (err) diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index fec679433f72..fd1a487443f0 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -801,6 +801,14 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result, break; case SSIF_GETTING_EVENTS: + if (!msg) { + /* Should never happen, but just in case. */ + dev_warn(&ssif_info->client->dev, + "No message set while getting events\n"); + ipmi_ssif_unlock_cond(ssif_info, flags); + break; + } + if ((result < 0) || (len < 3) || (msg->rsp[2] != 0)) { /* Error getting event, probably done. */ msg->done(msg); @@ -824,6 +832,14 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result, break; case SSIF_GETTING_MESSAGES: + if (!msg) { + /* Should never happen, but just in case. */ + dev_warn(&ssif_info->client->dev, + "No message set while getting messages\n"); + ipmi_ssif_unlock_cond(ssif_info, flags); + break; + } + if ((result < 0) || (len < 3) || (msg->rsp[2] != 0)) { /* Error getting event, probably done. */ msg->done(msg); @@ -846,6 +862,13 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result, deliver_recv_msg(ssif_info, msg); } break; + + default: + /* Should never happen, but just in case. */ + dev_warn(&ssif_info->client->dev, + "Invalid state in message done handling: %d\n", + ssif_info->ssif_state); + ipmi_ssif_unlock_cond(ssif_info, flags); } flags = ipmi_ssif_lock_cond(ssif_info, &oflags); diff --git a/drivers/char/lrng/Kconfig b/drivers/char/lrng/Kconfig new file mode 100644 index 000000000000..bfd6b3ea4f1b --- /dev/null +++ b/drivers/char/lrng/Kconfig @@ -0,0 +1,907 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Linux Random Number Generator configuration +# + +config RANDOM_DEFAULT_IMPL + bool "Kernel RNG Default Implementation" + default y + help + The default random number generator as provided with + drivers/char/random.c is selected with this option. + +config LRNG + bool "Linux Random Number Generator" + default n + select CRYPTO_LIB_SHA256 if CRYPTO + help + The Linux Random Number Generator (LRNG) generates entropy + from different entropy sources. Each entropy source can + be enabled and configured independently. The interrupt + entropy source can be configured to be SP800-90B compliant. + The entire LRNG can be configured to be SP800-90C compliant. + Runtime-switchable cryptographic support is available. + The LRNG delivers significant entropy during boot. + + The LRNG also provides compliance to SP800-90A/B/C. + +menu "Linux Random Number Generator Configuration" + depends on LRNG + +if LRNG + +config LRNG_SHA256 + bool + default y if CRYPTO_LIB_SHA256 + +config LRNG_SHA1 + bool + default y if !CRYPTO_LIB_SHA256 + +config LRNG_COMMON_DEV_IF + bool + +config LRNG_DRNG_ATOMIC + bool + select LRNG_DRNG_CHACHA20 + +config LRNG_SYSCTL + bool + depends on SYSCTL + +config LRNG_RANDOM_IF + bool + default n if RANDOM_DEFAULT_IMPL + default y if !RANDOM_DEFAULT_IMPL + select LRNG_COMMON_DEV_IF + select LRNG_DRNG_ATOMIC + select LRNG_SYSCTL + +menu "LRNG Interfaces" + +config LRNG_KCAPI_IF + tristate "Interface with Kernel Crypto API" + depends on CRYPTO_RNG + help + The LRNG can be registered with the kernel crypto API's + random number generator framework. This offers a random + number generator with the name "lrng" and a priority that + is intended to be higher than the existing RNG + implementations. + +config LRNG_HWRAND_IF + tristate "Interface with Hardware Random Number Generator Framework" + depends on HW_RANDOM + select LRNG_DRNG_ATOMIC + help + The LRNG can be registered with the hardware random number + generator framework. This offers a random number generator + with the name "lrng" that is accessible via the framework. + For example it allows pulling data from the LRNG via the + /dev/hwrng file. + +config LRNG_DEV_IF + bool "Character device file interface" + select LRNG_COMMON_DEV_IF + help + The LRNG can create a character device file that operates + identically to /dev/random including IOCTL, read and write + operations. + +endmenu # "LRNG Interfaces" + +menu "Entropy Source Configuration" + +config LRNG_RUNTIME_ES_CONFIG + bool "Enable runtime configuration of entropy sources" + help + When enabling this option, the LRNG provides the mechanism + allowing to alter the entropy rate of each entropy source + during boot time and runtime. + + Each entropy source allows its entropy rate changed with + a kernel command line option. When not providing any + option, the default specified during kernel compilation + is applied. + +comment "Common Timer-based Entropy Source Configuration" + +config LRNG_IRQ_DFLT_TIMER_ES + bool + +config LRNG_SCHED_DFLT_TIMER_ES + bool + +config LRNG_TIMER_COMMON + bool + +choice + prompt "Default Timer-based Entropy Source" + default LRNG_IRQ_DFLT_TIMER_ES + depends on LRNG_TIMER_COMMON + help + Select the timer-based entropy source that is credited + with entropy. The other timer-based entropy sources may + be operational and provide data, but are credited with no + entropy. + + config LRNG_IRQ_DFLT_TIMER_ES + bool "Interrupt Entropy Source" + depends on LRNG_IRQ + help + The interrupt entropy source is selected as a timer-based + entropy source to provide entropy. + + config LRNG_SCHED_DFLT_TIMER_ES + bool "Scheduler Entropy Source" + depends on LRNG_SCHED + help + The scheduler entropy source is selected as timer-based + entropy source to provide entropy. +endchoice + +choice + prompt "LRNG Entropy Collection Pool Size" + default LRNG_COLLECTION_SIZE_1024 + depends on LRNG_TIMER_COMMON + help + Select the size of the LRNG entropy collection pool + storing data for the interrupt as well as the scheduler + entropy sources without performing a compression + operation. The larger the collection size is, the faster + the average interrupt handling will be. The collection + size represents the number of bytes of the per-CPU memory + used to batch up entropy event data. + + The default value is good for regular operations. Choose + larger sizes for servers that have no memory limitations. + If runtime memory is precious, choose a smaller size. + + The collection size is unrelated to the entropy rate + or the amount of entropy the LRNG can process. + + config LRNG_COLLECTION_SIZE_32 + depends on LRNG_CONTINUOUS_COMPRESSION_ENABLED + depends on !LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION + depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES + bool "32 interrupt events" + + config LRNG_COLLECTION_SIZE_256 + depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES + bool "256 interrupt events" + + config LRNG_COLLECTION_SIZE_512 + bool "512 interrupt events" + + config LRNG_COLLECTION_SIZE_1024 + bool "1024 interrupt events (default)" + + config LRNG_COLLECTION_SIZE_2048 + bool "2048 interrupt events" + + config LRNG_COLLECTION_SIZE_4096 + bool "4096 interrupt events" + + config LRNG_COLLECTION_SIZE_8192 + bool "8192 interrupt events" + +endchoice + +config LRNG_COLLECTION_SIZE + int + default 32 if LRNG_COLLECTION_SIZE_32 + default 256 if LRNG_COLLECTION_SIZE_256 + default 512 if LRNG_COLLECTION_SIZE_512 + default 1024 if LRNG_COLLECTION_SIZE_1024 + default 2048 if LRNG_COLLECTION_SIZE_2048 + default 4096 if LRNG_COLLECTION_SIZE_4096 + default 8192 if LRNG_COLLECTION_SIZE_8192 + +config LRNG_HEALTH_TESTS + bool "Enable internal entropy source online health tests" + depends on LRNG_TIMER_COMMON + help + The online health tests applied to the interrupt entropy + source and to the scheduler entropy source to validate + the noise source at runtime for fatal errors. These tests + include SP800-90B compliant tests which are invoked if + the system is booted with fips=1. In case of fatal errors + during active SP800-90B tests, the issue is logged and + the noise data is discarded. These tests are required for + full compliance of the interrupt entropy source with + SP800-90B. + + If both, the scheduler and the interrupt entropy sources, + are enabled, the health tests for both are applied + independent of each other. + + If unsure, say Y. + +config LRNG_RCT_BROKEN + bool "SP800-90B RCT with dangerous low cutoff value" + depends on LRNG_HEALTH_TESTS + depends on BROKEN + default n + help + This option enables a dangerously low SP800-90B repetitive + count test (RCT) cutoff value which makes it very likely + that the RCT is triggered to raise a self test failure. + + This option is ONLY intended for developers wanting to + test the effectiveness of the SP800-90B RCT health test. + + If unsure, say N. + +config LRNG_APT_BROKEN + bool "SP800-90B APT with dangerous low cutoff value" + depends on LRNG_HEALTH_TESTS + depends on BROKEN + default n + help + This option enables a dangerously low SP800-90B adaptive + proportion test (APT) cutoff value which makes it very + likely that the APT is triggered to raise a self test + failure. + + This option is ONLY intended for developers wanting to + test the effectiveness of the SP800-90B APT health test. + + If unsure, say N. + +# Default taken from SP800-90B sec 4.4.1 - significance level 2^-30 +config LRNG_RCT_CUTOFF + int + default 31 if !LRNG_RCT_BROKEN + default 1 if LRNG_RCT_BROKEN + +# Default taken from SP800-90B sec 4.4.2 - significance level 2^-30 +config LRNG_APT_CUTOFF + int + default 325 if !LRNG_APT_BROKEN + default 32 if LRNG_APT_BROKEN + +comment "Interrupt Entropy Source" + +config LRNG_IRQ + bool "Enable Interrupt Entropy Source as LRNG Seed Source" + default y + depends on !RANDOM_DEFAULT_IMPL + select LRNG_TIMER_COMMON + help + The LRNG models an entropy source based on the timing of the + occurrence of interrupts. Enable this option to enable this + IRQ entropy source. + + The IRQ entropy source is triggered every time an interrupt + arrives and thus causes the interrupt handler to execute + slightly longer. Disabling the IRQ entropy source implies + that the performance penalty on the interrupt handler added + by the LRNG is eliminated. Yet, this entropy source is + considered to be an internal entropy source of the LRNG. + Thus, only disable it if you ensured that other entropy + sources are available that supply the LRNG with entropy. + + If you disable the IRQ entropy source, you MUST ensure + one or more entropy sources collectively have the + capability to deliver sufficient entropy with one invocation + at a rate compliant to the security strength of the DRNG + (usually 256 bits of entropy). In addition, if those + entropy sources do not deliver sufficient entropy during + first request, the reseed must be triggered from user + space or kernel space when sufficient entropy is considered + to be present. + + If unsure, say Y. + +choice + prompt "Continuous entropy compression boot time setting" + default LRNG_CONTINUOUS_COMPRESSION_ENABLED + depends on LRNG_IRQ + help + Select the default behavior of the interrupt entropy source + continuous compression operation. + + The LRNG IRQ ES collects entropy data during each interrupt. + For performance reasons, a amount of entropy data defined by + the LRNG entropy collection pool size is concatenated into + an array. When that array is filled up, a hash is calculated + to compress the entropy. That hash is calculated in + interrupt context. + + In case such hash calculation in interrupt context is deemed + too time-consuming, the continuous compression operation + can be disabled. If disabled, the collection of entropy will + not trigger a hash compression operation in interrupt context. + The compression happens only when the DRNG is reseeded which is + in process context. This implies that old entropy data + collected after the last DRNG-reseed is overwritten with newer + entropy data once the collection pool is full instead of + retaining its entropy with the compression operation. + + config LRNG_CONTINUOUS_COMPRESSION_ENABLED + bool "Enable continuous compression (default)" + + config LRNG_CONTINUOUS_COMPRESSION_DISABLED + bool "Disable continuous compression" + +endchoice + +config LRNG_ENABLE_CONTINUOUS_COMPRESSION + bool + default y if LRNG_CONTINUOUS_COMPRESSION_ENABLED + default n if LRNG_CONTINUOUS_COMPRESSION_DISABLED + +config LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION + bool "Runtime-switchable continuous entropy compression" + depends on LRNG_IRQ + help + Per default, the interrupt entropy source continuous + compression operation behavior is hard-wired into the kernel. + Enable this option to allow it to be configurable at boot time. + + To modify the default behavior of the continuous + compression operation, use the kernel command line option + of lrng_sw_noise.lrng_pcpu_continuous_compression. + + If unsure, say N. + +config LRNG_IRQ_ENTROPY_RATE + int "Interrupt Entropy Source Entropy Rate" + depends on LRNG_IRQ + range 256 4294967295 if LRNG_IRQ_DFLT_TIMER_ES + range 4294967295 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES + default 256 if LRNG_IRQ_DFLT_TIMER_ES + default 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES + help + The LRNG will collect the configured number of interrupts to + obtain 256 bits of entropy. This value can be set to any between + 256 and 4294967295. The LRNG guarantees that this value is not + lower than 256. This lower limit implies that one interrupt event + is credited with one bit of entropy. This value is subject to the + increase by the oversampling factor, if no high-resolution timer + is found. + + In order to effectively disable the interrupt entropy source, + the option has to be set to 4294967295. In this case, the + interrupt entropy source will still deliver data but without + being credited with entropy. + +comment "Jitter RNG Entropy Source" + +config LRNG_JENT + bool "Enable Jitter RNG as LRNG Seed Source" + depends on CRYPTO + select CRYPTO_JITTERENTROPY + help + The LRNG may use the Jitter RNG as entropy source. Enabling + this option enables the use of the Jitter RNG. Its default + entropy level is 16 bits of entropy per 256 data bits delivered + by the Jitter RNG. This entropy level can be changed at boot + time or at runtime with the lrng_base.jitterrng configuration + variable. + +config LRNG_JENT_ENTROPY_RATE + int "Jitter RNG Entropy Source Entropy Rate" + depends on LRNG_JENT + range 0 256 + default 16 + help + The option defines the amount of entropy the LRNG applies to 256 + bits of data obtained from the Jitter RNG entropy source. The + LRNG enforces the limit that this value must be in the range + between 0 and 256. + + When configuring this value to 0, the Jitter RNG entropy source + will provide 256 bits of data without being credited to contain + entropy. + +comment "CPU Entropy Source" + +config LRNG_CPU + bool "Enable CPU Entropy Source as LRNG Seed Source" + default y + help + Current CPUs commonly contain entropy sources which can be + used to seed the LRNG. For example, the Intel RDSEED + instruction, or the POWER DARN instruction will be sourced + to seed the LRNG if this option is enabled. + + Note, if this option is enabled and the underlying CPU + does not offer such entropy source, the LRNG will automatically + detect this and ignore the hardware. + +config LRNG_CPU_FULL_ENT_MULTIPLIER + int + default 1 if !LRNG_TEST_CPU_ES_COMPRESSION + default 123 if LRNG_TEST_CPU_ES_COMPRESSION + +config LRNG_CPU_ENTROPY_RATE + int "CPU Entropy Source Entropy Rate" + depends on LRNG_CPU + range 0 256 + default 8 + help + The option defines the amount of entropy the LRNG applies to 256 + bits of data obtained from the CPU entropy source. The LRNG + enforces the limit that this value must be in the range between + 0 and 256. + + When configuring this value to 0, the CPU entropy source will + provide 256 bits of data without being credited to contain + entropy. + + Note, this option is overwritten when the option + CONFIG_RANDOM_TRUST_CPU is set. + +comment "Scheduler Entropy Source" + +config LRNG_SCHED + bool "Enable Scheduer Entropy Source as LRNG Seed Source" + select LRNG_TIMER_COMMON + help + The LRNG models an entropy source based on the timing of the + occurrence of scheduler-triggered context switches. Enable + this option to enable this scheduler entropy source. + + The scheduler entropy source is triggered every time a + context switch is triggered thus causes the scheduler to + execute slightly longer. Disabling the scheduler entropy + source implies that the performance penalty on the scheduler + added by the LRNG is eliminated. Yet, this entropy source is + considered to be an internal entropy source of the LRNG. + Thus, only disable it if you ensured that other entropy + sources are available that supply the LRNG with entropy. + + If you disable the scheduler entropy source, you MUST + ensure one or more entropy sources collectively have the + capability to deliver sufficient entropy with one invocation + at a rate compliant to the security strength of the DRNG + (usually 256 bits of entropy). In addition, if those + entropy sources do not deliver sufficient entropy during + first request, the reseed must be triggered from user + space or kernel space when sufficient entropy is considered + to be present. + + If unsure, say Y. + +config LRNG_SCHED_ENTROPY_RATE + int "Scheduler Entropy Source Entropy Rate" + depends on LRNG_SCHED + range 256 4294967295 if LRNG_SCHED_DFLT_TIMER_ES + range 4294967295 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES + default 256 if LRNG_SCHED_DFLT_TIMER_ES + default 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES + help + The LRNG will collect the configured number of context switches + triggered by the scheduler to obtain 256 bits of entropy. This + value can be set to any between 256 and 4294967295. The LRNG + guarantees that this value is not lower than 256. This lower + limit implies that one interrupt event is credited with one bit + of entropy. This value is subject to the increase by the + oversampling factor, if no high-resolution timer is found. + + In order to effectively disable the scheduler entropy source, + the option has to be set to 4294967295. In this case, the + scheduler entropy source will still deliver data but without + being credited with entropy. + +comment "Kernel RNG Entropy Source" + +config LRNG_KERNEL_RNG + bool "Enable Kernel RNG as LRNG Seed Source" + depends on RANDOM_DEFAULT_IMPL + help + The LRNG may use the kernel RNG (random.c) as entropy + source. + +config LRNG_KERNEL_RNG_ENTROPY_RATE + int "Kernel RNG Entropy Source Entropy Rate" + depends on LRNG_KERNEL_RNG + range 0 256 + default 256 + help + The option defines the amount of entropy the LRNG applies to 256 + bits of data obtained from the kernel RNG entropy source. The + LRNG enforces the limit that this value must be in the range + between 0 and 256. + + When configuring this value to 0, the kernel RNG entropy source + will provide 256 bits of data without being credited to contain + entropy. + + Note: This value is set to 0 automatically when booting the + kernel in FIPS mode (with fips=1 kernel command line option). + This is due to the fact that random.c is not SP800-90B + compliant. + +endmenu # "Entropy Source Configuration" + +config LRNG_DRNG_CHACHA20 + tristate + +config LRNG_DRBG + tristate + depends on CRYPTO + select CRYPTO_DRBG_MENU + +config LRNG_DRNG_KCAPI + tristate + depends on CRYPTO + select CRYPTO_RNG + +config LRNG_SWITCH + bool + +menuconfig LRNG_SWITCH_HASH + bool "Support conditioning hash runtime switching" + select LRNG_SWITCH + help + The LRNG uses a default message digest. With this + configuration option other message digests can be selected + and loaded at runtime. + +if LRNG_SWITCH_HASH + +config LRNG_HASH_KCAPI + tristate "Kernel crypto API hashing support for LRNG" + select CRYPTO_HASH + select CRYPTO_SHA512 + help + Enable the kernel crypto API support for entropy compression + and conditioning functions. + +endif # LRNG_SWITCH_HASH + +menuconfig LRNG_SWITCH_DRNG + bool "Support DRNG runtime switching" + select LRNG_SWITCH + help + The LRNG uses a default DRNG With this configuration + option other DRNGs or message digests can be selected and + loaded at runtime. + +if LRNG_SWITCH_DRNG + +config LRNG_SWITCH_DRNG_CHACHA20 + tristate "ChaCha20-based DRNG support for LRNG" + depends on !LRNG_DFLT_DRNG_CHACHA20 + select LRNG_DRNG_CHACHA20 + help + Enable the ChaCha20-based DRNG. This DRNG implementation + does not depend on the kernel crypto API presence. + +config LRNG_SWITCH_DRBG + tristate "SP800-90A support for the LRNG" + depends on !LRNG_DFLT_DRNG_DRBG + select LRNG_DRBG + help + Enable the SP800-90A DRBG support for the LRNG. Once the + module is loaded, output from /dev/random, /dev/urandom, + getrandom(2), or get_random_bytes_full is provided by a DRBG. + +config LRNG_SWITCH_DRNG_KCAPI + tristate "Kernel Crypto API support for the LRNG" + depends on !LRNG_DFLT_DRNG_KCAPI + depends on !LRNG_SWITCH_DRBG + select LRNG_DRNG_KCAPI + help + Enable the support for generic pseudo-random number + generators offered by the kernel crypto API with the + LRNG. Once the module is loaded, output from /dev/random, + /dev/urandom, getrandom(2), or get_random_bytes is + provided by the selected kernel crypto API RNG. + +endif # LRNG_SWITCH_DRNG + +choice + prompt "LRNG Default DRNG" + default LRNG_DFLT_DRNG_CHACHA20 + help + Select the default deterministic random number generator + that is used by the LRNG. When enabling the switchable + cryptographic mechanism support, this DRNG can be + replaced at runtime. + + config LRNG_DFLT_DRNG_CHACHA20 + bool "ChaCha20-based DRNG" + select LRNG_DRNG_CHACHA20 + + config LRNG_DFLT_DRNG_DRBG + bool "SP800-90A DRBG" + select LRNG_DRBG + + config LRNG_DFLT_DRNG_KCAPI + bool "Kernel Crypto API DRNG" + select LRNG_DRNG_KCAPI +endchoice + +menuconfig LRNG_TESTING_MENU + bool "LRNG testing interfaces" + depends on DEBUG_FS + help + Enable one or more of the following test interfaces. + + If unsure, say N. + +if LRNG_TESTING_MENU + +config LRNG_TESTING + bool + +config LRNG_TESTING_RECORDING + bool + +comment "Interrupt Entropy Source Test Interfaces" + +config LRNG_RAW_HIRES_ENTROPY + bool "Interface to obtain raw unprocessed IRQ noise source data" + default y + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned high resolution time stamp noise that + is collected by the LRNG for statistical analysis. Extracted + noise data is not used to seed the LRNG. + + The raw noise data can be obtained using the lrng_raw_hires + debugfs file. Using the option lrng_testing.boot_raw_hires_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_JIFFIES_ENTROPY + bool "Entropy test interface to Jiffies of IRQ noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned Jiffies that is collected by + the LRNG for statistical analysis. This data is used for + seeding the LRNG if a high-resolution time stamp is not + available. If a high-resolution time stamp is detected, + the Jiffies value is not collected by the LRNG and no + data is provided via the test interface. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the lrng_raw_jiffies + debugfs file. Using the option lrng_testing.boot_raw_jiffies_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_IRQ_ENTROPY + bool "Entropy test interface to IRQ number noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned interrupt number that is collected by + the LRNG for statistical analysis. Extracted noise data is + not used to seed the random number generator. + + The raw noise data can be obtained using the lrng_raw_irq + debugfs file. Using the option lrng_testing.boot_raw_irq_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_RETIP_ENTROPY + bool "Entropy test interface to RETIP value of IRQ noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned return instruction pointer value + that is collected by the LRNG for statistical analysis. + Extracted noise data is not used to seed the random number + generator. + + The raw noise data can be obtained using the lrng_raw_retip + debugfs file. Using the option lrng_testing.boot_raw_retip_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_REGS_ENTROPY + bool "Entropy test interface to IRQ register value noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned interrupt register value that is + collected by the LRNG for statistical analysis. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the lrng_raw_regs + debugfs file. Using the option lrng_testing.boot_raw_regs_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_ARRAY + bool "Test interface to LRNG raw entropy IRQ storage array" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw noise data that is collected by the LRNG + in the per-CPU array for statistical analysis. The purpose + of this interface is to verify that the array handling code + truly only concatenates data and provides the same entropy + rate as the raw unconditioned noise source when assessing + the collected data byte-wise. + + The data can be obtained using the lrng_raw_array debugfs + file. Using the option lrng_testing.boot_raw_array=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_IRQ_PERF + bool "LRNG interrupt entropy source performance monitor" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + With this option, the performance monitor of the LRNG + interrupt handling code is enabled. The file provides + the execution time of the interrupt handler in + cycles. + + The interrupt performance data can be obtained using + the lrng_irq_perf debugfs file. Using the option + lrng_testing.boot_irq_perf=1 the performance data of + the first 1000 entropy events since boot can be sampled. + +comment "Scheduler Entropy Source Test Interfaces" + +config LRNG_RAW_SCHED_HIRES_ENTROPY + bool "Interface to obtain raw unprocessed scheduler noise source data" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned high resolution time stamp noise that + is collected by the LRNG for the Scheduler-based noise source + for statistical analysis. Extracted noise data is not used to + seed the LRNG. + + The raw noise data can be obtained using the lrng_raw_sched_hires + debugfs file. Using the option + lrng_testing.boot_raw_sched_hires_test=1 the raw noise of the + first 1000 entropy events since boot can be sampled. + +config LRNG_RAW_SCHED_PID_ENTROPY + bool "Entropy test interface to PID value" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned PID value that is collected by the + LRNG for statistical analysis. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the + lrng_raw_sched_pid debugfs file. Using the option + lrng_testing.boot_raw_sched_pid_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_SCHED_START_TIME_ENTROPY + bool "Entropy test interface to task start time value" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned task start time value that is collected + by the LRNG for statistical analysis. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the + lrng_raw_sched_starttime debugfs file. Using the option + lrng_testing.boot_raw_sched_starttime_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + + +config LRNG_RAW_SCHED_NVCSW_ENTROPY + bool "Entropy test interface to task context switch numbers" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned task numbers of context switches that + are collected by the LRNG for statistical analysis. Extracted + noise data is not used to seed the random number generator. + + The raw noise data can be obtained using the + lrng_raw_sched_nvcsw debugfs file. Using the option + lrng_testing.boot_raw_sched_nvcsw_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_SCHED_PERF + bool "LRNG scheduler entropy source performance monitor" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + With this option, the performance monitor of the LRNG + scheduler event handling code is enabled. The file provides + the execution time of the interrupt handler in cycles. + + The scheduler performance data can be obtained using + the lrng_sched_perf debugfs file. Using the option + lrng_testing.boot_sched_perf=1 the performance data of + the first 1000 entropy events since boot can be sampled. + +comment "Auxiliary Test Interfaces" + +config LRNG_ACVT_HASH + bool "Enable LRNG ACVT Hash interface" + select LRNG_TESTING + help + With this option, the LRNG built-in hash function used for + auxiliary pool management and prior to switching the + cryptographic backends is made available for ACVT. The + interface allows writing of the data to be hashed + into the interface. The read operation triggers the hash + operation to generate message digest. + + The ACVT interface is available with the lrng_acvt_hash + debugfs file. + +config LRNG_RUNTIME_MAX_WO_RESEED_CONFIG + bool "Enable runtime configuration of max reseed threshold" + help + When enabling this option, the LRNG provides an interface + allowing the setting of the maximum number of DRNG generate + operations without a reseed that has full entropy. The + interface is lrng_drng.max_wo_reseed. + +config LRNG_TEST_CPU_ES_COMPRESSION + bool "Force CPU ES compression operation" + help + When enabling this option, the CPU ES compression operation + is forced by setting an arbitrary value > 1 for the data + multiplier even when the CPU ES would deliver full entropy. + This allows testing of the compression operation. It + therefore forces to pull more data from the CPU ES + than what may be required. + +endif #LRNG_TESTING_MENU + +config LRNG_SELFTEST + bool "Enable power-on and on-demand self-tests" + help + The power-on self-tests are executed during boot time + covering the ChaCha20 DRNG, the hash operation used for + processing the entropy pools and the auxiliary pool, and + the time stamp management of the LRNG. + + The on-demand self-tests are triggered by writing any + value into the SysFS file selftest_status. At the same + time, when reading this file, the test status is + returned. A zero indicates that all tests were executed + successfully. + + If unsure, say Y. + +if LRNG_SELFTEST + +config LRNG_SELFTEST_PANIC + bool "Panic the kernel upon self-test failure" + help + If the option is enabled, the kernel is terminated if an + LRNG power-on self-test failure is detected. + +endif # LRNG_SELFTEST + +endif # LRNG + +endmenu # LRNG diff --git a/drivers/char/lrng/Makefile b/drivers/char/lrng/Makefile new file mode 100644 index 000000000000..f61fc40f4620 --- /dev/null +++ b/drivers/char/lrng/Makefile @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Entropy Source and DRNG Manager. +# + +obj-y += lrng_es_mgr.o lrng_drng_mgr.o \ + lrng_es_aux.o +obj-$(CONFIG_LRNG_SHA256) += lrng_sha256.o +obj-$(CONFIG_LRNG_SHA1) += lrng_sha1.o + +obj-$(CONFIG_SYSCTL) += lrng_proc.o +obj-$(CONFIG_LRNG_SYSCTL) += lrng_sysctl.o +obj-$(CONFIG_NUMA) += lrng_numa.o + +obj-$(CONFIG_LRNG_SWITCH) += lrng_switch.o +obj-$(CONFIG_LRNG_HASH_KCAPI) += lrng_hash_kcapi.o +obj-$(CONFIG_LRNG_DRNG_CHACHA20) += lrng_drng_chacha20.o +obj-$(CONFIG_LRNG_DRBG) += lrng_drng_drbg.o +obj-$(CONFIG_LRNG_DRNG_KCAPI) += lrng_drng_kcapi.o +obj-$(CONFIG_LRNG_DRNG_ATOMIC) += lrng_drng_atomic.o + +obj-$(CONFIG_LRNG_TIMER_COMMON) += lrng_es_timer_common.o +obj-$(CONFIG_LRNG_IRQ) += lrng_es_irq.o +obj-$(CONFIG_LRNG_KERNEL_RNG) += lrng_es_krng.o +obj-$(CONFIG_LRNG_SCHED) += lrng_es_sched.o +obj-$(CONFIG_LRNG_CPU) += lrng_es_cpu.o +obj-$(CONFIG_LRNG_JENT) += lrng_es_jent.o + +obj-$(CONFIG_LRNG_HEALTH_TESTS) += lrng_health.o +obj-$(CONFIG_LRNG_TESTING) += lrng_testing.o +obj-$(CONFIG_LRNG_SELFTEST) += lrng_selftest.o + +obj-$(CONFIG_LRNG_COMMON_DEV_IF) += lrng_interface_dev_common.o +obj-$(CONFIG_LRNG_RANDOM_IF) += lrng_interface_random_user.o \ + lrng_interface_random_kernel.o \ + lrng_interface_aux.o +obj-$(CONFIG_LRNG_KCAPI_IF) += lrng_interface_kcapi.o +obj-$(CONFIG_LRNG_DEV_IF) += lrng_interface_dev.o +obj-$(CONFIG_LRNG_HWRAND_IF) += lrng_interface_hwrand.o diff --git a/drivers/char/lrng/lrng_definitions.h b/drivers/char/lrng/lrng_definitions.h new file mode 100644 index 000000000000..5dbfa68d4223 --- /dev/null +++ b/drivers/char/lrng/lrng_definitions.h @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DEFINITIONS_H +#define _LRNG_DEFINITIONS_H + +#include +#include + +/*************************** General LRNG parameter ***************************/ + +/* + * Specific settings for different use cases + */ +#ifdef CONFIG_CRYPTO_FIPS +# define LRNG_OVERSAMPLE_ES_BITS 64 +# define LRNG_SEED_BUFFER_INIT_ADD_BITS 128 +#else /* CONFIG_CRYPTO_FIPS */ +# define LRNG_OVERSAMPLE_ES_BITS 0 +# define LRNG_SEED_BUFFER_INIT_ADD_BITS 0 +#endif /* CONFIG_CRYPTO_FIPS */ + +/* Security strength of LRNG -- this must match DRNG security strength */ +#define LRNG_DRNG_SECURITY_STRENGTH_BYTES 32 +#define LRNG_DRNG_SECURITY_STRENGTH_BITS (LRNG_DRNG_SECURITY_STRENGTH_BYTES * 8) +#define LRNG_DRNG_INIT_SEED_SIZE_BITS \ + (LRNG_DRNG_SECURITY_STRENGTH_BITS + LRNG_SEED_BUFFER_INIT_ADD_BITS) +#define LRNG_DRNG_INIT_SEED_SIZE_BYTES (LRNG_DRNG_INIT_SEED_SIZE_BITS >> 3) + +/* + * SP800-90A defines a maximum request size of 1<<16 bytes. The given value is + * considered a safer margin. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_MAX_REQSIZE (1<<12) + +/* + * SP800-90A defines a maximum number of requests between reseeds of 2^48. + * The given value is considered a much safer margin, balancing requests for + * frequent reseeds with the need to conserve entropy. This value MUST NOT be + * larger than INT_MAX because it is used in an atomic_t. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_RESEED_THRESH (1<<20) + +/* + * Maximum DRNG generation operations without reseed having full entropy + * This value defines the absolute maximum value of DRNG generation operations + * without a reseed holding full entropy. LRNG_DRNG_RESEED_THRESH is the + * threshold when a new reseed is attempted. But it is possible that this fails + * to deliver full entropy. In this case the DRNG will continue to provide data + * even though it was not reseeded with full entropy. To avoid in the extreme + * case that no reseed is performed for too long, this threshold is enforced. + * If that absolute low value is reached, the LRNG is marked as not operational. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_MAX_WITHOUT_RESEED (1<<30) + +/* + * Min required seed entropy is 128 bits covering the minimum entropy + * requirement of SP800-131A and the German BSI's TR02102. + * + * This value is allowed to be changed. + */ +#define LRNG_FULL_SEED_ENTROPY_BITS LRNG_DRNG_SECURITY_STRENGTH_BITS +#define LRNG_MIN_SEED_ENTROPY_BITS 128 +#define LRNG_INIT_ENTROPY_BITS 32 + +/* + * Wakeup value + * + * This value is allowed to be changed but must not be larger than the + * digest size of the hash operation used update the aux_pool. + */ +#ifdef CONFIG_LRNG_SHA256 +# define LRNG_ATOMIC_DIGEST_SIZE SHA256_DIGEST_SIZE +#else +# define LRNG_ATOMIC_DIGEST_SIZE SHA1_DIGEST_SIZE +#endif +#define LRNG_WRITE_WAKEUP_ENTROPY LRNG_ATOMIC_DIGEST_SIZE + +/* + * If the switching support is configured, we must provide support up to + * the largest digest size. Without switching support, we know it is only + * the built-in digest size. + */ +#ifdef CONFIG_LRNG_SWITCH +# define LRNG_MAX_DIGESTSIZE 64 +#else +# define LRNG_MAX_DIGESTSIZE LRNG_ATOMIC_DIGEST_SIZE +#endif + +/* + * Oversampling factor of timer-based events to obtain + * LRNG_DRNG_SECURITY_STRENGTH_BYTES. This factor is used when a + * high-resolution time stamp is not available. In this case, jiffies and + * register contents are used to fill the entropy pool. These noise sources + * are much less entropic than the high-resolution timer. The entropy content + * is the entropy content assumed with LRNG_[IRQ|SCHED]_ENTROPY_BITS divided by + * LRNG_ES_OVERSAMPLING_FACTOR. + * + * This value is allowed to be changed. + */ +#define LRNG_ES_OVERSAMPLING_FACTOR 10 + +/* Alignmask that is intended to be identical to CRYPTO_MINALIGN */ +#define LRNG_KCAPI_ALIGN ARCH_KMALLOC_MINALIGN + +/* + * This definition must provide a buffer that is equal to SHASH_DESC_ON_STACK + * as it will be casted into a struct shash_desc. + */ +#define LRNG_POOL_SIZE (sizeof(struct shash_desc) + HASH_MAX_DESCSIZE) + +/****************************** Helper code ***********************************/ + +static inline u32 lrng_fast_noise_entropylevel(u32 ent_bits, u32 requested_bits) +{ + /* Obtain entropy statement */ + ent_bits = ent_bits * requested_bits / LRNG_DRNG_SECURITY_STRENGTH_BITS; + /* Cap entropy to buffer size in bits */ + ent_bits = min_t(u32, ent_bits, requested_bits); + return ent_bits; +} + +/* Convert entropy in bits into nr. of events with the same entropy content. */ +static inline u32 lrng_entropy_to_data(u32 entropy_bits, u32 entropy_rate) +{ + return ((entropy_bits * entropy_rate) / + LRNG_DRNG_SECURITY_STRENGTH_BITS); +} + +/* Convert number of events into entropy value. */ +static inline u32 lrng_data_to_entropy(u32 num, u32 entropy_rate) +{ + return ((num * LRNG_DRNG_SECURITY_STRENGTH_BITS) / + entropy_rate); +} + +static inline u32 atomic_read_u32(atomic_t *v) +{ + return (u32)atomic_read(v); +} + +#endif /* _LRNG_DEFINITIONS_H */ diff --git a/drivers/char/lrng/lrng_drng_atomic.c b/drivers/char/lrng/lrng_drng_atomic.c new file mode 100644 index 000000000000..e3a8fece9935 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_atomic.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG DRNG for atomic contexts + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include + +#include "lrng_drng_atomic.h" +#include "lrng_drng_chacha20.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_sha.h" + +static struct chacha20_state chacha20_atomic = { + LRNG_CC20_INIT_RFC7539(.block) +}; + +/* + * DRNG usable in atomic context. This DRNG will always use the ChaCha20 + * DRNG. It will never benefit from a DRNG switch like the "regular" DRNG. If + * there was no DRNG switch, the atomic DRNG is identical to the "regular" DRNG. + * + * The reason for having this is due to the fact that DRNGs other than + * the ChaCha20 DRNG may sleep. + */ +static struct lrng_drng lrng_drng_atomic = { + LRNG_DRNG_STATE_INIT(lrng_drng_atomic, + &chacha20_atomic, NULL, + &lrng_cc20_drng_cb, &lrng_sha_hash_cb), + .spin_lock = __SPIN_LOCK_UNLOCKED(lrng_drng_atomic.spin_lock) +}; + +void lrng_drng_atomic_reset(void) +{ + unsigned long flags; + + spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); + lrng_drng_reset(&lrng_drng_atomic); + spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); +} + +void lrng_drng_atomic_force_reseed(void) +{ + lrng_drng_atomic.force_reseed = lrng_drng_atomic.fully_seeded; +} + +static bool lrng_drng_atomic_must_reseed(struct lrng_drng *drng) +{ + return (!drng->fully_seeded || + atomic_read(&lrng_drng_atomic.requests) == 0 || + drng->force_reseed || + time_after(jiffies, + drng->last_seeded + lrng_drng_reseed_max_time * HZ)); +} + +void lrng_drng_atomic_seed_drng(struct lrng_drng *regular_drng) +{ + u8 seedbuf[LRNG_DRNG_SECURITY_STRENGTH_BYTES] + __aligned(LRNG_KCAPI_ALIGN); + int ret; + + if (!lrng_drng_atomic_must_reseed(&lrng_drng_atomic)) + return; + + /* + * Reseed atomic DRNG another DRNG "regular" while this regular DRNG + * is reseeded. Therefore, this code operates in non-atomic context and + * thus can use the lrng_drng_get function to get random numbers from + * the just freshly seeded DRNG. + */ + ret = lrng_drng_get(regular_drng, seedbuf, sizeof(seedbuf)); + + if (ret < 0) { + pr_warn("Error generating random numbers for atomic DRNG: %d\n", + ret); + } else { + unsigned long flags; + + spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); + lrng_drng_inject(&lrng_drng_atomic, seedbuf, ret, + regular_drng->fully_seeded, "atomic"); + spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); + } + memzero_explicit(&seedbuf, sizeof(seedbuf)); +} + +void lrng_drng_atomic_seed_es(void) +{ + struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); + struct lrng_drng *drng = &lrng_drng_atomic; + unsigned long flags; + u32 requested_bits = LRNG_MIN_SEED_ENTROPY_BITS; + + if (lrng_drng_atomic.fully_seeded) + return; + + /* + * When the LRNG reached the minimally seeded level, leave 256 bits of + * entropy for the regular DRNG. In addition ensure that additional + * 256 bits are available for the atomic DRNG to get to the fully + * seeded stage of the LRNG. + */ + if (lrng_state_min_seeded()) { + u32 avail_bits = lrng_avail_entropy(); + + requested_bits = + (avail_bits >= 2 * LRNG_FULL_SEED_ENTROPY_BITS) ? + LRNG_FULL_SEED_ENTROPY_BITS : 0; + + if (!requested_bits) + return; + } + + pr_debug("atomic DRNG seeding attempt to pull %u bits of entropy directly from entropy sources\n", + requested_bits); + + lrng_fill_seed_buffer(&seedbuf, requested_bits); + spin_lock_irqsave(&drng->spin_lock, flags); + lrng_drng_inject(&lrng_drng_atomic, (u8 *)&seedbuf, sizeof(seedbuf), + lrng_fully_seeded_eb(drng->fully_seeded, &seedbuf), + "atomic"); + spin_unlock_irqrestore(&drng->spin_lock, flags); + lrng_init_ops(&seedbuf); + + /* + * Do not call lrng_init_ops(seedbuf) here as the atomic DRNG does not + * serve common users. + */ + memzero_explicit(&seedbuf, sizeof(seedbuf)); +} + +static void lrng_drng_atomic_get(u8 *outbuf, u32 outbuflen) +{ + struct lrng_drng *drng = &lrng_drng_atomic; + unsigned long flags; + + if (!outbuf || !outbuflen) + return; + + outbuflen = min_t(size_t, outbuflen, INT_MAX); + + while (outbuflen) { + u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); + int ret; + + atomic_dec(&drng->requests); + + spin_lock_irqsave(&drng->spin_lock, flags); + ret = drng->drng_cb->drng_generate(drng->drng, outbuf, todo); + spin_unlock_irqrestore(&drng->spin_lock, flags); + if (ret <= 0) { + pr_warn("getting random data from DRNG failed (%d)\n", + ret); + return; + } + outbuflen -= ret; + outbuf += ret; + } +} + +void lrng_get_random_bytes(void *buf, int nbytes) +{ + lrng_drng_atomic_get((u8 *)buf, (u32)nbytes); + lrng_debug_report_seedlevel("lrng_get_random_bytes"); +} +EXPORT_SYMBOL(lrng_get_random_bytes); diff --git a/drivers/char/lrng/lrng_drng_atomic.h b/drivers/char/lrng/lrng_drng_atomic.h new file mode 100644 index 000000000000..d5d24fc94f2e --- /dev/null +++ b/drivers/char/lrng/lrng_drng_atomic.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DRNG_ATOMIC_H +#define _LRNG_DRNG_ATOMIC_H + +#include "lrng_drng_mgr.h" + +#ifdef CONFIG_LRNG_DRNG_ATOMIC +void lrng_drng_atomic_reset(void); +void lrng_drng_atomic_seed_drng(struct lrng_drng *drng); +void lrng_drng_atomic_seed_es(void); +void lrng_drng_atomic_force_reseed(void); +#else /* CONFIG_LRNG_DRNG_ATOMIC */ +static inline void lrng_drng_atomic_reset(void) { } +static inline void lrng_drng_atomic_seed_drng(struct lrng_drng *drng) { } +static inline void lrng_drng_atomic_seed_es(void) { } +static inline void lrng_drng_atomic_force_reseed(void) { } +#endif /* CONFIG_LRNG_DRNG_ATOMIC */ + +#endif /* _LRNG_DRNG_ATOMIC_H */ diff --git a/drivers/char/lrng/lrng_drng_chacha20.c b/drivers/char/lrng/lrng_drng_chacha20.c new file mode 100644 index 000000000000..31be102e3007 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_chacha20.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the cryptographic primitives using + * ChaCha20 cipher implementations. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_drng_chacha20.h" + +/******************************* ChaCha20 DRNG *******************************/ + +#define CHACHA_BLOCK_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) + +/* + * Update of the ChaCha20 state by either using an unused buffer part or by + * generating one ChaCha20 block which is half of the state of the ChaCha20. + * The block is XORed into the key part of the state. This shall ensure + * backtracking resistance as well as a proper mix of the ChaCha20 state once + * the key is injected. + */ +static void lrng_chacha20_update(struct chacha20_state *chacha20_state, + __le32 *buf, u32 used_words) +{ + struct chacha20_block *chacha20 = &chacha20_state->block; + u32 i; + __le32 tmp[CHACHA_BLOCK_WORDS]; + + BUILD_BUG_ON(sizeof(struct chacha20_block) != CHACHA_BLOCK_SIZE); + BUILD_BUG_ON(CHACHA_BLOCK_SIZE != 2 * CHACHA_KEY_SIZE); + + if (used_words > CHACHA_KEY_SIZE_WORDS) { + chacha20_block(&chacha20->constants[0], (u8 *)tmp); + for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) + chacha20->key.u[i] ^= le32_to_cpu(tmp[i]); + memzero_explicit(tmp, sizeof(tmp)); + } else { + for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) + chacha20->key.u[i] ^= le32_to_cpu(buf[i + used_words]); + } + + /* Deterministic increment of nonce as required in RFC 7539 chapter 4 */ + chacha20->nonce[0]++; + if (chacha20->nonce[0] == 0) { + chacha20->nonce[1]++; + if (chacha20->nonce[1] == 0) + chacha20->nonce[2]++; + } + + /* Leave counter untouched as it is start value is undefined in RFC */ +} + +/* + * Seed the ChaCha20 DRNG by injecting the input data into the key part of + * the ChaCha20 state. If the input data is longer than the ChaCha20 key size, + * perform a ChaCha20 operation after processing of key size input data. + * This operation shall spread out the entropy into the ChaCha20 state before + * new entropy is injected into the key part. + */ +static int lrng_cc20_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) +{ + struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; + struct chacha20_block *chacha20 = &chacha20_state->block; + + while (inbuflen) { + u32 i, todo = min_t(u32, inbuflen, CHACHA_KEY_SIZE); + + for (i = 0; i < todo; i++) + chacha20->key.b[i] ^= inbuf[i]; + + /* Break potential dependencies between the inbuf key blocks */ + lrng_chacha20_update(chacha20_state, NULL, + CHACHA_BLOCK_WORDS); + inbuf += todo; + inbuflen -= todo; + } + + return 0; +} + +/* + * Chacha20 DRNG generation of random numbers: the stream output of ChaCha20 + * is the random number. After the completion of the generation of the + * stream, the entire ChaCha20 state is updated. + * + * Note, as the ChaCha20 implements a 32 bit counter, we must ensure + * that this function is only invoked for at most 2^32 - 1 ChaCha20 blocks + * before a reseed or an update happens. This is ensured by the variable + * outbuflen which is a 32 bit integer defining the number of bytes to be + * generated by the ChaCha20 DRNG. At the end of this function, an update + * operation is invoked which implies that the 32 bit counter will never be + * overflown in this implementation. + */ +static int lrng_cc20_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) +{ + struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; + struct chacha20_block *chacha20 = &chacha20_state->block; + __le32 aligned_buf[CHACHA_BLOCK_WORDS]; + u32 ret = outbuflen, used = CHACHA_BLOCK_WORDS; + int zeroize_buf = 0; + + while (outbuflen >= CHACHA_BLOCK_SIZE) { + chacha20_block(&chacha20->constants[0], outbuf); + outbuf += CHACHA_BLOCK_SIZE; + outbuflen -= CHACHA_BLOCK_SIZE; + } + + if (outbuflen) { + chacha20_block(&chacha20->constants[0], (u8 *)aligned_buf); + memcpy(outbuf, aligned_buf, outbuflen); + used = ((outbuflen + sizeof(aligned_buf[0]) - 1) / + sizeof(aligned_buf[0])); + zeroize_buf = 1; + } + + lrng_chacha20_update(chacha20_state, aligned_buf, used); + + if (zeroize_buf) + memzero_explicit(aligned_buf, sizeof(aligned_buf)); + + return ret; +} + +/* + * Allocation of the DRNG state + */ +static void *lrng_cc20_drng_alloc(u32 sec_strength) +{ + struct chacha20_state *state = NULL; + + if (sec_strength > CHACHA_KEY_SIZE) { + pr_err("Security strength of ChaCha20 DRNG (%u bits) lower than requested by LRNG (%u bits)\n", + CHACHA_KEY_SIZE * 8, sec_strength * 8); + return ERR_PTR(-EINVAL); + } + if (sec_strength < CHACHA_KEY_SIZE) + pr_warn("Security strength of ChaCha20 DRNG (%u bits) higher than requested by LRNG (%u bits)\n", + CHACHA_KEY_SIZE * 8, sec_strength * 8); + + state = kmalloc(sizeof(struct chacha20_state), GFP_KERNEL); + if (!state) + return ERR_PTR(-ENOMEM); + pr_debug("memory for ChaCha20 core allocated\n"); + + lrng_cc20_init_rfc7539(&state->block); + + return state; +} + +static void lrng_cc20_drng_dealloc(void *drng) +{ + struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; + + pr_debug("ChaCha20 core zeroized and freed\n"); + kfree_sensitive(chacha20_state); +} + +static const char *lrng_cc20_drng_name(void) +{ + return "ChaCha20 DRNG"; +} + +const struct lrng_drng_cb lrng_cc20_drng_cb = { + .drng_name = lrng_cc20_drng_name, + .drng_alloc = lrng_cc20_drng_alloc, + .drng_dealloc = lrng_cc20_drng_dealloc, + .drng_seed = lrng_cc20_drng_seed_helper, + .drng_generate = lrng_cc20_drng_generate_helper, +}; + +#if !defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) && \ + !defined(CONFIG_LRNG_DRNG_ATOMIC) +static int __init lrng_cc20_drng_init(void) +{ + return lrng_set_drng_cb(&lrng_cc20_drng_cb); +} + +static void __exit lrng_cc20_drng_exit(void) +{ + lrng_set_drng_cb(NULL); +} + +late_initcall(lrng_cc20_drng_init); +module_exit(lrng_cc20_drng_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - ChaCha20-based DRNG backend"); +#endif /* CONFIG_LRNG_DFLT_DRNG_CHACHA20 */ diff --git a/drivers/char/lrng/lrng_drng_chacha20.h b/drivers/char/lrng/lrng_drng_chacha20.h new file mode 100644 index 000000000000..61736afee464 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_chacha20.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG ChaCha20 definitions + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_CHACHA20_H +#define _LRNG_CHACHA20_H + +#include + +/* State according to RFC 7539 section 2.3 */ +struct chacha20_block { + u32 constants[4]; + union { +#define CHACHA_KEY_SIZE_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) + u32 u[CHACHA_KEY_SIZE_WORDS]; + u8 b[CHACHA_KEY_SIZE]; + } key; + u32 counter; + u32 nonce[3]; +}; + +struct chacha20_state { + struct chacha20_block block; +}; + +static inline void lrng_cc20_init_rfc7539(struct chacha20_block *chacha20) +{ + /* String "expand 32-byte k" */ + chacha20->constants[0] = 0x61707865; + chacha20->constants[1] = 0x3320646e; + chacha20->constants[2] = 0x79622d32; + chacha20->constants[3] = 0x6b206574; +} + +#define LRNG_CC20_INIT_RFC7539(x) \ + x.constants[0] = 0x61707865, \ + x.constants[1] = 0x3320646e, \ + x.constants[2] = 0x79622d32, \ + x.constants[3] = 0x6b206574, + +extern const struct lrng_drng_cb lrng_cc20_drng_cb; + +#endif /* _LRNG_CHACHA20_H */ diff --git a/drivers/char/lrng/lrng_drng_drbg.c b/drivers/char/lrng/lrng_drng_drbg.c new file mode 100644 index 000000000000..1feec2c095a5 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_drbg.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the cryptographic primitives using the + * kernel crypto API and its DRBG. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_drng_drbg.h" + +/* + * Define a DRBG plus a hash / MAC used to extract data from the entropy pool. + * For LRNG_HASH_NAME you can use a hash or a MAC (HMAC or CMAC) of your choice + * (Note, you should use the suggested selections below -- using SHA-1 or MD5 + * is not wise). The idea is that the used cipher primitive can be selected to + * be the same as used for the DRBG. I.e. the LRNG only uses one cipher + * primitive using the same cipher implementation with the options offered in + * the following. This means, if the CTR DRBG is selected and AES-NI is present, + * both the CTR DRBG and the selected cmac(aes) use AES-NI. + * + * The security strengths of the DRBGs are all 256 bits according to + * SP800-57 section 5.6.1. + * + * This definition is allowed to be changed. + */ +#ifdef CONFIG_CRYPTO_DRBG_CTR +static unsigned int lrng_drbg_type = 0; +#elif defined CONFIG_CRYPTO_DRBG_HMAC +static unsigned int lrng_drbg_type = 1; +#elif defined CONFIG_CRYPTO_DRBG_HASH +static unsigned int lrng_drbg_type = 2; +#else +#error "Unknown DRBG in use" +#endif + +/* The parameter must be r/o in sysfs as otherwise races appear. */ +module_param(lrng_drbg_type, uint, 0444); +MODULE_PARM_DESC(lrng_drbg_type, "DRBG type used for LRNG (0->CTR_DRBG, 1->HMAC_DRBG, 2->Hash_DRBG)"); + +struct lrng_drbg { + const char *hash_name; + const char *drbg_core; +}; + +static const struct lrng_drbg lrng_drbg_types[] = { + { /* CTR_DRBG with AES-256 using derivation function */ + .drbg_core = "drbg_nopr_ctr_aes256", + }, { /* HMAC_DRBG with SHA-512 */ + .drbg_core = "drbg_nopr_hmac_sha512", + }, { /* Hash_DRBG with SHA-512 using derivation function */ + .drbg_core = "drbg_nopr_sha512" + } +}; + +static int lrng_drbg_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) +{ + struct drbg_state *drbg = (struct drbg_state *)drng; + LIST_HEAD(seedlist); + struct drbg_string data; + int ret; + + drbg_string_fill(&data, inbuf, inbuflen); + list_add_tail(&data.list, &seedlist); + ret = drbg->d_ops->update(drbg, &seedlist, drbg->seeded); + + if (ret >= 0) + drbg->seeded = true; + + return ret; +} + +static int lrng_drbg_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) +{ + struct drbg_state *drbg = (struct drbg_state *)drng; + + return drbg->d_ops->generate(drbg, outbuf, outbuflen, NULL); +} + +static void *lrng_drbg_drng_alloc(u32 sec_strength) +{ + struct drbg_state *drbg; + int coreref = -1; + bool pr = false; + int ret; + + drbg_convert_tfm_core(lrng_drbg_types[lrng_drbg_type].drbg_core, + &coreref, &pr); + if (coreref < 0) + return ERR_PTR(-EFAULT); + + drbg = kzalloc(sizeof(struct drbg_state), GFP_KERNEL); + if (!drbg) + return ERR_PTR(-ENOMEM); + + drbg->core = &drbg_cores[coreref]; + drbg->seeded = false; + ret = drbg_alloc_state(drbg); + if (ret) + goto err; + + if (sec_strength > drbg_sec_strength(drbg->core->flags)) { + pr_err("Security strength of DRBG (%u bits) lower than requested by LRNG (%u bits)\n", + drbg_sec_strength(drbg->core->flags) * 8, + sec_strength * 8); + goto dealloc; + } + + if (sec_strength < drbg_sec_strength(drbg->core->flags)) + pr_warn("Security strength of DRBG (%u bits) higher than requested by LRNG (%u bits)\n", + drbg_sec_strength(drbg->core->flags) * 8, + sec_strength * 8); + + pr_info("DRBG with %s core allocated\n", drbg->core->backend_cra_name); + + return drbg; + +dealloc: + if (drbg->d_ops) + drbg->d_ops->crypto_fini(drbg); + drbg_dealloc_state(drbg); +err: + kfree(drbg); + return ERR_PTR(-EINVAL); +} + +static void lrng_drbg_drng_dealloc(void *drng) +{ + struct drbg_state *drbg = (struct drbg_state *)drng; + + if (drbg && drbg->d_ops) + drbg->d_ops->crypto_fini(drbg); + drbg_dealloc_state(drbg); + kfree_sensitive(drbg); + pr_info("DRBG deallocated\n"); +} + +static const char *lrng_drbg_name(void) +{ + return lrng_drbg_types[lrng_drbg_type].drbg_core; +} + +const struct lrng_drng_cb lrng_drbg_cb = { + .drng_name = lrng_drbg_name, + .drng_alloc = lrng_drbg_drng_alloc, + .drng_dealloc = lrng_drbg_drng_dealloc, + .drng_seed = lrng_drbg_drng_seed_helper, + .drng_generate = lrng_drbg_drng_generate_helper, +}; + +#ifndef CONFIG_LRNG_DFLT_DRNG_DRBG +static int __init lrng_drbg_init(void) +{ + if (lrng_drbg_type >= ARRAY_SIZE(lrng_drbg_types)) { + pr_err("lrng_drbg_type parameter too large (given %u - max: %lu)", + lrng_drbg_type, + (unsigned long)ARRAY_SIZE(lrng_drbg_types) - 1); + return -EAGAIN; + } + return lrng_set_drng_cb(&lrng_drbg_cb); +} + +static void __exit lrng_drbg_exit(void) +{ + lrng_set_drng_cb(NULL); +} + +late_initcall(lrng_drbg_init); +module_exit(lrng_drbg_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - SP800-90A DRBG backend"); +#endif /* CONFIG_LRNG_DFLT_DRNG_DRBG */ diff --git a/drivers/char/lrng/lrng_drng_drbg.h b/drivers/char/lrng/lrng_drng_drbg.h new file mode 100644 index 000000000000..b3d556caf294 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_drbg.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG SP800-90A definitions + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DRBG_H +#define _LRNG_DRBG_H + +extern const struct lrng_drng_cb lrng_drbg_cb; + +#endif /* _LRNG_DRBG_H */ diff --git a/drivers/char/lrng/lrng_drng_kcapi.c b/drivers/char/lrng/lrng_drng_kcapi.c new file mode 100644 index 000000000000..469e707a4edc --- /dev/null +++ b/drivers/char/lrng/lrng_drng_kcapi.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the cryptographic primitives using the + * kernel crypto API. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +#include "lrng_drng_kcapi.h" + +static char *drng_name = NULL; +module_param(drng_name, charp, 0444); +MODULE_PARM_DESC(drng_name, "Kernel crypto API name of DRNG"); + +static char *seed_hash = NULL; +module_param(seed_hash, charp, 0444); +MODULE_PARM_DESC(seed_hash, + "Kernel crypto API name of hash with output size equal to seedsize of DRNG to bring seed string to the size required by the DRNG"); + +struct lrng_drng_info { + struct crypto_rng *kcapi_rng; + struct crypto_shash *hash_tfm; +}; + +static int lrng_kcapi_drng_seed_helper(void *drng, const u8 *inbuf, + u32 inbuflen) +{ + struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; + struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; + struct crypto_shash *hash_tfm = lrng_drng_info->hash_tfm; + SHASH_DESC_ON_STACK(shash, hash_tfm); + u32 digestsize; + u8 digest[64] __aligned(8); + int ret; + + if (!hash_tfm) + return crypto_rng_reset(kcapi_rng, inbuf, inbuflen); + + shash->tfm = hash_tfm; + digestsize = crypto_shash_digestsize(hash_tfm); + + ret = crypto_shash_digest(shash, inbuf, inbuflen, digest); + shash_desc_zero(shash); + if (ret) + return ret; + + ret = crypto_rng_reset(kcapi_rng, digest, digestsize); + if (ret) + return ret; + + memzero_explicit(digest, digestsize); + return 0; +} + +static int lrng_kcapi_drng_generate_helper(void *drng, u8 *outbuf, + u32 outbuflen) +{ + struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; + struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; + int ret = crypto_rng_get_bytes(kcapi_rng, outbuf, outbuflen); + + if (ret < 0) + return ret; + + return outbuflen; +} + +static void *lrng_kcapi_drng_alloc(u32 sec_strength) +{ + struct lrng_drng_info *lrng_drng_info; + struct crypto_rng *kcapi_rng; + u32 time = random_get_entropy(); + int seedsize, rv; + void *ret = ERR_PTR(-ENOMEM); + + if (!drng_name) { + pr_err("DRNG name missing\n"); + return ERR_PTR(-EINVAL); + } + + if (!memcmp(drng_name, "stdrng", 6) || + !memcmp(drng_name, "lrng", 4) || + !memcmp(drng_name, "drbg", 4) || + !memcmp(drng_name, "jitterentropy_rng", 17)) { + pr_err("Refusing to load the requested random number generator\n"); + return ERR_PTR(-EINVAL); + } + + lrng_drng_info = kzalloc(sizeof(*lrng_drng_info), GFP_KERNEL); + if (!lrng_drng_info) + return ERR_PTR(-ENOMEM); + + kcapi_rng = crypto_alloc_rng(drng_name, 0, 0); + if (IS_ERR(kcapi_rng)) { + pr_err("DRNG %s cannot be allocated\n", drng_name); + ret = ERR_CAST(kcapi_rng); + goto free; + } + + lrng_drng_info->kcapi_rng = kcapi_rng; + + seedsize = crypto_rng_seedsize(kcapi_rng); + if (seedsize) { + struct crypto_shash *hash_tfm; + + if (!seed_hash) { + switch (seedsize) { + case 32: + seed_hash = "sha256"; + break; + case 48: + seed_hash = "sha384"; + break; + case 64: + seed_hash = "sha512"; + break; + default: + pr_err("Seed size %d cannot be processed\n", + seedsize); + goto dealloc; + } + } + + hash_tfm = crypto_alloc_shash(seed_hash, 0, 0); + if (IS_ERR(hash_tfm)) { + ret = ERR_CAST(hash_tfm); + goto dealloc; + } + + if (seedsize != crypto_shash_digestsize(hash_tfm)) { + pr_err("Seed hash output size not equal to DRNG seed size\n"); + crypto_free_shash(hash_tfm); + ret = ERR_PTR(-EINVAL); + goto dealloc; + } + + lrng_drng_info->hash_tfm = hash_tfm; + + pr_info("Seed hash %s allocated\n", seed_hash); + } + + rv = lrng_kcapi_drng_seed_helper(lrng_drng_info, (u8 *)(&time), + sizeof(time)); + if (rv) { + ret = ERR_PTR(rv); + goto dealloc; + } + + pr_info("Kernel crypto API DRNG %s allocated\n", drng_name); + + return lrng_drng_info; + +dealloc: + crypto_free_rng(kcapi_rng); +free: + kfree(lrng_drng_info); + return ret; +} + +static void lrng_kcapi_drng_dealloc(void *drng) +{ + struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; + struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; + + crypto_free_rng(kcapi_rng); + if (lrng_drng_info->hash_tfm) + crypto_free_shash(lrng_drng_info->hash_tfm); + kfree(lrng_drng_info); + pr_info("DRNG %s deallocated\n", drng_name); +} + +static const char *lrng_kcapi_drng_name(void) +{ + return drng_name; +} + +const struct lrng_drng_cb lrng_kcapi_drng_cb = { + .drng_name = lrng_kcapi_drng_name, + .drng_alloc = lrng_kcapi_drng_alloc, + .drng_dealloc = lrng_kcapi_drng_dealloc, + .drng_seed = lrng_kcapi_drng_seed_helper, + .drng_generate = lrng_kcapi_drng_generate_helper, +}; + +#ifndef CONFIG_LRNG_DFLT_DRNG_KCAPI +static int __init lrng_kcapi_init(void) +{ + return lrng_set_drng_cb(&lrng_kcapi_drng_cb); +} +static void __exit lrng_kcapi_exit(void) +{ + lrng_set_drng_cb(NULL); +} + +late_initcall(lrng_kcapi_init); +module_exit(lrng_kcapi_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - kernel crypto API DRNG backend"); +#endif /* CONFIG_LRNG_DFLT_DRNG_KCAPI */ diff --git a/drivers/char/lrng/lrng_drng_kcapi.h b/drivers/char/lrng/lrng_drng_kcapi.h new file mode 100644 index 000000000000..5db25aaf830c --- /dev/null +++ b/drivers/char/lrng/lrng_drng_kcapi.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG kernel crypto API DRNG definition + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_KCAPI_DRNG_H +#define _LRNG_KCAPI_DRNG_H + +extern const struct lrng_drng_cb lrng_kcapi_drng_cb; + +#endif /* _LRNG_KCAPI_DRNG_H */ diff --git a/drivers/char/lrng/lrng_drng_mgr.c b/drivers/char/lrng/lrng_drng_mgr.c new file mode 100644 index 000000000000..2a4615113c64 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_mgr.c @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG DRNG management + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +#include "lrng_drng_atomic.h" +#include "lrng_drng_chacha20.h" +#include "lrng_drng_drbg.h" +#include "lrng_drng_kcapi.h" +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_interface_random_kernel.h" +#include "lrng_numa.h" +#include "lrng_sha.h" + +/* + * Maximum number of seconds between DRNG reseed intervals of the DRNG. Note, + * this is enforced with the next request of random numbers from the + * DRNG. Setting this value to zero implies a reseeding attempt before every + * generated random number. + */ +int lrng_drng_reseed_max_time = 600; + +/* + * Is LRNG for general-purpose use (i.e. is at least the lrng_drng_init + * fully allocated)? + */ +static atomic_t lrng_avail = ATOMIC_INIT(0); + +/* Guard protecting all crypto callback update operation of all DRNGs. */ +DEFINE_MUTEX(lrng_crypto_cb_update); + +/* + * Default hash callback that provides the crypto primitive right from the + * kernel start. It must not perform any memory allocation operation, but + * simply perform the hash calculation. + */ +const struct lrng_hash_cb *lrng_default_hash_cb = &lrng_sha_hash_cb; + +/* + * Default DRNG callback that provides the crypto primitive which is + * allocated either during late kernel boot stage. So, it is permissible for + * the callback to perform memory allocation operations. + */ +const struct lrng_drng_cb *lrng_default_drng_cb = +#if defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) + &lrng_cc20_drng_cb; +#elif defined(CONFIG_LRNG_DFLT_DRNG_DRBG) + &lrng_drbg_cb; +#elif defined(CONFIG_LRNG_DFLT_DRNG_KCAPI) + &lrng_kcapi_drng_cb; +#else +#error "Unknown default DRNG selected" +#endif + +/* DRNG for non-atomic use cases */ +static struct lrng_drng lrng_drng_init = { + LRNG_DRNG_STATE_INIT(lrng_drng_init, NULL, NULL, NULL, + &lrng_sha_hash_cb), + .lock = __MUTEX_INITIALIZER(lrng_drng_init.lock), +}; + +static u32 max_wo_reseed = LRNG_DRNG_MAX_WITHOUT_RESEED; +#ifdef CONFIG_LRNG_RUNTIME_MAX_WO_RESEED_CONFIG +module_param(max_wo_reseed, uint, 0444); +MODULE_PARM_DESC(max_wo_reseed, + "Maximum number of DRNG generate operation without full reseed\n"); +#endif + +/* Wait queue to wait until the LRNG is initialized - can freely be used */ +DECLARE_WAIT_QUEUE_HEAD(lrng_init_wait); + +/********************************** Helper ************************************/ + +bool lrng_get_available(void) +{ + return likely(atomic_read(&lrng_avail)); +} + +struct lrng_drng *lrng_drng_init_instance(void) +{ + return &lrng_drng_init; +} + +struct lrng_drng *lrng_drng_node_instance(void) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + int node = numa_node_id(); + + if (lrng_drng && lrng_drng[node]) + return lrng_drng[node]; + + return lrng_drng_init_instance(); +} + +void lrng_drng_reset(struct lrng_drng *drng) +{ + atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); + atomic_set(&drng->requests_since_fully_seeded, 0); + drng->last_seeded = jiffies; + drng->fully_seeded = false; + drng->force_reseed = true; + pr_debug("reset DRNG\n"); +} + +/* Initialize the DRNG, except the mutex lock */ +int lrng_drng_alloc_common(struct lrng_drng *drng, + const struct lrng_drng_cb *drng_cb) +{ + if (!drng || !drng_cb) + return -EINVAL; + + drng->drng_cb = drng_cb; + drng->drng = drng_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); + if (IS_ERR(drng->drng)) + return PTR_ERR(drng->drng); + + lrng_drng_reset(drng); + return 0; +} + +/* Initialize the default DRNG during boot and perform its seeding */ +int lrng_drng_initalize(void) +{ + int ret; + + if (lrng_get_available()) + return 0; + + /* Catch programming error */ + WARN_ON(lrng_drng_init.hash_cb != lrng_default_hash_cb); + + mutex_lock(&lrng_drng_init.lock); + if (lrng_get_available()) { + mutex_unlock(&lrng_drng_init.lock); + return 0; + } + + ret = lrng_drng_alloc_common(&lrng_drng_init, lrng_default_drng_cb); + mutex_unlock(&lrng_drng_init.lock); + if (ret) + return ret; + + pr_debug("LRNG for general use is available\n"); + atomic_set(&lrng_avail, 1); + + /* Seed the DRNG with any entropy available */ + if (!lrng_pool_trylock()) { + pr_info("Initial DRNG initialized triggering first seeding\n"); + lrng_drng_seed_work(NULL); + } else { + pr_info("Initial DRNG initialized without seeding\n"); + } + + return 0; +} + +static int __init lrng_drng_make_available(void) +{ + return lrng_drng_initalize(); +} +late_initcall(lrng_drng_make_available); + +bool lrng_sp80090c_compliant(void) +{ + /* SP800-90C compliant oversampling is only requested in FIPS mode */ + return fips_enabled; +} + +/************************* Random Number Generation ***************************/ + +/* Inject a data buffer into the DRNG - caller must hold its lock */ +void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, + bool fully_seeded, const char *drng_type) +{ + BUILD_BUG_ON(LRNG_DRNG_RESEED_THRESH > INT_MAX); + pr_debug("seeding %s DRNG with %u bytes\n", drng_type, inbuflen); + if (drng->drng_cb->drng_seed(drng->drng, inbuf, inbuflen) < 0) { + pr_warn("seeding of %s DRNG failed\n", drng_type); + drng->force_reseed = true; + } else { + int gc = LRNG_DRNG_RESEED_THRESH - atomic_read(&drng->requests); + + pr_debug("%s DRNG stats since last seeding: %lu secs; generate calls: %d\n", + drng_type, + (time_after(jiffies, drng->last_seeded) ? + (jiffies - drng->last_seeded) : 0) / HZ, gc); + + /* Count the numbers of generate ops since last fully seeded */ + if (fully_seeded) + atomic_set(&drng->requests_since_fully_seeded, 0); + else + atomic_add(gc, &drng->requests_since_fully_seeded); + + drng->last_seeded = jiffies; + atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); + drng->force_reseed = false; + + if (!drng->fully_seeded) { + drng->fully_seeded = fully_seeded; + if (drng->fully_seeded) + pr_debug("%s DRNG fully seeded\n", drng_type); + } + } +} + +/* Perform the seeding of the DRNG with data from entropy source */ +static void lrng_drng_seed_es(struct lrng_drng *drng) +{ + struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); + + lrng_fill_seed_buffer(&seedbuf, + lrng_get_seed_entropy_osr(drng->fully_seeded)); + + mutex_lock(&drng->lock); + lrng_drng_inject(drng, (u8 *)&seedbuf, sizeof(seedbuf), + lrng_fully_seeded_eb(drng->fully_seeded, &seedbuf), + "regular"); + mutex_unlock(&drng->lock); + + /* Set the seeding state of the LRNG */ + lrng_init_ops(&seedbuf); + + memzero_explicit(&seedbuf, sizeof(seedbuf)); +} + +static void lrng_drng_seed(struct lrng_drng *drng) +{ + BUILD_BUG_ON(LRNG_MIN_SEED_ENTROPY_BITS > + LRNG_DRNG_SECURITY_STRENGTH_BITS); + + if (lrng_get_available()) { + /* (Re-)Seed DRNG */ + lrng_drng_seed_es(drng); + /* (Re-)Seed atomic DRNG from regular DRNG */ + lrng_drng_atomic_seed_drng(drng); + } else { + /* + * If no-one is waiting for the DRNG, seed the atomic DRNG + * directly from the entropy sources. + */ + if (!wq_has_sleeper(&lrng_init_wait) && + !lrng_ready_list_has_sleeper()) + lrng_drng_atomic_seed_es(); + else + lrng_init_ops(NULL); + } +} + +static void _lrng_drng_seed_work(struct lrng_drng *drng, u32 node) +{ + pr_debug("reseed triggered by system events for DRNG on NUMA node %d\n", + node); + lrng_drng_seed(drng); + if (drng->fully_seeded) { + /* Prevent reseed storm */ + drng->last_seeded += node * 100 * HZ; + } +} + +/* + * DRNG reseed trigger: Kernel thread handler triggered by the schedule_work() + */ +void lrng_drng_seed_work(struct work_struct *dummy) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + u32 node; + + if (lrng_drng) { + for_each_online_node(node) { + struct lrng_drng *drng = lrng_drng[node]; + + if (drng && !drng->fully_seeded) { + _lrng_drng_seed_work(drng, node); + goto out; + } + } + } else { + if (!lrng_drng_init.fully_seeded) { + _lrng_drng_seed_work(&lrng_drng_init, 0); + goto out; + } + } + + lrng_pool_all_numa_nodes_seeded(true); + +out: + /* Allow the seeding operation to be called again */ + lrng_pool_unlock(); +} + +/* Force all DRNGs to reseed before next generation */ +void lrng_drng_force_reseed(void) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + u32 node; + + /* + * If the initial DRNG is over the reseed threshold, allow a forced + * reseed only for the initial DRNG as this is the fallback for all. It + * must be kept seeded before all others to keep the LRNG operational. + */ + if (!lrng_drng || + (atomic_read_u32(&lrng_drng_init.requests_since_fully_seeded) > + LRNG_DRNG_RESEED_THRESH)) { + lrng_drng_init.force_reseed = lrng_drng_init.fully_seeded; + pr_debug("force reseed of initial DRNG\n"); + return; + } + for_each_online_node(node) { + struct lrng_drng *drng = lrng_drng[node]; + + if (!drng) + continue; + + drng->force_reseed = drng->fully_seeded; + pr_debug("force reseed of DRNG on node %u\n", node); + } + lrng_drng_atomic_force_reseed(); +} +EXPORT_SYMBOL(lrng_drng_force_reseed); + +static bool lrng_drng_must_reseed(struct lrng_drng *drng) +{ + return (atomic_dec_and_test(&drng->requests) || + drng->force_reseed || + time_after(jiffies, + drng->last_seeded + lrng_drng_reseed_max_time * HZ)); +} + +/* + * lrng_drng_get() - Get random data out of the DRNG which is reseeded + * frequently. + * + * @drng: DRNG instance + * @outbuf: buffer for storing random data + * @outbuflen: length of outbuf + * + * Return: + * * < 0 in error case (DRNG generation or update failed) + * * >=0 returning the returned number of bytes + */ +int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen) +{ + u32 processed = 0; + + if (!outbuf || !outbuflen) + return 0; + + if (!lrng_get_available()) + return -EOPNOTSUPP; + + outbuflen = min_t(size_t, outbuflen, INT_MAX); + + /* If DRNG operated without proper reseed for too long, block LRNG */ + BUILD_BUG_ON(LRNG_DRNG_MAX_WITHOUT_RESEED < LRNG_DRNG_RESEED_THRESH); + if (atomic_read_u32(&drng->requests_since_fully_seeded) > max_wo_reseed) + lrng_unset_fully_seeded(drng); + + while (outbuflen) { + u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); + int ret; + + if (lrng_drng_must_reseed(drng)) { + if (lrng_pool_trylock()) { + drng->force_reseed = true; + } else { + lrng_drng_seed(drng); + lrng_pool_unlock(); + } + } + + mutex_lock(&drng->lock); + ret = drng->drng_cb->drng_generate(drng->drng, + outbuf + processed, todo); + mutex_unlock(&drng->lock); + if (ret <= 0) { + pr_warn("getting random data from DRNG failed (%d)\n", + ret); + return -EFAULT; + } + processed += ret; + outbuflen -= ret; + } + + return processed; +} + +int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + struct lrng_drng *drng = &lrng_drng_init; + int ret, node = numa_node_id(); + + might_sleep(); + + if (lrng_drng && lrng_drng[node] && lrng_drng[node]->fully_seeded) + drng = lrng_drng[node]; + + ret = lrng_drng_initalize(); + if (ret) + return ret; + + return lrng_drng_get(drng, outbuf, outbuflen); +} + +/* Reset LRNG such that all existing entropy is gone */ +static void _lrng_reset(struct work_struct *work) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + + if (!lrng_drng) { + mutex_lock(&lrng_drng_init.lock); + lrng_drng_reset(&lrng_drng_init); + mutex_unlock(&lrng_drng_init.lock); + } else { + u32 node; + + for_each_online_node(node) { + struct lrng_drng *drng = lrng_drng[node]; + + if (!drng) + continue; + mutex_lock(&drng->lock); + lrng_drng_reset(drng); + mutex_unlock(&drng->lock); + } + } + lrng_drng_atomic_reset(); + lrng_set_entropy_thresh(LRNG_INIT_ENTROPY_BITS); + + lrng_reset_state(); +} + +static DECLARE_WORK(lrng_reset_work, _lrng_reset); + +void lrng_reset(void) +{ + schedule_work(&lrng_reset_work); +} + +/******************* Generic LRNG kernel output interfaces ********************/ + +int lrng_drng_sleep_while_nonoperational(int nonblock) +{ + if (likely(lrng_state_operational())) + return 0; + if (nonblock) + return -EAGAIN; + return wait_event_interruptible(lrng_init_wait, + lrng_state_operational()); +} + +int lrng_drng_sleep_while_non_min_seeded(void) +{ + if (likely(lrng_state_min_seeded())) + return 0; + return wait_event_interruptible(lrng_init_wait, + lrng_state_min_seeded()); +} + +void lrng_get_random_bytes_full(void *buf, int nbytes) +{ + lrng_drng_sleep_while_nonoperational(0); + lrng_drng_get_sleep((u8 *)buf, (u32)nbytes); +} +EXPORT_SYMBOL(lrng_get_random_bytes_full); + +void lrng_get_random_bytes_min(void *buf, int nbytes) +{ + lrng_drng_sleep_while_non_min_seeded(); + lrng_drng_get_sleep((u8 *)buf, (u32)nbytes); +} +EXPORT_SYMBOL(lrng_get_random_bytes_min); diff --git a/drivers/char/lrng/lrng_drng_mgr.h b/drivers/char/lrng/lrng_drng_mgr.h new file mode 100644 index 000000000000..34ca5913dbd2 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_mgr.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DRNG_H +#define _LRNG_DRNG_H + +#include +#include +#include + +#include "lrng_definitions.h" + +extern struct wait_queue_head lrng_init_wait; +extern int lrng_drng_reseed_max_time; +extern struct mutex lrng_crypto_cb_update; +extern const struct lrng_drng_cb *lrng_default_drng_cb; +extern const struct lrng_hash_cb *lrng_default_hash_cb; + +/* DRNG state handle */ +struct lrng_drng { + void *drng; /* DRNG handle */ + void *hash; /* Hash handle */ + const struct lrng_drng_cb *drng_cb; /* DRNG callbacks */ + const struct lrng_hash_cb *hash_cb; /* Hash callbacks */ + atomic_t requests; /* Number of DRNG requests */ + atomic_t requests_since_fully_seeded; /* Number DRNG requests since + * last fully seeded + */ + unsigned long last_seeded; /* Last time it was seeded */ + bool fully_seeded; /* Is DRNG fully seeded? */ + bool force_reseed; /* Force a reseed */ + + rwlock_t hash_lock; /* Lock hash_cb replacement */ + /* Lock write operations on DRNG state, DRNG replacement of drng_cb */ + struct mutex lock; /* Non-atomic DRNG operation */ + spinlock_t spin_lock; /* Atomic DRNG operation */ +}; + +#define LRNG_DRNG_STATE_INIT(x, d, h, d_cb, h_cb) \ + .drng = d, \ + .hash = h, \ + .drng_cb = d_cb, \ + .hash_cb = h_cb, \ + .requests = ATOMIC_INIT(LRNG_DRNG_RESEED_THRESH),\ + .requests_since_fully_seeded = ATOMIC_INIT(0), \ + .last_seeded = 0, \ + .fully_seeded = false, \ + .force_reseed = true, \ + .hash_lock = __RW_LOCK_UNLOCKED(x.hash_lock) + +struct lrng_drng *lrng_drng_init_instance(void); +struct lrng_drng *lrng_drng_node_instance(void); + +void lrng_reset(void); +int lrng_drng_alloc_common(struct lrng_drng *drng, + const struct lrng_drng_cb *crypto_cb); +int lrng_drng_initalize(void); +bool lrng_sp80090c_compliant(void); +bool lrng_get_available(void); +void lrng_drng_reset(struct lrng_drng *drng); +void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, + bool fully_seeded, const char *drng_type); +int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen); +int lrng_drng_sleep_while_nonoperational(int nonblock); +int lrng_drng_sleep_while_non_min_seeded(void); +int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen); +void lrng_drng_seed_work(struct work_struct *dummy); +void lrng_drng_force_reseed(void); + +static inline u32 lrng_compress_osr(void) +{ + return lrng_sp80090c_compliant() ? LRNG_OVERSAMPLE_ES_BITS : 0; +} + +static inline u32 lrng_reduce_by_osr(u32 entropy_bits) +{ + u32 osr_bits = lrng_compress_osr(); + + return (entropy_bits >= osr_bits) ? (entropy_bits - osr_bits) : 0; +} + +#endif /* _LRNG_DRNG_H */ diff --git a/drivers/char/lrng/lrng_es_aux.c b/drivers/char/lrng/lrng_es_aux.c new file mode 100644 index 000000000000..245bc829998b --- /dev/null +++ b/drivers/char/lrng/lrng_es_aux.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Auxiliary entropy pool + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include + +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_sysctl.h" + +/* + * This is the auxiliary pool + * + * The aux pool array is aligned to 8 bytes to comfort the kernel crypto API + * cipher implementations of the hash functions used to read the pool: for some + * accelerated implementations, we need an alignment to avoid a realignment + * which involves memcpy(). The alignment to 8 bytes should satisfy all crypto + * implementations. + */ +struct lrng_pool { + u8 aux_pool[LRNG_POOL_SIZE]; /* Aux pool: digest state */ + atomic_t aux_entropy_bits; + atomic_t digestsize; /* Digest size of used hash */ + bool initialized; /* Aux pool initialized? */ + + /* Serialize read of entropy pool and update of aux pool */ + spinlock_t lock; +}; + +static struct lrng_pool lrng_pool __aligned(LRNG_KCAPI_ALIGN) = { + .aux_entropy_bits = ATOMIC_INIT(0), + .digestsize = ATOMIC_INIT(LRNG_ATOMIC_DIGEST_SIZE), + .initialized = false, + .lock = __SPIN_LOCK_UNLOCKED(lrng_pool.lock) +}; + +/********************************** Helper ***********************************/ + +/* Entropy in bits present in aux pool */ +static u32 lrng_aux_avail_entropy(u32 __unused) +{ + /* Cap available entropy with max entropy */ + u32 avail_bits = min_t(u32, lrng_get_digestsize(), + atomic_read_u32(&lrng_pool.aux_entropy_bits)); + + /* Consider oversampling rate due to aux pool conditioning */ + return lrng_reduce_by_osr(avail_bits); +} + +/* Set the digest size of the used hash in bytes */ +static void lrng_set_digestsize(u32 digestsize) +{ + struct lrng_pool *pool = &lrng_pool; + u32 ent_bits = atomic_xchg_relaxed(&pool->aux_entropy_bits, 0), + old_digestsize = lrng_get_digestsize(); + + atomic_set(&lrng_pool.digestsize, digestsize); + + /* + * Update the write wakeup threshold which must not be larger + * than the digest size of the current conditioning hash. + */ + digestsize = lrng_reduce_by_osr(digestsize << 3); + lrng_sysctl_update_max_write_thresh(digestsize); + lrng_write_wakeup_bits = digestsize; + + /* + * In case the new digest is larger than the old one, cap the available + * entropy to the old message digest used to process the existing data. + */ + ent_bits = min_t(u32, ent_bits, old_digestsize); + atomic_add(ent_bits, &pool->aux_entropy_bits); +} + +static int __init lrng_init_wakeup_bits(void) +{ + u32 digestsize = lrng_reduce_by_osr(lrng_get_digestsize()); + + lrng_sysctl_update_max_write_thresh(digestsize); + lrng_write_wakeup_bits = digestsize; + return 0; +} +core_initcall(lrng_init_wakeup_bits); + +/* Obtain the digest size provided by the used hash in bits */ +u32 lrng_get_digestsize(void) +{ + return atomic_read_u32(&lrng_pool.digestsize) << 3; +} + +/* Set entropy content in user-space controllable aux pool */ +void lrng_pool_set_entropy(u32 entropy_bits) +{ + atomic_set(&lrng_pool.aux_entropy_bits, entropy_bits); +} + +static void lrng_aux_reset(void) +{ + lrng_pool_set_entropy(0); +} + +/* + * Replace old with new hash for auxiliary pool handling + * + * Assumption: the caller must guarantee that the new_cb is available during the + * entire operation (e.g. it must hold the write lock against pointer updating). + */ +static int +lrng_aux_switch_hash(struct lrng_drng *drng, int __unused, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb) +{ + struct lrng_drng *init_drng = lrng_drng_init_instance(); + struct lrng_pool *pool = &lrng_pool; + struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; + u8 digest[LRNG_MAX_DIGESTSIZE]; + int ret; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) + return -EOPNOTSUPP; + + if (unlikely(!pool->initialized)) + return 0; + + /* We only switch if the processed DRNG is the initial DRNG. */ + if (init_drng != drng) + return 0; + + /* Get the aux pool hash with old digest ... */ + ret = old_cb->hash_final(shash, digest) ?: + /* ... re-initialize the hash with the new digest ... */ + new_cb->hash_init(shash, new_hash) ?: + /* + * ... feed the old hash into the new state. We may feed + * uninitialized memory into the new state, but this is + * considered no issue and even good as we have some more + * uncertainty here. + */ + new_cb->hash_update(shash, digest, sizeof(digest)); + if (!ret) { + lrng_set_digestsize(new_cb->hash_digestsize(new_hash)); + pr_debug("Re-initialize aux entropy pool with hash %s\n", + new_cb->hash_name()); + } + + memzero_explicit(digest, sizeof(digest)); + return ret; +} + +/* Insert data into auxiliary pool by using the hash update function. */ +static int +lrng_aux_pool_insert_locked(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) +{ + struct lrng_pool *pool = &lrng_pool; + struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; + struct lrng_drng *drng = lrng_drng_init_instance(); + const struct lrng_hash_cb *hash_cb; + unsigned long flags; + void *hash; + int ret; + + entropy_bits = min_t(u32, entropy_bits, inbuflen << 3); + + read_lock_irqsave(&drng->hash_lock, flags); + hash_cb = drng->hash_cb; + hash = drng->hash; + + if (unlikely(!pool->initialized)) { + ret = hash_cb->hash_init(shash, hash); + if (ret) + goto out; + pool->initialized = true; + } + + ret = hash_cb->hash_update(shash, inbuf, inbuflen); + if (ret) + goto out; + + /* + * Cap the available entropy to the hash output size compliant to + * SP800-90B section 3.1.5.1 table 1. + */ + entropy_bits += atomic_read_u32(&pool->aux_entropy_bits); + atomic_set(&pool->aux_entropy_bits, + min_t(u32, entropy_bits, + hash_cb->hash_digestsize(hash) << 3)); + +out: + read_unlock_irqrestore(&drng->hash_lock, flags); + return ret; +} + +int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) +{ + struct lrng_pool *pool = &lrng_pool; + unsigned long flags; + int ret; + + spin_lock_irqsave(&pool->lock, flags); + ret = lrng_aux_pool_insert_locked(inbuf, inbuflen, entropy_bits); + spin_unlock_irqrestore(&pool->lock, flags); + + lrng_es_add_entropy(); + + return ret; +} +EXPORT_SYMBOL(lrng_pool_insert_aux); + +/************************* Get data from entropy pool *************************/ + +/* + * Get auxiliary entropy pool and its entropy content for seed buffer. + * Caller must hold lrng_pool.pool->lock. + * @outbuf: buffer to store data in with size requested_bits + * @requested_bits: Requested amount of entropy + * @return: amount of entropy in outbuf in bits. + */ +static u32 lrng_aux_get_pool(u8 *outbuf, u32 requested_bits) +{ + struct lrng_pool *pool = &lrng_pool; + struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; + struct lrng_drng *drng = lrng_drng_init_instance(); + const struct lrng_hash_cb *hash_cb; + unsigned long flags; + void *hash; + u32 collected_ent_bits, returned_ent_bits, unused_bits = 0, + digestsize, digestsize_bits, requested_bits_osr; + u8 aux_output[LRNG_MAX_DIGESTSIZE]; + + if (unlikely(!pool->initialized)) + return 0; + + read_lock_irqsave(&drng->hash_lock, flags); + + hash_cb = drng->hash_cb; + hash = drng->hash; + digestsize = hash_cb->hash_digestsize(hash); + digestsize_bits = digestsize << 3; + + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(digestsize_bits, requested_bits); + + /* Ensure that no more than the size of aux_pool can be requested */ + requested_bits = min_t(u32, requested_bits, (LRNG_MAX_DIGESTSIZE << 3)); + requested_bits_osr = requested_bits + lrng_compress_osr(); + + /* Cap entropy with entropy counter from aux pool and the used digest */ + collected_ent_bits = min_t(u32, digestsize_bits, + atomic_xchg_relaxed(&pool->aux_entropy_bits, 0)); + + /* We collected too much entropy and put the overflow back */ + if (collected_ent_bits > requested_bits_osr) { + /* Amount of bits we collected too much */ + unused_bits = collected_ent_bits - requested_bits_osr; + /* Put entropy back */ + atomic_add(unused_bits, &pool->aux_entropy_bits); + /* Fix collected entropy */ + collected_ent_bits = requested_bits_osr; + } + + /* Apply oversampling: discount requested oversampling rate */ + returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); + + pr_debug("obtained %u bits by collecting %u bits of entropy from aux pool, %u bits of entropy remaining\n", + returned_ent_bits, collected_ent_bits, unused_bits); + + /* Get the digest for the aux pool to be returned to the caller ... */ + if (hash_cb->hash_final(shash, aux_output) || + /* + * ... and re-initialize the aux state. Do not add the aux pool + * digest for backward secrecy as it will be added with the + * insertion of the complete seed buffer after it has been filled. + */ + hash_cb->hash_init(shash, hash)) { + returned_ent_bits = 0; + } else { + /* + * Do not truncate the output size exactly to collected_ent_bits + * as the aux pool may contain data that is not credited with + * entropy, but we want to use them to stir the DRNG state. + */ + memcpy(outbuf, aux_output, requested_bits >> 3); + } + + read_unlock_irqrestore(&drng->hash_lock, flags); + memzero_explicit(aux_output, digestsize); + return returned_ent_bits; +} + +static void lrng_aux_get_backtrack(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + struct lrng_pool *pool = &lrng_pool; + unsigned long flags; + + /* Ensure aux pool extraction and backtracking op are atomic */ + spin_lock_irqsave(&pool->lock, flags); + + eb->e_bits[lrng_ext_es_aux] = lrng_aux_get_pool(eb->e[lrng_ext_es_aux], + requested_bits); + + /* Mix the extracted data back into pool for backtracking resistance */ + if (lrng_aux_pool_insert_locked((u8 *)eb, + sizeof(struct entropy_buf), 0)) + pr_warn("Backtracking resistance operation failed\n"); + + spin_unlock_irqrestore(&pool->lock, flags); +} + +static void lrng_aux_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for operating entropy pool: %s\n" + " Available entropy: %u\n", + lrng_drng_init->hash_cb->hash_name(), + lrng_aux_avail_entropy(0)); +} + +struct lrng_es_cb lrng_es_aux = { + .name = "Auxiliary", + .get_ent = lrng_aux_get_backtrack, + .curr_entropy = lrng_aux_avail_entropy, + .max_entropy = lrng_get_digestsize, + .state = lrng_aux_es_state, + .reset = lrng_aux_reset, + .switch_hash = lrng_aux_switch_hash, +}; diff --git a/drivers/char/lrng/lrng_es_aux.h b/drivers/char/lrng/lrng_es_aux.h new file mode 100644 index 000000000000..bc41e6474aad --- /dev/null +++ b/drivers/char/lrng/lrng_es_aux.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_AUX_H +#define _LRNG_ES_AUX_H + +#include "lrng_drng_mgr.h" +#include "lrng_es_mgr_cb.h" + +u32 lrng_get_digestsize(void); +void lrng_pool_set_entropy(u32 entropy_bits); +int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits); + +extern struct lrng_es_cb lrng_es_aux; + +/****************************** Helper code ***********************************/ + +/* Obtain the security strength of the LRNG in bits */ +static inline u32 lrng_security_strength(void) +{ + /* + * We use a hash to read the entropy in the entropy pool. According to + * SP800-90B table 1, the entropy can be at most the digest size. + * Considering this together with the last sentence in section 3.1.5.1.2 + * the security strength of a (approved) hash is equal to its output + * size. On the other hand the entropy cannot be larger than the + * security strength of the used DRBG. + */ + return min_t(u32, LRNG_FULL_SEED_ENTROPY_BITS, lrng_get_digestsize()); +} + +static inline u32 lrng_get_seed_entropy_osr(bool fully_seeded) +{ + u32 requested_bits = lrng_security_strength(); + + /* Apply oversampling during initialization according to SP800-90C */ + if (lrng_sp80090c_compliant() && !fully_seeded) + requested_bits += LRNG_SEED_BUFFER_INIT_ADD_BITS; + return requested_bits; +} + +#endif /* _LRNG_ES_AUX_H */ diff --git a/drivers/char/lrng/lrng_es_cpu.c b/drivers/char/lrng/lrng_es_cpu.c new file mode 100644 index 000000000000..3910108cfd08 --- /dev/null +++ b/drivers/char/lrng/lrng_es_cpu.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Fast Entropy Source: CPU-based entropy source + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_definitions.h" +#include "lrng_es_aux.h" +#include "lrng_es_cpu.h" + +/* + * Estimated entropy of data is a 32th of LRNG_DRNG_SECURITY_STRENGTH_BITS. + * As we have no ability to review the implementation of those noise sources, + * it is prudent to have a conservative estimate here. + */ +#define LRNG_ARCHRANDOM_DEFAULT_STRENGTH CONFIG_LRNG_CPU_ENTROPY_RATE +#define LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH LRNG_DRNG_SECURITY_STRENGTH_BITS +#ifdef CONFIG_RANDOM_TRUST_CPU +static u32 cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; +#else +static u32 cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; +#endif +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(cpu_entropy, uint, 0644); +MODULE_PARM_DESC(cpu_entropy, "Entropy in bits of 256 data bits from CPU noise source (e.g. RDSEED)"); +#endif + +static int __init lrng_parse_trust_cpu(char *arg) +{ + int ret; + bool trust_cpu = false; + + ret = kstrtobool(arg, &trust_cpu); + if (ret) + return ret; + + if (trust_cpu) { + cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; + lrng_es_add_entropy(); + } else { + cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; + } + + return 0; +} +early_param("random.trust_cpu", lrng_parse_trust_cpu); + +static u32 lrng_cpu_entropylevel(u32 requested_bits) +{ + return lrng_fast_noise_entropylevel(cpu_entropy, requested_bits); +} + +static u32 lrng_cpu_poolsize(void) +{ + return lrng_cpu_entropylevel(lrng_security_strength()); +} + +static u32 lrng_get_cpu_data(u8 *outbuf, u32 requested_bits) +{ + u32 i; + + /* operate on full blocks */ + BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BYTES % sizeof(unsigned long)); + BUILD_BUG_ON(LRNG_SEED_BUFFER_INIT_ADD_BITS % sizeof(unsigned long)); + /* ensure we have aligned buffers */ + BUILD_BUG_ON(LRNG_KCAPI_ALIGN % sizeof(unsigned long)); + + for (i = 0; i < (requested_bits >> 3); + i += sizeof(unsigned long)) { + if (!arch_get_random_seed_long((unsigned long *)(outbuf + i)) && + !arch_get_random_long((unsigned long *)(outbuf + i))) { + cpu_entropy = 0; + return 0; + } + } + + return requested_bits; +} + +static u32 lrng_get_cpu_data_compress(u8 *outbuf, u32 requested_bits, + u32 data_multiplier) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb; + struct lrng_drng *drng = lrng_drng_node_instance(); + unsigned long flags; + u32 ent_bits = 0, i, partial_bits = 0, digestsize, digestsize_bits, + full_bits; + void *hash; + + read_lock_irqsave(&drng->hash_lock, flags); + hash_cb = drng->hash_cb; + hash = drng->hash; + + digestsize = hash_cb->hash_digestsize(hash); + digestsize_bits = digestsize << 3; + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(digestsize_bits, requested_bits); + full_bits = requested_bits * data_multiplier; + + /* Calculate oversampling for SP800-90C */ + if (lrng_sp80090c_compliant()) { + /* Complete amount of bits to be pulled */ + full_bits += LRNG_OVERSAMPLE_ES_BITS * data_multiplier; + /* Full blocks that will be pulled */ + data_multiplier = full_bits / requested_bits; + /* Partial block in bits to be pulled */ + partial_bits = full_bits - (data_multiplier * requested_bits); + } + + if (hash_cb->hash_init(shash, hash)) + goto out; + + /* Hash all data from the CPU entropy source */ + for (i = 0; i < data_multiplier; i++) { + ent_bits = lrng_get_cpu_data(outbuf, requested_bits); + if (!ent_bits) + goto out; + + if (hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) + goto err; + } + + /* Hash partial block, if applicable */ + ent_bits = lrng_get_cpu_data(outbuf, partial_bits); + if (ent_bits && + hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) + goto err; + + pr_debug("pulled %u bits from CPU RNG entropy source\n", full_bits); + ent_bits = requested_bits; + + /* Generate the compressed data to be returned to the caller */ + if (requested_bits < digestsize_bits) { + u8 digest[LRNG_MAX_DIGESTSIZE]; + + if (hash_cb->hash_final(shash, digest)) + goto err; + + /* Truncate output data to requested size */ + memcpy(outbuf, digest, requested_bits >> 3); + memzero_explicit(digest, digestsize); + } else { + if (hash_cb->hash_final(shash, outbuf)) + goto err; + } + +out: + hash_cb->hash_desc_zero(shash); + read_unlock_irqrestore(&drng->hash_lock, flags); + return ent_bits; + +err: + ent_bits = 0; + goto out; +} + +/* + * If CPU entropy source requires does not return full entropy, return the + * multiplier of how much data shall be sampled from it. + */ +static u32 lrng_cpu_multiplier(void) +{ + static u32 data_multiplier = 0; + unsigned long v; + + if (data_multiplier > 0) + return data_multiplier; + + if (IS_ENABLED(CONFIG_X86) && !arch_get_random_seed_long(&v)) { + /* + * Intel SPEC: pulling 512 blocks from RDRAND ensures + * one reseed making it logically equivalent to RDSEED. + */ + data_multiplier = 512; + } else if (IS_ENABLED(CONFIG_PPC)) { + /* + * PowerISA defines DARN to deliver at least 0.5 bits of + * entropy per data bit. + */ + data_multiplier = 2; + } else if (IS_ENABLED(CONFIG_RISCV)) { + /* + * riscv-crypto-spec-scalar-1.0.0-rc6.pdf section 4.2 defines + * this requirement. + */ + data_multiplier = 2; + } else { + /* CPU provides full entropy */ + data_multiplier = CONFIG_LRNG_CPU_FULL_ENT_MULTIPLIER; + } + return data_multiplier; +} + +static int +lrng_cpu_switch_hash(struct lrng_drng *drng, int node, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb) +{ + u32 digestsize, multiplier; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) + return -EOPNOTSUPP; + + digestsize = lrng_get_digestsize(); + multiplier = lrng_cpu_multiplier(); + + /* + * It would be security violation if the new digestsize is smaller than + * the set CPU entropy rate. + */ + WARN_ON(multiplier > 1 && digestsize < cpu_entropy); + cpu_entropy = min_t(u32, digestsize, cpu_entropy); + return 0; +} + +/* + * lrng_get_arch() - Get CPU entropy source entropy + * + * @eb: entropy buffer to store entropy + * @requested_bits: requested entropy in bits + */ +static void lrng_cpu_get(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + u32 ent_bits, data_multiplier = lrng_cpu_multiplier(); + + if (data_multiplier <= 1) { + ent_bits = lrng_get_cpu_data(eb->e[lrng_ext_es_cpu], + requested_bits); + } else { + ent_bits = lrng_get_cpu_data_compress(eb->e[lrng_ext_es_cpu], + requested_bits, + data_multiplier); + } + + ent_bits = lrng_cpu_entropylevel(ent_bits); + pr_debug("obtained %u bits of entropy from CPU RNG entropy source\n", + ent_bits); + eb->e_bits[lrng_ext_es_cpu] = ent_bits; +} + +static void lrng_cpu_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + u32 data_multiplier = lrng_cpu_multiplier(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for compressing data: %s\n" + " Available entropy: %u\n" + " Data multiplier: %u\n", + (data_multiplier <= 1) ? + "N/A" : lrng_drng_init->hash_cb->hash_name(), + lrng_cpu_poolsize(), + data_multiplier); +} + +struct lrng_es_cb lrng_es_cpu = { + .name = "CPU", + .get_ent = lrng_cpu_get, + .curr_entropy = lrng_cpu_entropylevel, + .max_entropy = lrng_cpu_poolsize, + .state = lrng_cpu_es_state, + .reset = NULL, + .switch_hash = lrng_cpu_switch_hash, +}; diff --git a/drivers/char/lrng/lrng_es_cpu.h b/drivers/char/lrng/lrng_es_cpu.h new file mode 100644 index 000000000000..8dbb4d9a2926 --- /dev/null +++ b/drivers/char/lrng/lrng_es_cpu.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_CPU_H +#define _LRNG_ES_CPU_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_CPU + +extern struct lrng_es_cb lrng_es_cpu; + +#endif /* CONFIG_LRNG_CPU */ + +#endif /* _LRNG_ES_CPU_H */ diff --git a/drivers/char/lrng/lrng_es_irq.c b/drivers/char/lrng/lrng_es_irq.c new file mode 100644 index 000000000000..7f36fb363d79 --- /dev/null +++ b/drivers/char/lrng/lrng_es_irq.c @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Interrupt data collection + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lrng_es_aux.h" +#include "lrng_es_irq.h" +#include "lrng_es_timer_common.h" +#include "lrng_health.h" +#include "lrng_numa.h" +#include "lrng_testing.h" + +/* + * Number of interrupts to be recorded to assume that DRNG security strength + * bits of entropy are received. + * Note: a value below the DRNG security strength should not be defined as this + * may imply the DRNG can never be fully seeded in case other noise + * sources are unavailable. + */ +#define LRNG_IRQ_ENTROPY_BITS LRNG_UINT32_C(CONFIG_LRNG_IRQ_ENTROPY_RATE) + + +/* Number of interrupts required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ +static u32 lrng_irq_entropy_bits = LRNG_IRQ_ENTROPY_BITS; + +static u32 irq_entropy __read_mostly = LRNG_IRQ_ENTROPY_BITS; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(irq_entropy, uint, 0444); +MODULE_PARM_DESC(irq_entropy, + "How many interrupts must be collected for obtaining 256 bits of entropy\n"); +#endif + +/* Per-CPU array holding concatenated IRQ entropy events */ +static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_irq_array) + __aligned(LRNG_KCAPI_ALIGN); +static DEFINE_PER_CPU(u32, lrng_irq_array_ptr) = 0; +static DEFINE_PER_CPU(atomic_t, lrng_irq_array_irqs) = ATOMIC_INIT(0); + +/* + * The entropy collection is performed by executing the following steps: + * 1. fill up the per-CPU array holding the time stamps + * 2. once the per-CPU array is full, a compression of the data into + * the entropy pool is performed - this happens in interrupt context + * + * If step 2 is not desired in interrupt context, the following boolean + * needs to be set to false. This implies that old entropy data in the + * per-CPU array collected since the last DRNG reseed is overwritten with + * new entropy data instead of retaining the entropy with the compression + * operation. + * + * Impact on entropy: + * + * If continuous compression is enabled, the maximum entropy that is collected + * per CPU between DRNG reseeds is equal to the digest size of the used hash. + * + * If continuous compression is disabled, the maximum number of entropy events + * that can be collected per CPU is equal to LRNG_DATA_ARRAY_SIZE. This amount + * of events is converted into an entropy statement which then represents the + * maximum amount of entropy collectible per CPU between DRNG reseeds. + */ +static bool lrng_irq_continuous_compression __read_mostly = + IS_ENABLED(CONFIG_LRNG_ENABLE_CONTINUOUS_COMPRESSION); + +#ifdef CONFIG_LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION +module_param(lrng_irq_continuous_compression, bool, 0444); +MODULE_PARM_DESC(lrng_irq_continuous_compression, + "Perform entropy compression if per-CPU entropy data array is full\n"); +#endif + +/* + * Per-CPU entropy pool with compressed entropy event + * + * The per-CPU entropy pool is defined as the hash state. New data is simply + * inserted into the entropy pool by performing a hash update operation. + * To read the entropy pool, a hash final must be invoked. However, before + * the entropy pool is released again after a hash final, the hash init must + * be performed. + */ +static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_irq_pool) + __aligned(LRNG_KCAPI_ALIGN); +/* + * Lock to allow other CPUs to read the pool - as this is only done during + * reseed which is infrequent, this lock is hardly contended. + */ +static DEFINE_PER_CPU(spinlock_t, lrng_irq_lock); +static DEFINE_PER_CPU(bool, lrng_irq_lock_init) = false; + +static bool lrng_irq_pool_online(int cpu) +{ + return per_cpu(lrng_irq_lock_init, cpu); +} + +static void __init lrng_irq_check_compression_state(void) +{ + /* One pool must hold sufficient entropy for disabled compression */ + if (!lrng_irq_continuous_compression) { + u32 max_ent = min_t(u32, lrng_get_digestsize(), + lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, + lrng_irq_entropy_bits)); + if (max_ent < lrng_security_strength()) { + pr_warn("Force continuous compression operation to ensure LRNG can hold enough entropy\n"); + lrng_irq_continuous_compression = true; + } + } +} + +void __init lrng_irq_es_init(bool highres_timer) +{ + /* Set a minimum number of interrupts that must be collected */ + irq_entropy = max_t(u32, LRNG_IRQ_ENTROPY_BITS, irq_entropy); + + if (highres_timer) { + lrng_irq_entropy_bits = irq_entropy; + } else { + u32 new_entropy = irq_entropy * LRNG_ES_OVERSAMPLING_FACTOR; + + lrng_irq_entropy_bits = (irq_entropy < new_entropy) ? + new_entropy : irq_entropy; + pr_warn("operating without high-resolution timer and applying IRQ oversampling factor %u\n", + LRNG_ES_OVERSAMPLING_FACTOR); + } + + lrng_irq_check_compression_state(); +} + +/* + * Reset all per-CPU pools - reset entropy estimator but leave the pool data + * that may or may not have entropy unchanged. + */ +static void lrng_irq_reset(void) +{ + int cpu; + + /* Trigger GCD calculation anew. */ + lrng_gcd_set(0); + + for_each_online_cpu(cpu) + atomic_set(per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); +} + +static u32 lrng_irq_avail_pool_size(void) +{ + u32 max_size = 0, max_pool = lrng_get_digestsize(); + int cpu; + + if (!lrng_irq_continuous_compression) + max_pool = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); + + for_each_online_cpu(cpu) { + if (lrng_irq_pool_online(cpu)) + max_size += max_pool; + } + + return max_size; +} + +/* Return entropy of unused IRQs present in all per-CPU pools. */ +static u32 lrng_irq_avail_entropy(u32 __unused) +{ + u32 digestsize_irqs, irq = 0; + int cpu; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) + return 0; + + /* Obtain the cap of maximum numbers of IRQs we count */ + digestsize_irqs = lrng_entropy_to_data(lrng_get_digestsize(), + lrng_irq_entropy_bits); + if (!lrng_irq_continuous_compression) { + /* Cap to max. number of IRQs the array can hold */ + digestsize_irqs = min_t(u32, digestsize_irqs, + LRNG_DATA_NUM_VALUES); + } + + for_each_online_cpu(cpu) { + if (!lrng_irq_pool_online(cpu)) + continue; + irq += min_t(u32, digestsize_irqs, + atomic_read_u32(per_cpu_ptr(&lrng_irq_array_irqs, + cpu))); + } + + /* Consider oversampling rate */ + return lrng_reduce_by_osr(lrng_data_to_entropy(irq, + lrng_irq_entropy_bits)); +} + +/* + * Trigger a switch of the hash implementation for the per-CPU pool. + * + * For each per-CPU pool, obtain the message digest with the old hash + * implementation, initialize the per-CPU pool again with the new hash + * implementation and inject the message digest into the new state. + * + * Assumption: the caller must guarantee that the new_cb is available during the + * entire operation (e.g. it must hold the lock against pointer updating). + */ +static int +lrng_irq_switch_hash(struct lrng_drng *drng, int node, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb) +{ + u8 digest[LRNG_MAX_DIGESTSIZE]; + u32 digestsize_irqs, found_irqs; + int ret = 0, cpu; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) + return -EOPNOTSUPP; + + for_each_online_cpu(cpu) { + struct shash_desc *pcpu_shash; + + /* + * Only switch the per-CPU pools for the current node because + * the hash_cb only applies NUMA-node-wide. + */ + if (cpu_to_node(cpu) != node || !lrng_irq_pool_online(cpu)) + continue; + + pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, + cpu); + + digestsize_irqs = old_cb->hash_digestsize(pcpu_shash); + digestsize_irqs = lrng_entropy_to_data(digestsize_irqs << 3, + lrng_irq_entropy_bits); + + if (pcpu_shash->tfm == new_hash) + continue; + + /* Get the per-CPU pool hash with old digest ... */ + ret = old_cb->hash_final(pcpu_shash, digest) ?: + /* ... re-initialize the hash with the new digest ... */ + new_cb->hash_init(pcpu_shash, new_hash) ?: + /* + * ... feed the old hash into the new state. We may feed + * uninitialized memory into the new state, but this is + * considered no issue and even good as we have some more + * uncertainty here. + */ + new_cb->hash_update(pcpu_shash, digest, + sizeof(digest)); + if (ret) + goto out; + + /* + * In case the new digest is larger than the old one, cap + * the available entropy to the old message digest used to + * process the existing data. + */ + found_irqs = atomic_xchg_relaxed( + per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); + found_irqs = min_t(u32, found_irqs, digestsize_irqs); + atomic_add_return_relaxed(found_irqs, + per_cpu_ptr(&lrng_irq_array_irqs, cpu)); + + pr_debug("Re-initialize per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n", + cpu, node, new_cb->hash_name()); + } + +out: + memzero_explicit(digest, sizeof(digest)); + return ret; +} + +/* + * When reading the per-CPU message digest, make sure we use the crypto + * callbacks defined for the NUMA node the per-CPU pool is defined for because + * the LRNG crypto switch support is only atomic per NUMA node. + */ +static u32 +lrng_irq_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, + void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) +{ + struct shash_desc *pcpu_shash = + (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, cpu); + spinlock_t *lock = per_cpu_ptr(&lrng_irq_lock, cpu); + unsigned long flags; + u32 digestsize_irqs, found_irqs; + + /* Lock guarding against reading / writing to per-CPU pool */ + spin_lock_irqsave(lock, flags); + + *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); + digestsize_irqs = lrng_entropy_to_data(*digestsize << 3, + lrng_irq_entropy_bits); + + /* Obtain entropy statement like for the entropy pool */ + found_irqs = atomic_xchg_relaxed( + per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); + /* Cap to maximum amount of data we can hold in hash */ + found_irqs = min_t(u32, found_irqs, digestsize_irqs); + + /* Cap to maximum amount of data we can hold in array */ + if (!lrng_irq_continuous_compression) + found_irqs = min_t(u32, found_irqs, LRNG_DATA_NUM_VALUES); + + /* Store all not-yet compressed data in data array into hash, ... */ + if (pcpu_hash_cb->hash_update(pcpu_shash, + (u8 *)per_cpu_ptr(lrng_irq_array, cpu), + LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: + /* ... get the per-CPU pool digest, ... */ + pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: + /* ... re-initialize the hash, ... */ + pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: + /* ... feed the old hash into the new state. */ + pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) + found_irqs = 0; + + spin_unlock_irqrestore(lock, flags); + return found_irqs; +} + +/* + * Hash all per-CPU pools and return the digest to be used as seed data for + * seeding a DRNG. The caller must guarantee backtracking resistance. + * The function will only copy as much data as entropy is available into the + * caller-provided output buffer. + * + * This function handles the translation from the number of received interrupts + * into an entropy statement. The conversion depends on LRNG_IRQ_ENTROPY_BITS + * which defines how many interrupts must be received to obtain 256 bits of + * entropy. With this value, the function lrng_data_to_entropy converts a given + * data size (received interrupts, requested amount of data, etc.) into an + * entropy statement. lrng_entropy_to_data does the reverse. + * + * @eb: entropy buffer to store entropy + * @requested_bits: Requested amount of entropy + * @fully_seeded: indicator whether LRNG is fully seeded + */ +static void lrng_irq_pool_hash(struct entropy_buf *eb, u32 requested_bits, + bool fully_seeded) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb; + struct lrng_drng **lrng_drng = lrng_drng_instances(); + struct lrng_drng *drng = lrng_drng_init_instance(); + u8 digest[LRNG_MAX_DIGESTSIZE]; + unsigned long flags, flags2; + u32 found_irqs, collected_irqs = 0, collected_ent_bits, requested_irqs, + returned_ent_bits; + int ret, cpu; + void *hash; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) { + eb->e_bits[lrng_int_es_irq] = 0; + return; + } + + /* Lock guarding replacement of per-NUMA hash */ + read_lock_irqsave(&drng->hash_lock, flags); + + hash_cb = drng->hash_cb; + hash = drng->hash; + + /* The hash state of filled with all per-CPU pool hashes. */ + ret = hash_cb->hash_init(shash, hash); + if (ret) + goto err; + + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); + requested_irqs = lrng_entropy_to_data(requested_bits + + lrng_compress_osr(), + lrng_irq_entropy_bits); + + /* + * Harvest entropy from each per-CPU hash state - even though we may + * have collected sufficient entropy, we will hash all per-CPU pools. + */ + for_each_online_cpu(cpu) { + struct lrng_drng *pcpu_drng = drng; + u32 digestsize, pcpu_unused_irqs = 0; + int node = cpu_to_node(cpu); + + /* If pool is not online, then no entropy is present. */ + if (!lrng_irq_pool_online(cpu)) + continue; + + if (lrng_drng && lrng_drng[node]) + pcpu_drng = lrng_drng[node]; + + if (pcpu_drng == drng) { + found_irqs = lrng_irq_pool_hash_one(hash_cb, hash, + cpu, digest, + &digestsize); + } else { + read_lock_irqsave(&pcpu_drng->hash_lock, flags2); + found_irqs = + lrng_irq_pool_hash_one(pcpu_drng->hash_cb, + pcpu_drng->hash, cpu, + digest, &digestsize); + read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); + } + + /* Inject the digest into the state of all per-CPU pools */ + ret = hash_cb->hash_update(shash, digest, digestsize); + if (ret) + goto err; + + collected_irqs += found_irqs; + if (collected_irqs > requested_irqs) { + pcpu_unused_irqs = collected_irqs - requested_irqs; + atomic_add_return_relaxed(pcpu_unused_irqs, + per_cpu_ptr(&lrng_irq_array_irqs, cpu)); + collected_irqs = requested_irqs; + } + pr_debug("%u interrupts used from entropy pool of CPU %d, %u interrupts remain unused\n", + found_irqs - pcpu_unused_irqs, cpu, pcpu_unused_irqs); + } + + ret = hash_cb->hash_final(shash, digest); + if (ret) + goto err; + + collected_ent_bits = lrng_data_to_entropy(collected_irqs, + lrng_irq_entropy_bits); + /* Apply oversampling: discount requested oversampling rate */ + returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); + + pr_debug("obtained %u bits by collecting %u bits of entropy from entropy pool noise source\n", + returned_ent_bits, collected_ent_bits); + + /* + * Truncate to available entropy as implicitly allowed by SP800-90B + * section 3.1.5.1.1 table 1 which awards truncated hashes full + * entropy. + * + * During boot time, we read requested_bits data with + * returned_ent_bits entropy. In case our conservative entropy + * estimate underestimates the available entropy we can transport as + * much available entropy as possible. + */ + memcpy(eb->e[lrng_int_es_irq], digest, + fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); + eb->e_bits[lrng_int_es_irq] = returned_ent_bits; + +out: + hash_cb->hash_desc_zero(shash); + read_unlock_irqrestore(&drng->hash_lock, flags); + memzero_explicit(digest, sizeof(digest)); + return; + +err: + eb->e_bits[lrng_int_es_irq] = 0; + goto out; +} + +/* Compress the lrng_irq_array array into lrng_irq_pool */ +static void lrng_irq_array_compress(void) +{ + struct shash_desc *shash = + (struct shash_desc *)this_cpu_ptr(lrng_irq_pool); + struct lrng_drng *drng = lrng_drng_node_instance(); + const struct lrng_hash_cb *hash_cb; + spinlock_t *lock = this_cpu_ptr(&lrng_irq_lock); + unsigned long flags, flags2; + void *hash; + bool init = false; + + read_lock_irqsave(&drng->hash_lock, flags); + hash_cb = drng->hash_cb; + hash = drng->hash; + + if (unlikely(!this_cpu_read(lrng_irq_lock_init))) { + init = true; + spin_lock_init(lock); + this_cpu_write(lrng_irq_lock_init, true); + pr_debug("Initializing per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n", + raw_smp_processor_id(), numa_node_id(), + hash_cb->hash_name()); + } + + spin_lock_irqsave(lock, flags2); + + if (unlikely(init) && hash_cb->hash_init(shash, hash)) { + this_cpu_write(lrng_irq_lock_init, false); + pr_warn("Initialization of hash failed\n"); + } else if (lrng_irq_continuous_compression) { + /* Add entire per-CPU data array content into entropy pool. */ + if (hash_cb->hash_update(shash, + (u8 *)this_cpu_ptr(lrng_irq_array), + LRNG_DATA_ARRAY_SIZE * sizeof(u32))) + pr_warn_ratelimited("Hashing of entropy data failed\n"); + } + + spin_unlock_irqrestore(lock, flags2); + read_unlock_irqrestore(&drng->hash_lock, flags); +} + +/* Compress data array into hash */ +static void lrng_irq_array_to_hash(u32 ptr) +{ + u32 *array = this_cpu_ptr(lrng_irq_array); + + /* + * During boot time the hash operation is triggered more often than + * during regular operation. + */ + if (unlikely(!lrng_state_fully_seeded())) { + if ((ptr & 31) && (ptr < LRNG_DATA_WORD_MASK)) + return; + } else if (ptr < LRNG_DATA_WORD_MASK) { + return; + } + + if (lrng_raw_array_entropy_store(*array)) { + u32 i; + + /* + * If we fed even a part of the array to external analysis, we + * mark that the entire array and the per-CPU pool to have no + * entropy. This is due to the non-IID property of the data as + * we do not fully know whether the existing dependencies + * diminish the entropy beyond to what we expect it has. + */ + atomic_set(this_cpu_ptr(&lrng_irq_array_irqs), 0); + + for (i = 1; i < LRNG_DATA_ARRAY_SIZE; i++) + lrng_raw_array_entropy_store(*(array + i)); + } else { + lrng_irq_array_compress(); + /* Ping pool handler about received entropy */ + if (lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) + lrng_es_add_entropy(); + } +} + +/* + * Concatenate full 32 bit word at the end of time array even when current + * ptr is not aligned to sizeof(data). + */ +static void _lrng_irq_array_add_u32(u32 data) +{ + /* Increment pointer by number of slots taken for input value */ + u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_irq_array_ptr, + LRNG_DATA_SLOTS_PER_UINT); + unsigned int pre_array; + + /* + * This function injects a unit into the array - guarantee that + * array unit size is equal to data type of input data. + */ + BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != (sizeof(data) << 3)); + + /* + * The following logic requires at least two units holding + * the data as otherwise the pointer would immediately wrap when + * injection an u32 word. + */ + BUILD_BUG_ON(LRNG_DATA_NUM_VALUES <= LRNG_DATA_SLOTS_PER_UINT); + + lrng_data_split_u32(&ptr, &pre_ptr, &mask); + + /* MSB of data go into previous unit */ + pre_array = lrng_data_idx2array(pre_ptr); + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_irq_array[pre_array], ~(0xffffffff & ~mask)); + this_cpu_or(lrng_irq_array[pre_array], data & ~mask); + + /* Invoke compression as we just filled data array completely */ + if (unlikely(pre_ptr > ptr)) + lrng_irq_array_to_hash(LRNG_DATA_WORD_MASK); + + /* LSB of data go into current unit */ + this_cpu_write(lrng_irq_array[lrng_data_idx2array(ptr)], + data & mask); + + if (likely(pre_ptr <= ptr)) + lrng_irq_array_to_hash(ptr); +} + +/* Concatenate a 32-bit word at the end of the per-CPU array */ +void lrng_irq_array_add_u32(u32 data) +{ + /* + * Disregard entropy-less data without continuous compression to + * avoid it overwriting data with entropy when array ptr wraps. + */ + if (lrng_irq_continuous_compression) + _lrng_irq_array_add_u32(data); +} + +/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ +static void lrng_irq_array_add_slot(u32 data) +{ + /* Get slot */ + u32 ptr = this_cpu_inc_return(lrng_irq_array_ptr) & + LRNG_DATA_WORD_MASK; + unsigned int array = lrng_data_idx2array(ptr); + unsigned int slot = lrng_data_idx2slot(ptr); + + BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS % LRNG_DATA_SLOTSIZE_BITS); + /* Ensure consistency of values */ + BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != + sizeof(lrng_irq_array[0]) << 3); + + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_irq_array[array], + ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, + slot))); + /* Store data into slot */ + this_cpu_or(lrng_irq_array[array], lrng_data_slot_val(data, slot)); + + lrng_irq_array_to_hash(ptr); +} + +static void +lrng_time_process_common(u32 time, void(*add_time)(u32 data)) +{ + enum lrng_health_res health_test; + + if (lrng_raw_hires_entropy_store(time)) + return; + + health_test = lrng_health_test(time, lrng_int_es_irq); + if (health_test > lrng_health_fail_use) + return; + + if (health_test == lrng_health_pass) + atomic_inc_return(this_cpu_ptr(&lrng_irq_array_irqs)); + + add_time(time); +} + +/* + * Batching up of entropy in per-CPU array before injecting into entropy pool. + */ +static void lrng_time_process(void) +{ + u32 now_time = random_get_entropy(); + + if (unlikely(!lrng_gcd_tested())) { + /* When GCD is unknown, we process the full time stamp */ + lrng_time_process_common(now_time, _lrng_irq_array_add_u32); + lrng_gcd_add_value(now_time); + } else { + /* GCD is known and applied */ + lrng_time_process_common((now_time / lrng_gcd_get()) & + LRNG_DATA_SLOTSIZE_MASK, + lrng_irq_array_add_slot); + } + + lrng_perf_time(now_time); +} + +/* Hot code path - Callback for interrupt handler */ +void add_interrupt_randomness(int irq, int irq_flg) +{ + if (lrng_highres_timer()) { + lrng_time_process(); + } else { + struct pt_regs *regs = get_irq_regs(); + static atomic_t reg_idx = ATOMIC_INIT(0); + u64 ip; + u32 tmp; + + if (regs) { + u32 *ptr = (u32 *)regs; + int reg_ptr = atomic_add_return_relaxed(1, ®_idx); + size_t n = (sizeof(struct pt_regs) / sizeof(u32)); + + ip = instruction_pointer(regs); + tmp = *(ptr + (reg_ptr % n)); + tmp = lrng_raw_regs_entropy_store(tmp) ? 0 : tmp; + _lrng_irq_array_add_u32(tmp); + } else { + ip = _RET_IP_; + } + + lrng_time_process(); + + /* + * The XOR operation combining the different values is not + * considered to destroy entropy since the entirety of all + * processed values delivers the entropy (and not each + * value separately of the other values). + */ + tmp = lrng_raw_jiffies_entropy_store(jiffies) ? 0 : jiffies; + tmp ^= lrng_raw_irq_entropy_store(irq) ? 0 : irq; + tmp ^= lrng_raw_retip_entropy_store(ip) ? 0 : ip; + tmp ^= irq_flg; + tmp ^= ip >> 32; + _lrng_irq_array_add_u32(tmp); + } +} +EXPORT_SYMBOL(add_interrupt_randomness); + +static void lrng_irq_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for operating entropy pool: %s\n" + " Available entropy: %u\n" + " per-CPU interrupt collection size: %u\n" + " Standards compliance: %s\n" + " High-resolution timer: %s\n" + " Continuous compression: %s\n", + lrng_drng_init->hash_cb->hash_name(), + lrng_irq_avail_entropy(0), + LRNG_DATA_NUM_VALUES, + lrng_sp80090b_compliant(lrng_int_es_irq) ? "SP800-90B " : "", + lrng_highres_timer() ? "true" : "false", + lrng_irq_continuous_compression ? "true" : "false"); +} + +struct lrng_es_cb lrng_es_irq = { + .name = "IRQ", + .get_ent = lrng_irq_pool_hash, + .curr_entropy = lrng_irq_avail_entropy, + .max_entropy = lrng_irq_avail_pool_size, + .state = lrng_irq_es_state, + .reset = lrng_irq_reset, + .switch_hash = lrng_irq_switch_hash, +}; diff --git a/drivers/char/lrng/lrng_es_irq.h b/drivers/char/lrng/lrng_es_irq.h new file mode 100644 index 000000000000..2cd746611cf0 --- /dev/null +++ b/drivers/char/lrng/lrng_es_irq.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_IRQ_H +#define _LRNG_ES_IRQ_H + +#include + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_IRQ +void lrng_irq_es_init(bool highres_timer); +void lrng_irq_array_add_u32(u32 data); + +extern struct lrng_es_cb lrng_es_irq; + +#else /* CONFIG_LRNG_IRQ */ +static inline void lrng_irq_es_init(bool highres_timer) { } +static inline void lrng_irq_array_add_u32(u32 data) { } +#endif /* CONFIG_LRNG_IRQ */ + +#endif /* _LRNG_ES_IRQ_H */ diff --git a/drivers/char/lrng/lrng_es_jent.c b/drivers/char/lrng/lrng_es_jent.c new file mode 100644 index 000000000000..dbb2fce591f0 --- /dev/null +++ b/drivers/char/lrng/lrng_es_jent.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Fast Entropy Source: Jitter RNG + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_definitions.h" +#include "lrng_es_aux.h" +#include "lrng_es_jent.h" + +/* + * Estimated entropy of data is a 16th of LRNG_DRNG_SECURITY_STRENGTH_BITS. + * Albeit a full entropy assessment is provided for the noise source indicating + * that it provides high entropy rates and considering that it deactivates + * when it detects insufficient hardware, the chosen under estimation of + * entropy is considered to be acceptable to all reviewers. + */ +static u32 jent_entropy = CONFIG_LRNG_JENT_ENTROPY_RATE; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(jent_entropy, uint, 0644); +MODULE_PARM_DESC(jent_entropy, "Entropy in bits of 256 data bits from Jitter RNG noise source"); +#endif + +static bool lrng_jent_initialized = false; +static struct rand_data *lrng_jent_state; + +static int __init lrng_jent_initialize(void) +{ + /* Initialize the Jitter RNG after the clocksources are initialized. */ + if (jent_entropy_init() || + (lrng_jent_state = jent_entropy_collector_alloc(1, 0)) == NULL) { + jent_entropy = 0; + pr_info("Jitter RNG unusable on current system\n"); + return 0; + } + lrng_jent_initialized = true; + pr_debug("Jitter RNG working on current system\n"); + + /* + * In FIPS mode, the Jitter RNG is defined to have full of entropy + * unless a different value has been specified at the command line + * (i.e. the user overrides the default), and the default value is + * larger than zero (if it is zero, it is assumed that an RBG2(P) or + * RBG2(NP) construction is attempted that intends to exclude the + * Jitter RNG). + */ + if (fips_enabled && + CONFIG_LRNG_JENT_ENTROPY_RATE > 0 && + jent_entropy == CONFIG_LRNG_JENT_ENTROPY_RATE) + jent_entropy = LRNG_DRNG_SECURITY_STRENGTH_BITS; + + lrng_drng_force_reseed(); + if (jent_entropy) + lrng_es_add_entropy(); + + return 0; +} +device_initcall(lrng_jent_initialize); + +static u32 lrng_jent_entropylevel(u32 requested_bits) +{ + return lrng_fast_noise_entropylevel(lrng_jent_initialized ? + jent_entropy : 0, requested_bits); +} + +static u32 lrng_jent_poolsize(void) +{ + return lrng_jent_entropylevel(lrng_security_strength()); +} + +/* + * lrng_get_jent() - Get Jitter RNG entropy + * + * @eb: entropy buffer to store entropy + * @requested_bits: requested entropy in bits + */ +static void lrng_jent_get(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + int ret; + u32 ent_bits = lrng_jent_entropylevel(requested_bits); + unsigned long flags; + static DEFINE_SPINLOCK(lrng_jent_lock); + + spin_lock_irqsave(&lrng_jent_lock, flags); + + if (!lrng_jent_initialized) { + spin_unlock_irqrestore(&lrng_jent_lock, flags); + goto err; + } + + ret = jent_read_entropy(lrng_jent_state, eb->e[lrng_ext_es_jitter], + requested_bits >> 3); + spin_unlock_irqrestore(&lrng_jent_lock, flags); + + if (ret) { + pr_debug("Jitter RNG failed with %d\n", ret); + goto err; + } + + pr_debug("obtained %u bits of entropy from Jitter RNG noise source\n", + ent_bits); + + eb->e_bits[lrng_ext_es_jitter] = ent_bits; + return; + +err: + eb->e_bits[lrng_ext_es_jitter] = 0; +} + +static void lrng_jent_es_state(unsigned char *buf, size_t buflen) +{ + snprintf(buf, buflen, + " Available entropy: %u\n" + " Enabled: %s\n", + lrng_jent_poolsize(), + lrng_jent_initialized ? "true" : "false"); +} + +struct lrng_es_cb lrng_es_jent = { + .name = "JitterRNG", + .get_ent = lrng_jent_get, + .curr_entropy = lrng_jent_entropylevel, + .max_entropy = lrng_jent_poolsize, + .state = lrng_jent_es_state, + .reset = NULL, + .switch_hash = NULL, +}; diff --git a/drivers/char/lrng/lrng_es_jent.h b/drivers/char/lrng/lrng_es_jent.h new file mode 100644 index 000000000000..32882d4bdf99 --- /dev/null +++ b/drivers/char/lrng/lrng_es_jent.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_JENT_H +#define _LRNG_ES_JENT_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_JENT + +extern struct lrng_es_cb lrng_es_jent; + +#endif /* CONFIG_LRNG_JENT */ + +#endif /* _LRNG_ES_JENT_H */ diff --git a/drivers/char/lrng/lrng_es_krng.c b/drivers/char/lrng/lrng_es_krng.c new file mode 100644 index 000000000000..77e1bf9cead6 --- /dev/null +++ b/drivers/char/lrng/lrng_es_krng.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Fast Entropy Source: Linux kernel RNG (random.c) + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_es_aux.h" +#include "lrng_es_krng.h" + +static u32 krng_entropy = CONFIG_LRNG_KERNEL_RNG_ENTROPY_RATE; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(krng_entropy, uint, 0644); +MODULE_PARM_DESC(krng_entropy, "Entropy in bits of 256 data bits from the kernel RNG noise source"); +#endif + +static atomic_t lrng_krng_initial_rate = ATOMIC_INIT(0); + +static struct random_ready_callback lrng_krng_ready = { + .owner = THIS_MODULE, + .func = NULL, +}; + +static u32 lrng_krng_fips_entropylevel(u32 entropylevel) +{ + return fips_enabled ? 0 : entropylevel; +} + +static void lrng_krng_adjust_entropy(struct random_ready_callback *rdy) +{ + u32 entropylevel; + + krng_entropy = atomic_read_u32(&lrng_krng_initial_rate); + + entropylevel = lrng_krng_fips_entropylevel(krng_entropy); + pr_debug("Kernel RNG is fully seeded, setting entropy rate to %u bits of entropy\n", + entropylevel); + lrng_drng_force_reseed(); + if (entropylevel) + lrng_es_add_entropy(); +} + +static u32 lrng_krng_entropylevel(u32 requested_bits) +{ + if (lrng_krng_ready.func == NULL) { + int err; + + lrng_krng_ready.func = lrng_krng_adjust_entropy; + + err = add_random_ready_callback(&lrng_krng_ready); + switch (err) { + case 0: + atomic_set(&lrng_krng_initial_rate, krng_entropy); + krng_entropy = 0; + pr_debug("Kernel RNG is not yet seeded, setting entropy rate to 0 bits of entropy\n"); + break; + + case -EALREADY: + pr_debug("Kernel RNG is fully seeded, setting entropy rate to %u bits of entropy\n", + lrng_krng_fips_entropylevel(krng_entropy)); + break; + default: + lrng_krng_ready.func = NULL; + return 0; + } + } + + return lrng_fast_noise_entropylevel( + lrng_krng_fips_entropylevel(krng_entropy), requested_bits); +} + +static u32 lrng_krng_poolsize(void) +{ + return lrng_krng_entropylevel(lrng_security_strength()); +} + +/* + * lrng_krng_get() - Get kernel RNG entropy + * + * @eb: entropy buffer to store entropy + * @requested_bits: requested entropy in bits + */ +static void lrng_krng_get(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + u32 ent_bits = lrng_krng_entropylevel(requested_bits); + + get_random_bytes(eb->e[lrng_ext_es_krng], requested_bits >> 3); + + pr_debug("obtained %u bits of entropy from kernel RNG noise source\n", + ent_bits); + + eb->e_bits[lrng_ext_es_krng] = ent_bits; +} + +static void lrng_krng_es_state(unsigned char *buf, size_t buflen) +{ + snprintf(buf, buflen, + " Available entropy: %u\n" + " Entropy Rate per 256 data bits: %u\n", + lrng_krng_poolsize(), + lrng_krng_entropylevel(256)); +} + +struct lrng_es_cb lrng_es_krng = { + .name = "KernelRNG", + .get_ent = lrng_krng_get, + .curr_entropy = lrng_krng_entropylevel, + .max_entropy = lrng_krng_poolsize, + .state = lrng_krng_es_state, + .reset = NULL, + .switch_hash = NULL, +}; diff --git a/drivers/char/lrng/lrng_es_krng.h b/drivers/char/lrng/lrng_es_krng.h new file mode 100644 index 000000000000..cf982b9eea05 --- /dev/null +++ b/drivers/char/lrng/lrng_es_krng.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_RANDOM_H +#define _LRNG_ES_RANDOM_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_KERNEL_RNG + +extern struct lrng_es_cb lrng_es_krng; + +#endif /* CONFIG_LRNG_KERNEL_RNG */ + +#endif /* _LRNG_ES_RANDOM_H */ diff --git a/drivers/char/lrng/lrng_es_mgr.c b/drivers/char/lrng/lrng_es_mgr.c new file mode 100644 index 000000000000..7586f6a12b33 --- /dev/null +++ b/drivers/char/lrng/lrng_es_mgr.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Entropy sources management + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_cpu.h" +#include "lrng_es_irq.h" +#include "lrng_es_jent.h" +#include "lrng_es_krng.h" +#include "lrng_es_mgr.h" +#include "lrng_es_sched.h" +#include "lrng_interface_dev_common.h" +#include "lrng_interface_random_kernel.h" + +struct lrng_state { + bool can_invalidate; /* Can invalidate batched entropy? */ + bool perform_seedwork; /* Can seed work be performed? */ + bool lrng_operational; /* Is DRNG operational? */ + bool lrng_fully_seeded; /* Is DRNG fully seeded? */ + bool lrng_min_seeded; /* Is DRNG minimally seeded? */ + bool all_online_numa_node_seeded;/* All NUMA DRNGs seeded? */ + + /* + * To ensure that external entropy providers cannot dominate the + * internal noise sources but yet cannot be dominated by internal + * noise sources, the following booleans are intended to allow + * external to provide seed once when a DRNG reseed occurs. This + * triggering of external noise source is performed even when the + * entropy pool has sufficient entropy. + */ + + atomic_t boot_entropy_thresh; /* Reseed threshold */ + atomic_t reseed_in_progress; /* Flag for on executing reseed */ + struct work_struct lrng_seed_work; /* (re)seed work queue */ +}; + +static struct lrng_state lrng_state = { + false, false, false, false, false, false, + .boot_entropy_thresh = ATOMIC_INIT(LRNG_INIT_ENTROPY_BITS), + .reseed_in_progress = ATOMIC_INIT(0), +}; + +/* + * If the entropy count falls under this number of bits, then we + * should wake up processes which are selecting or polling on write + * access to /dev/random. + */ +u32 lrng_write_wakeup_bits = (LRNG_WRITE_WAKEUP_ENTROPY << 3); + +/* + * The entries must be in the same order as defined by enum lrng_internal_es and + * enum lrng_external_es + */ +struct lrng_es_cb *lrng_es[] = { +#ifdef CONFIG_LRNG_IRQ + &lrng_es_irq, +#endif +#ifdef CONFIG_LRNG_SCHED + &lrng_es_sched, +#endif +#ifdef CONFIG_LRNG_JENT + &lrng_es_jent, +#endif +#ifdef CONFIG_LRNG_CPU + &lrng_es_cpu, +#endif +#ifdef CONFIG_LRNG_KERNEL_RNG + &lrng_es_krng, +#endif + &lrng_es_aux +}; + +/********************************** Helper ***********************************/ + +void lrng_debug_report_seedlevel(const char *name) +{ +#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM + static void *previous = NULL; + void *caller = (void *) _RET_IP_; + + if (READ_ONCE(previous) == caller) + return; + + if (!lrng_state_min_seeded()) + pr_notice("%pS %s called without reaching minimally seeded level (available entropy %u)\n", + caller, name, lrng_avail_entropy()); + + WRITE_ONCE(previous, caller); +#endif +} + +/* + * Reading of the LRNG pool is only allowed by one caller. The reading is + * only performed to (re)seed DRNGs. Thus, if this "lock" is already taken, + * the reseeding operation is in progress. The caller is not intended to wait + * but continue with its other operation. + */ +int lrng_pool_trylock(void) +{ + return atomic_cmpxchg(&lrng_state.reseed_in_progress, 0, 1); +} + +void lrng_pool_unlock(void) +{ + atomic_set(&lrng_state.reseed_in_progress, 0); +} + +/* Set new entropy threshold for reseeding during boot */ +void lrng_set_entropy_thresh(u32 new_entropy_bits) +{ + atomic_set(&lrng_state.boot_entropy_thresh, new_entropy_bits); +} + +/* + * Reset LRNG state - the entropy counters are reset, but the data that may + * or may not have entropy remains in the pools as this data will not hurt. + */ +void lrng_reset_state(void) +{ + u32 i; + + for_each_lrng_es(i) { + if (lrng_es[i]->reset) + lrng_es[i]->reset(); + } + lrng_state.lrng_operational = false; + lrng_state.lrng_fully_seeded = false; + lrng_state.lrng_min_seeded = false; + lrng_state.all_online_numa_node_seeded = false; + pr_debug("reset LRNG\n"); +} + +/* Set flag that all DRNGs are fully seeded */ +void lrng_pool_all_numa_nodes_seeded(bool set) +{ + lrng_state.all_online_numa_node_seeded = set; +} + +/* Return boolean whether LRNG reached minimally seed level */ +bool lrng_state_min_seeded(void) +{ + return lrng_state.lrng_min_seeded; +} + +/* Return boolean whether LRNG reached fully seed level */ +bool lrng_state_fully_seeded(void) +{ + return lrng_state.lrng_fully_seeded; +} + +/* Return boolean whether LRNG is considered fully operational */ +bool lrng_state_operational(void) +{ + return lrng_state.lrng_operational; +} + +static void lrng_init_wakeup(void) +{ + wake_up_all(&lrng_init_wait); + lrng_init_wakeup_dev(); +} + +static bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy) +{ + return (collected_entropy >= lrng_get_seed_entropy_osr(fully_seeded)); +} + +/* Policy to check whether entropy buffer contains full seeded entropy */ +bool lrng_fully_seeded_eb(bool fully_seeded, struct entropy_buf *eb) +{ + u32 i, collected_entropy = 0; + + for_each_lrng_es(i) + collected_entropy += eb->e_bits[i]; + + return lrng_fully_seeded(fully_seeded, collected_entropy); +} + +/* Mark one DRNG as not fully seeded */ +void lrng_unset_fully_seeded(struct lrng_drng *drng) +{ + drng->fully_seeded = false; + lrng_pool_all_numa_nodes_seeded(false); + + /* + * The init DRNG instance must always be fully seeded as this instance + * is the fall-back if any of the per-NUMA node DRNG instances is + * insufficiently seeded. Thus, we mark the entire LRNG as + * non-operational if the initial DRNG becomes not fully seeded. + */ + if (drng == lrng_drng_init_instance() && lrng_state_operational()) { + pr_debug("LRNG set to non-operational\n"); + lrng_state.lrng_operational = false; + lrng_state.lrng_fully_seeded = false; + + /* If sufficient entropy is available, reseed now. */ + lrng_es_add_entropy(); + } +} + +/* Policy to enable LRNG operational mode */ +static void lrng_set_operational(void) +{ + /* + * LRNG is operational if the initial DRNG is fully seeded. This state + * can only occur if either the external entropy sources provided + * sufficient entropy, or the SP800-90B startup test completed for + * the internal ES to supply also entropy data. + */ + if (lrng_state.lrng_fully_seeded) { + lrng_state.lrng_operational = true; + lrng_process_ready_list(); + lrng_init_wakeup(); + pr_info("LRNG fully operational\n"); + } +} + +static u32 lrng_avail_entropy_thresh(void) +{ + u32 ent_thresh = lrng_security_strength(); + + /* + * Apply oversampling during initialization according to SP800-90C as + * we request a larger buffer from the ES. + */ + if (lrng_sp80090c_compliant() && + !lrng_state.all_online_numa_node_seeded) + ent_thresh += LRNG_SEED_BUFFER_INIT_ADD_BITS; + + return ent_thresh; +} + +/* Available entropy in the entire LRNG considering all entropy sources */ +u32 lrng_avail_entropy(void) +{ + u32 i, ent = 0, ent_thresh = lrng_avail_entropy_thresh(); + + BUILD_BUG_ON(ARRAY_SIZE(lrng_es) != lrng_ext_es_last); + for_each_lrng_es(i) + ent += lrng_es[i]->curr_entropy(ent_thresh); + return ent; +} + +/* + * lrng_init_ops() - Set seed stages of LRNG + * + * Set the slow noise source reseed trigger threshold. The initial threshold + * is set to the minimum data size that can be read from the pool: a word. Upon + * reaching this value, the next seed threshold of 128 bits is set followed + * by 256 bits. + * + * @eb: buffer containing the size of entropy currently injected into DRNG - if + * NULL, the function obtains the available entropy from the ES. + */ +void lrng_init_ops(struct entropy_buf *eb) +{ + struct lrng_state *state = &lrng_state; + u32 i, requested_bits, seed_bits = 0; + + if (state->lrng_operational) + return; + + requested_bits = lrng_get_seed_entropy_osr( + state->all_online_numa_node_seeded); + + if (eb) { + for_each_lrng_es(i) + seed_bits += eb->e_bits[i]; + } else { + u32 ent_thresh = lrng_avail_entropy_thresh(); + + for_each_lrng_es(i) + seed_bits += lrng_es[i]->curr_entropy(ent_thresh); + } + + /* DRNG is seeded with full security strength */ + if (state->lrng_fully_seeded) { + lrng_set_operational(); + lrng_set_entropy_thresh(requested_bits); + } else if (lrng_fully_seeded(state->all_online_numa_node_seeded, + seed_bits)) { + if (state->can_invalidate) + invalidate_batched_entropy(); + + state->lrng_fully_seeded = true; + lrng_set_operational(); + state->lrng_min_seeded = true; + pr_info("LRNG fully seeded with %u bits of entropy\n", + seed_bits); + lrng_set_entropy_thresh(requested_bits); + } else if (!state->lrng_min_seeded) { + + /* DRNG is seeded with at least 128 bits of entropy */ + if (seed_bits >= LRNG_MIN_SEED_ENTROPY_BITS) { + if (state->can_invalidate) + invalidate_batched_entropy(); + + state->lrng_min_seeded = true; + pr_info("LRNG minimally seeded with %u bits of entropy\n", + seed_bits); + lrng_set_entropy_thresh(requested_bits); + lrng_init_wakeup(); + + /* DRNG is seeded with at least LRNG_INIT_ENTROPY_BITS bits */ + } else if (seed_bits >= LRNG_INIT_ENTROPY_BITS) { + pr_info("LRNG initial entropy level %u bits of entropy\n", + seed_bits); + lrng_set_entropy_thresh(LRNG_MIN_SEED_ENTROPY_BITS); + } + } +} + +int __init lrng_rand_initialize(void) +{ + struct seed { + ktime_t time; + unsigned long data[(LRNG_MAX_DIGESTSIZE / + sizeof(unsigned long))]; + struct new_utsname utsname; + } seed __aligned(LRNG_KCAPI_ALIGN); + unsigned int i; + + BUILD_BUG_ON(LRNG_MAX_DIGESTSIZE % sizeof(unsigned long)); + + seed.time = ktime_get_real(); + + for (i = 0; i < ARRAY_SIZE(seed.data); i++) { +#ifdef CONFIG_LRNG_RANDOM_IF + if (!arch_get_random_seed_long_early(&(seed.data[i])) && + !arch_get_random_long_early(&seed.data[i])) +#else + if (!arch_get_random_seed_long(&(seed.data[i])) && + !arch_get_random_long(&seed.data[i])) +#endif + seed.data[i] = random_get_entropy(); + } + memcpy(&seed.utsname, utsname(), sizeof(*(utsname()))); + + lrng_pool_insert_aux((u8 *)&seed, sizeof(seed), 0); + memzero_explicit(&seed, sizeof(seed)); + + /* Initialize the seed work queue */ + INIT_WORK(&lrng_state.lrng_seed_work, lrng_drng_seed_work); + lrng_state.perform_seedwork = true; + + invalidate_batched_entropy(); + + lrng_state.can_invalidate = true; + + return 0; +} + +#ifndef CONFIG_LRNG_RANDOM_IF +early_initcall(lrng_rand_initialize); +#endif + +/* Interface requesting a reseed of the DRNG */ +void lrng_es_add_entropy(void) +{ + /* + * Once all DRNGs are fully seeded, the system-triggered arrival of + * entropy will not cause any reseeding any more. + */ + if (likely(lrng_state.all_online_numa_node_seeded)) + return; + + /* Only trigger the DRNG reseed if we have collected entropy. */ + if (lrng_avail_entropy() < + atomic_read_u32(&lrng_state.boot_entropy_thresh)) + return; + + /* Ensure that the seeding only occurs once at any given time. */ + if (lrng_pool_trylock()) + return; + + /* Seed the DRNG with any available noise. */ + if (lrng_state.perform_seedwork) + schedule_work(&lrng_state.lrng_seed_work); + else + lrng_drng_seed_work(NULL); +} + +/* Fill the seed buffer with data from the noise sources */ +void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits) +{ + struct lrng_state *state = &lrng_state; + u32 i, req_ent = lrng_sp80090c_compliant() ? + lrng_security_strength() : LRNG_MIN_SEED_ENTROPY_BITS; + + /* Guarantee that requested bits is a multiple of bytes */ + BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BITS % 8); + + /* always reseed the DRNG with the current time stamp */ + eb->now = random_get_entropy(); + + /* + * Require at least 128 bits of entropy for any reseed. If the LRNG is + * operated SP800-90C compliant we want to comply with SP800-90A section + * 9.2 mandating that DRNG is reseeded with the security strength. + */ + if (state->lrng_fully_seeded && (lrng_avail_entropy() < req_ent)) { + for_each_lrng_es(i) + eb->e_bits[i] = 0; + + goto wakeup; + } + + /* Concatenate the output of the entropy sources. */ + for_each_lrng_es(i) { + lrng_es[i]->get_ent(eb, requested_bits, + state->lrng_fully_seeded); + } + + /* allow external entropy provider to provide seed */ + lrng_state_exseed_allow_all(); + +wakeup: + lrng_writer_wakeup(); +} diff --git a/drivers/char/lrng/lrng_es_mgr.h b/drivers/char/lrng/lrng_es_mgr.h new file mode 100644 index 000000000000..e24f101de5a0 --- /dev/null +++ b/drivers/char/lrng/lrng_es_mgr.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_MGR_H +#define _LRNG_ES_MGR_H + +#include "lrng_es_mgr_cb.h" + +/*************************** General LRNG parameter ***************************/ + +#define LRNG_DRNG_BLOCKSIZE 64 /* Maximum of DRNG block sizes */ + +/* Helper to concatenate a macro with an integer type */ +#define LRNG_PASTER(x, y) x ## y +#define LRNG_UINT32_C(x) LRNG_PASTER(x, U) + +/************************* Entropy sources management *************************/ + +extern struct lrng_es_cb *lrng_es[]; + +#define for_each_lrng_es(ctr) \ + for ((ctr) = 0; (ctr) < lrng_ext_es_last; (ctr)++) + +bool lrng_state_min_seeded(void); +void lrng_debug_report_seedlevel(const char *name); +int lrng_rand_initialize(void); +bool lrng_state_operational(void); + +extern u32 lrng_write_wakeup_bits; +void lrng_set_entropy_thresh(u32 new); +u32 lrng_avail_entropy(void); +void lrng_reset_state(void); + +bool lrng_state_fully_seeded(void); + +int lrng_pool_trylock(void); +void lrng_pool_unlock(void); +void lrng_pool_all_numa_nodes_seeded(bool set); + +bool lrng_fully_seeded_eb(bool fully_seeded, struct entropy_buf *eb); +void lrng_unset_fully_seeded(struct lrng_drng *drng); +void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits); +void lrng_init_ops(struct entropy_buf *eb); + +#endif /* _LRNG_ES_MGR_H */ diff --git a/drivers/char/lrng/lrng_es_mgr_cb.h b/drivers/char/lrng/lrng_es_mgr_cb.h new file mode 100644 index 000000000000..08b24e1b7766 --- /dev/null +++ b/drivers/char/lrng/lrng_es_mgr_cb.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + * + * Definition of an entropy source. + */ + +#ifndef _LRNG_ES_MGR_CB_H +#define _LRNG_ES_MGR_CB_H + +#include + +#include "lrng_definitions.h" +#include "lrng_drng_mgr.h" + +enum lrng_internal_es { +#ifdef CONFIG_LRNG_IRQ + lrng_int_es_irq, /* IRQ-based entropy source */ +#endif +#ifdef CONFIG_LRNG_SCHED + lrng_int_es_sched, /* Scheduler entropy source */ +#endif + lrng_int_es_last, /* MUST be the last entry */ +}; + +enum lrng_external_es { + lrng_ext_link = lrng_int_es_last - 1, /* Link entry */ +#ifdef CONFIG_LRNG_JENT + lrng_ext_es_jitter, /* Jitter RNG */ +#endif +#ifdef CONFIG_LRNG_CPU + lrng_ext_es_cpu, /* CPU-based, e.g. RDSEED */ +#endif +#ifdef CONFIG_LRNG_KERNEL_RNG + lrng_ext_es_krng, /* random.c */ +#endif + lrng_ext_es_aux, /* MUST BE LAST ES! */ + lrng_ext_es_last /* MUST be the last entry */ +}; + +struct entropy_buf { + u8 e[lrng_ext_es_last][LRNG_DRNG_INIT_SEED_SIZE_BYTES]; + u32 now, e_bits[lrng_ext_es_last]; +}; + +/* + * struct lrng_es_cb - callback defining an entropy source + * @name: Name of the entropy source. + * @get_ent: Fetch entropy into the entropy_buf. The ES shall only deliver + * data if its internal initialization is complete, including any + * SP800-90B startup testing or similar. + * @curr_entropy: Return amount of currently available entropy. + * @max_entropy: Maximum amount of entropy the entropy source is able to + * maintain. + * @state: Buffer with human-readable ES state. + * @reset: Reset entropy source (drop all entropy and reinitialize). + * This callback may be NULL. + * @switch_hash: callback to switch from an old hash callback definition to + * a new one. This callback may be NULL. + */ +struct lrng_es_cb { + const char *name; + void (*get_ent)(struct entropy_buf *eb, u32 requested_bits, + bool fully_seeded); + u32 (*curr_entropy)(u32 requested_bits); + u32 (*max_entropy)(void); + void (*state)(unsigned char *buf, size_t buflen); + void (*reset)(void); + int (*switch_hash)(struct lrng_drng *drng, int node, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb); +}; + +/* Allow entropy sources to tell the ES manager that new entropy is there */ +void lrng_es_add_entropy(void); + +/* Cap to maximum entropy that can ever be generated with given hash */ +#define lrng_cap_requested(__digestsize_bits, __requested_bits) \ + do { \ + if (__digestsize_bits < __requested_bits) { \ + pr_debug("Cannot satisfy requested entropy %u due to insufficient hash size %u\n",\ + __requested_bits, __digestsize_bits); \ + __requested_bits = __digestsize_bits; \ + } \ + } while (0) + +#endif /* _LRNG_ES_MGR_CB_H */ diff --git a/drivers/char/lrng/lrng_es_sched.c b/drivers/char/lrng/lrng_es_sched.c new file mode 100644 index 000000000000..f7e85c381e22 --- /dev/null +++ b/drivers/char/lrng/lrng_es_sched.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Scheduler-based data collection + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#include "lrng_es_aux.h" +#include "lrng_es_sched.h" +#include "lrng_es_timer_common.h" +#include "lrng_health.h" +#include "lrng_numa.h" +#include "lrng_testing.h" + +/* + * Number of scheduler-based context switches to be recorded to assume that + * DRNG security strength bits of entropy are received. + * Note: a value below the DRNG security strength should not be defined as this + * may imply the DRNG can never be fully seeded in case other noise + * sources are unavailable. + */ +#define LRNG_SCHED_ENTROPY_BITS \ + LRNG_UINT32_C(CONFIG_LRNG_SCHED_ENTROPY_RATE) + +/* Number of events required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ +static u32 lrng_sched_entropy_bits = LRNG_SCHED_ENTROPY_BITS; + +static u32 sched_entropy __read_mostly = LRNG_SCHED_ENTROPY_BITS; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(sched_entropy, uint, 0444); +MODULE_PARM_DESC(sched_entropy, + "How many scheduler-based context switches must be collected for obtaining 256 bits of entropy\n"); +#endif + +/* Per-CPU array holding concatenated entropy events */ +static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_sched_array) + __aligned(LRNG_KCAPI_ALIGN); +static DEFINE_PER_CPU(u32, lrng_sched_array_ptr) = 0; +static DEFINE_PER_CPU(atomic_t, lrng_sched_array_events) = ATOMIC_INIT(0); + +static void __init lrng_sched_check_compression_state(void) +{ + /* One pool should hold sufficient entropy for disabled compression */ + u32 max_ent = min_t(u32, lrng_get_digestsize(), + lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, + lrng_sched_entropy_bits)); + if (max_ent < lrng_security_strength()) { + pr_devel("Scheduler entropy source will never provide %u bits of entropy required for fully seeding the DRNG all by itself\n", + lrng_security_strength()); + } +} + +void __init lrng_sched_es_init(bool highres_timer) +{ + /* Set a minimum number of scheduler events that must be collected */ + sched_entropy = max_t(u32, LRNG_SCHED_ENTROPY_BITS, sched_entropy); + + if (highres_timer) { + lrng_sched_entropy_bits = sched_entropy; + } else { + u32 new_entropy = sched_entropy * LRNG_ES_OVERSAMPLING_FACTOR; + + lrng_sched_entropy_bits = (sched_entropy < new_entropy) ? + new_entropy : sched_entropy; + pr_warn("operating without high-resolution timer and applying oversampling factor %u\n", + LRNG_ES_OVERSAMPLING_FACTOR); + } + + lrng_sched_check_compression_state(); +} + +static u32 lrng_sched_avail_pool_size(void) +{ + u32 max_pool = lrng_get_digestsize(), + max_size = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); + int cpu; + + for_each_online_cpu(cpu) + max_size += max_pool; + + return max_size; +} + +/* Return entropy of unused scheduler events present in all per-CPU pools. */ +static u32 lrng_sched_avail_entropy(u32 __unused) +{ + u32 digestsize_events, events = 0; + int cpu; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) + return 0; + + /* Obtain the cap of maximum numbers of scheduler events we count */ + digestsize_events = lrng_entropy_to_data(lrng_get_digestsize(), + lrng_sched_entropy_bits); + /* Cap to max. number of scheduler events the array can hold */ + digestsize_events = min_t(u32, digestsize_events, LRNG_DATA_NUM_VALUES); + + for_each_online_cpu(cpu) { + events += min_t(u32, digestsize_events, + atomic_read_u32(per_cpu_ptr(&lrng_sched_array_events, + cpu))); + } + + /* Consider oversampling rate */ + return lrng_reduce_by_osr( + lrng_data_to_entropy(events, lrng_sched_entropy_bits)); +} + +/* + * Reset all per-CPU pools - reset entropy estimator but leave the pool data + * that may or may not have entropy unchanged. + */ +static void lrng_sched_reset(void) +{ + int cpu; + + /* Trigger GCD calculation anew. */ + lrng_gcd_set(0); + + for_each_online_cpu(cpu) + atomic_set(per_cpu_ptr(&lrng_sched_array_events, cpu), 0); +} + +/* + * Hash all per-CPU arrays and return the digest to be used as seed data for + * seeding a DRNG. The caller must guarantee backtracking resistance. + * The function will only copy as much data as entropy is available into the + * caller-provided output buffer. + * + * This function handles the translation from the number of received scheduler + * events into an entropy statement. The conversion depends on + * LRNG_SCHED_ENTROPY_BITS which defines how many scheduler events must be + * received to obtain 256 bits of entropy. With this value, the function + * lrng_data_to_entropy converts a given data size (received scheduler events, + * requested amount of data, etc.) into an entropy statement. + * lrng_entropy_to_data does the reverse. + * + * @eb: entropy buffer to store entropy + * @requested_bits: Requested amount of entropy + * @fully_seeded: indicator whether LRNG is fully seeded + */ +static void lrng_sched_pool_hash(struct entropy_buf *eb, u32 requested_bits, + bool fully_seeded) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb; + struct lrng_drng *drng = lrng_drng_node_instance(); + u8 digest[LRNG_MAX_DIGESTSIZE]; + unsigned long flags; + u32 found_events, collected_events = 0, collected_ent_bits, + requested_events, returned_ent_bits, digestsize, digestsize_events; + int ret, cpu; + void *hash; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) { + eb->e_bits[lrng_int_es_sched] = 0; + return; + } + + /* Lock guarding replacement of per-NUMA hash */ + read_lock_irqsave(&drng->hash_lock, flags); + hash_cb = drng->hash_cb; + hash = drng->hash; + + /* The hash state of filled with all per-CPU pool hashes. */ + ret = hash_cb->hash_init(shash, hash); + if (ret) + goto err; + + digestsize = hash_cb->hash_digestsize(shash); + + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(digestsize << 3, requested_bits); + requested_events = lrng_entropy_to_data(requested_bits + + lrng_compress_osr(), + lrng_sched_entropy_bits); + digestsize_events = lrng_entropy_to_data(digestsize << 3, + lrng_sched_entropy_bits); + + /* + * Harvest entropy from each per-CPU hash state - even though we may + * have collected sufficient entropy, we will hash all per-CPU pools. + */ + for_each_online_cpu(cpu) { + u32 unused_events = 0; + + ret = hash_cb->hash_update(shash, + (u8 *)per_cpu_ptr(lrng_sched_array, cpu), + LRNG_DATA_ARRAY_SIZE * sizeof(u32)); + + /* Store all not-yet compressed data in data array into hash */ + ret = hash_cb->hash_update(shash, digest, digestsize); + if (ret) + goto err; + + /* Obtain entropy statement like for the entropy pool */ + found_events = atomic_xchg_relaxed( + per_cpu_ptr(&lrng_sched_array_events, cpu), 0); + /* Cap to maximum amount of data we can hold in hash */ + found_events = min_t(u32, found_events, digestsize_events); + /* Cap to maximum amount of data we can hold in array */ + found_events = min_t(u32, found_events, LRNG_DATA_NUM_VALUES); + + collected_events += found_events; + if (collected_events > requested_events) { + unused_events = collected_events - requested_events; + atomic_add_return_relaxed(unused_events, + per_cpu_ptr(&lrng_sched_array_events, cpu)); + collected_events = requested_events; + } + pr_debug("%u scheduler-based events used from entropy array of CPU %d, %u scheduler-based events remain unused\n", + found_events - unused_events, cpu, unused_events); + } + + ret = hash_cb->hash_final(shash, digest); + if (ret) + goto err; + + collected_ent_bits = lrng_data_to_entropy(collected_events, + lrng_sched_entropy_bits); + /* Apply oversampling: discount requested oversampling rate */ + returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); + + pr_debug("obtained %u bits by collecting %u bits of entropy from scheduler-based noise source\n", + returned_ent_bits, collected_ent_bits); + + /* + * Truncate to available entropy as implicitly allowed by SP800-90B + * section 3.1.5.1.1 table 1 which awards truncated hashes full + * entropy. + * + * During boot time, we read requested_bits data with + * returned_ent_bits entropy. In case our conservative entropy + * estimate underestimates the available entropy we can transport as + * much available entropy as possible. + */ + memcpy(eb->e[lrng_int_es_sched], digest, + fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); + eb->e_bits[lrng_int_es_sched] = returned_ent_bits; + +out: + hash_cb->hash_desc_zero(shash); + read_unlock_irqrestore(&drng->hash_lock, flags); + memzero_explicit(digest, sizeof(digest)); + return; + +err: + eb->e_bits[lrng_int_es_sched] = 0; + goto out; +} + +/* + * Concatenate full 32 bit word at the end of time array even when current + * ptr is not aligned to sizeof(data). + */ +static void lrng_sched_array_add_u32(u32 data) +{ + /* Increment pointer by number of slots taken for input value */ + u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_sched_array_ptr, + LRNG_DATA_SLOTS_PER_UINT); + unsigned int pre_array; + + lrng_data_split_u32(&ptr, &pre_ptr, &mask); + + /* MSB of data go into previous unit */ + pre_array = lrng_data_idx2array(pre_ptr); + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_sched_array[pre_array], ~(0xffffffff & ~mask)); + this_cpu_or(lrng_sched_array[pre_array], data & ~mask); + + /* + * Continuous compression is not allowed for scheduler noise source, + * so do not call lrng_sched_array_to_hash here. + */ + + /* LSB of data go into current unit */ + this_cpu_write(lrng_sched_array[lrng_data_idx2array(ptr)], + data & mask); +} + +/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ +static void lrng_sched_array_add_slot(u32 data) +{ + /* Get slot */ + u32 ptr = this_cpu_inc_return(lrng_sched_array_ptr) & + LRNG_DATA_WORD_MASK; + unsigned int array = lrng_data_idx2array(ptr); + unsigned int slot = lrng_data_idx2slot(ptr); + + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_sched_array[array], + ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, + slot))); + /* Store data into slot */ + this_cpu_or(lrng_sched_array[array], lrng_data_slot_val(data, slot)); + + /* + * Continuous compression is not allowed for scheduler noise source, + * so do not call lrng_sched_array_to_hash here. + */ +} + +static void +lrng_time_process_common(u32 time, void(*add_time)(u32 data)) +{ + enum lrng_health_res health_test; + + if (lrng_raw_sched_hires_entropy_store(time)) + return; + + health_test = lrng_health_test(time, lrng_int_es_sched); + if (health_test > lrng_health_fail_use) + return; + + if (health_test == lrng_health_pass) + atomic_inc_return(this_cpu_ptr(&lrng_sched_array_events)); + + add_time(time); + + /* + * We cannot call lrng_es_add_entropy() as this would call a schedule + * operation that is not permissible in scheduler context. + * As the scheduler ES provides a high bandwidth of entropy, we assume + * that other reseed triggers happen to pick up the scheduler ES + * entropy in due time. + */ +} + +/* Batching up of entropy in per-CPU array */ +static void lrng_sched_time_process(void) +{ + u32 now_time = random_get_entropy(); + + if (unlikely(!lrng_gcd_tested())) { + /* When GCD is unknown, we process the full time stamp */ + lrng_time_process_common(now_time, lrng_sched_array_add_u32); + lrng_gcd_add_value(now_time); + } else { + /* GCD is known and applied */ + lrng_time_process_common((now_time / lrng_gcd_get()) & + LRNG_DATA_SLOTSIZE_MASK, + lrng_sched_array_add_slot); + } + + lrng_sched_perf_time(now_time); +} + +void add_sched_randomness(const struct task_struct *p, int cpu) +{ + if (lrng_highres_timer()) { + lrng_sched_time_process(); + } else { + u32 tmp = cpu; + + tmp ^= lrng_raw_sched_pid_entropy_store(p->pid) ? + 0 : (u32)p->pid; + tmp ^= lrng_raw_sched_starttime_entropy_store(p->start_time) ? + 0 : (u32)p->start_time; + tmp ^= lrng_raw_sched_nvcsw_entropy_store(p->nvcsw) ? + 0 : (u32)p->nvcsw; + + lrng_sched_time_process(); + lrng_sched_array_add_u32(tmp); + } +} +EXPORT_SYMBOL(add_sched_randomness); + +static void lrng_sched_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for operating entropy pool: %s\n" + " Available entropy: %u\n" + " per-CPU scheduler event collection size: %u\n" + " Standards compliance: %s\n" + " High-resolution timer: %s\n", + lrng_drng_init->hash_cb->hash_name(), + lrng_sched_avail_entropy(0), + LRNG_DATA_NUM_VALUES, + lrng_sp80090b_compliant(lrng_int_es_sched) ? "SP800-90B " : "", + lrng_highres_timer() ? "true" : "false"); +} + +struct lrng_es_cb lrng_es_sched = { + .name = "Scheduler", + .get_ent = lrng_sched_pool_hash, + .curr_entropy = lrng_sched_avail_entropy, + .max_entropy = lrng_sched_avail_pool_size, + .state = lrng_sched_es_state, + .reset = lrng_sched_reset, + .switch_hash = NULL, +}; diff --git a/drivers/char/lrng/lrng_es_sched.h b/drivers/char/lrng/lrng_es_sched.h new file mode 100644 index 000000000000..f1e596dd89d9 --- /dev/null +++ b/drivers/char/lrng/lrng_es_sched.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_SCHED_H +#define _LRNG_ES_SCHED_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_SCHED +void lrng_sched_es_init(bool highres_timer); + +extern struct lrng_es_cb lrng_es_sched; + +#else /* CONFIG_LRNG_SCHED */ +static inline void lrng_sched_es_init(bool highres_timer) { } +#endif /* CONFIG_LRNG_SCHED */ + +#endif /* _LRNG_ES_SCHED_H */ diff --git a/drivers/char/lrng/lrng_es_timer_common.c b/drivers/char/lrng/lrng_es_timer_common.c new file mode 100644 index 000000000000..70f3ff074fe6 --- /dev/null +++ b/drivers/char/lrng/lrng_es_timer_common.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Interrupt data collection + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_es_irq.h" +#include "lrng_es_sched.h" +#include "lrng_es_timer_common.h" +#include "lrng_health.h" + +/* Is high-resolution timer present? */ +static bool lrng_highres_timer_val = false; + +/* Number of time stamps analyzed to calculate a GCD */ +#define LRNG_GCD_WINDOW_SIZE 100 +static u32 lrng_gcd_history[LRNG_GCD_WINDOW_SIZE]; +static atomic_t lrng_gcd_history_ptr = ATOMIC_INIT(-1); + +/* The common divisor for all timestamps */ +static u32 lrng_gcd_timer = 0; + +bool lrng_gcd_tested(void) +{ + return (lrng_gcd_timer != 0); +} + +u32 lrng_gcd_get(void) +{ + return lrng_gcd_timer; +} + +/* Set the GCD for use in IRQ ES - if 0, the GCD calculation is restarted. */ +void lrng_gcd_set(u32 running_gcd) +{ + lrng_gcd_timer = running_gcd; + /* Ensure that update to global variable lrng_gcd_timer is visible */ + mb(); +} + +static void lrng_gcd_set_check(u32 running_gcd) +{ + if (!lrng_gcd_tested()) { + lrng_gcd_set(running_gcd); + pr_debug("Setting GCD to %u\n", running_gcd); + } +} + +u32 lrng_gcd_analyze(u32 *history, size_t nelem) +{ + u32 running_gcd = 0; + size_t i; + + /* Now perform the analysis on the accumulated time data. */ + for (i = 0; i < nelem; i++) { + /* + * NOTE: this would be the place to add more analysis on the + * appropriateness of the timer like checking the presence + * of sufficient variations in the timer. + */ + + /* + * This calculates the gcd of all the time values. that is + * gcd(time_1, time_2, ..., time_nelem) + * + * Some timers increment by a fixed (non-1) amount each step. + * This code checks for such increments, and allows the library + * to output the number of such changes have occurred. + */ + running_gcd = (u32)gcd(history[i], running_gcd); + + /* Zeroize data */ + history[i] = 0; + } + + return running_gcd; +} + +void lrng_gcd_add_value(u32 time) +{ + u32 ptr = (u32)atomic_inc_return_relaxed(&lrng_gcd_history_ptr); + + if (ptr < LRNG_GCD_WINDOW_SIZE) { + lrng_gcd_history[ptr] = time; + } else if (ptr == LRNG_GCD_WINDOW_SIZE) { + u32 gcd = lrng_gcd_analyze(lrng_gcd_history, + LRNG_GCD_WINDOW_SIZE); + + if (!gcd) + gcd = 1; + + /* + * Ensure that we have variations in the time stamp below the + * given value. This is just a safety measure to prevent the GCD + * becoming too large. + */ + if (gcd >= 1000) { + pr_warn("calculated GCD is larger than expected: %u\n", + gcd); + gcd = 1000; + } + + /* Adjust all deltas by the observed (small) common factor. */ + lrng_gcd_set_check(gcd); + atomic_set(&lrng_gcd_history_ptr, 0); + } +} + +/* Return boolean whether LRNG identified presence of high-resolution timer */ +bool lrng_highres_timer(void) +{ + return lrng_highres_timer_val; +} + +static int __init lrng_init_time_source(void) +{ + if ((random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK) || + (random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK)) { + /* + * As the highres timer is identified here, previous interrupts + * obtained during boot time are treated like a lowres-timer + * would have been present. + */ + lrng_highres_timer_val = true; + } else { + lrng_health_disable(); + lrng_highres_timer_val = false; + } + + lrng_irq_es_init(lrng_highres_timer_val); + lrng_sched_es_init(lrng_highres_timer_val); + + /* Ensure that changes to global variables are visible */ + mb(); + + return 0; +} +core_initcall(lrng_init_time_source); diff --git a/drivers/char/lrng/lrng_es_timer_common.h b/drivers/char/lrng/lrng_es_timer_common.h new file mode 100644 index 000000000000..b45b9f683dea --- /dev/null +++ b/drivers/char/lrng/lrng_es_timer_common.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG Slow Noise Source: Time stamp array handling + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_TIMER_COMMON_H +#define _LRNG_ES_TIMER_COMMON_H + +bool lrng_gcd_tested(void); +void lrng_gcd_set(u32 running_gcd); +u32 lrng_gcd_get(void); +u32 lrng_gcd_analyze(u32 *history, size_t nelem); +void lrng_gcd_add_value(u32 time); +bool lrng_highres_timer(void); + +/* + * To limit the impact on the interrupt handling, the LRNG concatenates + * entropic LSB parts of the time stamps in a per-CPU array and only + * injects them into the entropy pool when the array is full. + */ + +/* Store multiple integers in one u32 */ +#define LRNG_DATA_SLOTSIZE_BITS (8) +#define LRNG_DATA_SLOTSIZE_MASK ((1 << LRNG_DATA_SLOTSIZE_BITS) - 1) +#define LRNG_DATA_ARRAY_MEMBER_BITS (4 << 3) /* ((sizeof(u32)) << 3) */ +#define LRNG_DATA_SLOTS_PER_UINT (LRNG_DATA_ARRAY_MEMBER_BITS / \ + LRNG_DATA_SLOTSIZE_BITS) + +/* + * Number of time values to store in the array - in small environments + * only one atomic_t variable per CPU is used. + */ +#define LRNG_DATA_NUM_VALUES (CONFIG_LRNG_COLLECTION_SIZE) +/* Mask of LSB of time stamp to store */ +#define LRNG_DATA_WORD_MASK (LRNG_DATA_NUM_VALUES - 1) + +#define LRNG_DATA_SLOTS_MASK (LRNG_DATA_SLOTS_PER_UINT - 1) +#define LRNG_DATA_ARRAY_SIZE (LRNG_DATA_NUM_VALUES / \ + LRNG_DATA_SLOTS_PER_UINT) + +/* Starting bit index of slot */ +static inline unsigned int lrng_data_slot2bitindex(unsigned int slot) +{ + return (LRNG_DATA_SLOTSIZE_BITS * slot); +} + +/* Convert index into the array index */ +static inline unsigned int lrng_data_idx2array(unsigned int idx) +{ + return idx / LRNG_DATA_SLOTS_PER_UINT; +} + +/* Convert index into the slot of a given array index */ +static inline unsigned int lrng_data_idx2slot(unsigned int idx) +{ + return idx & LRNG_DATA_SLOTS_MASK; +} + +/* Convert value into slot value */ +static inline unsigned int lrng_data_slot_val(unsigned int val, + unsigned int slot) +{ + return val << lrng_data_slot2bitindex(slot); +} + +/* + * Return the pointers for the previous and current units to inject a u32 into. + * Also return the mask which the u32 word is to be processed. + */ +static inline void lrng_data_split_u32(u32 *ptr, u32 *pre_ptr, u32 *mask) +{ + /* ptr to previous unit */ + *pre_ptr = (*ptr - LRNG_DATA_SLOTS_PER_UINT) & LRNG_DATA_WORD_MASK; + *ptr &= LRNG_DATA_WORD_MASK; + + /* mask to split data into the two parts for the two units */ + *mask = ((1 << (*pre_ptr & (LRNG_DATA_SLOTS_PER_UINT - 1)) * + LRNG_DATA_SLOTSIZE_BITS)) - 1; +} + +#endif /* _LRNG_ES_TIMER_COMMON_H */ \ No newline at end of file diff --git a/drivers/char/lrng/lrng_hash_kcapi.c b/drivers/char/lrng/lrng_hash_kcapi.c new file mode 100644 index 000000000000..13e62db9b6c8 --- /dev/null +++ b/drivers/char/lrng/lrng_hash_kcapi.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for providing the hash primitive using the kernel crypto API. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + + +static char *lrng_hash_name = "sha512"; + +/* The parameter must be r/o in sysfs as otherwise races appear. */ +module_param(lrng_hash_name, charp, 0444); +MODULE_PARM_DESC(lrng_hash_name, "Kernel crypto API hash name"); + +struct lrng_hash_info { + struct crypto_shash *tfm; +}; + +static const char *lrng_kcapi_hash_name(void) +{ + return lrng_hash_name; +} + +static void _lrng_kcapi_hash_free(struct lrng_hash_info *lrng_hash) +{ + struct crypto_shash *tfm = lrng_hash->tfm; + + crypto_free_shash(tfm); + kfree(lrng_hash); +} + +static void *lrng_kcapi_hash_alloc(const char *name) +{ + struct lrng_hash_info *lrng_hash; + struct crypto_shash *tfm; + int ret; + + if (!name) { + pr_err("Hash name missing\n"); + return ERR_PTR(-EINVAL); + } + + tfm = crypto_alloc_shash(name, 0, 0); + if (IS_ERR(tfm)) { + pr_err("could not allocate hash %s\n", name); + return ERR_CAST(tfm); + } + + ret = sizeof(struct lrng_hash_info); + lrng_hash = kmalloc(ret, GFP_KERNEL); + if (!lrng_hash) { + crypto_free_shash(tfm); + return ERR_PTR(-ENOMEM); + } + + lrng_hash->tfm = tfm; + + pr_info("Hash %s allocated\n", name); + + return lrng_hash; +} + +static void *lrng_kcapi_hash_name_alloc(void) +{ + return lrng_kcapi_hash_alloc(lrng_kcapi_hash_name()); +} + +static u32 lrng_kcapi_hash_digestsize(void *hash) +{ + struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; + struct crypto_shash *tfm = lrng_hash->tfm; + + return crypto_shash_digestsize(tfm); +} + +static void lrng_kcapi_hash_dealloc(void *hash) +{ + struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; + + _lrng_kcapi_hash_free(lrng_hash); + pr_info("Hash deallocated\n"); +} + +static int lrng_kcapi_hash_init(struct shash_desc *shash, void *hash) +{ + struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; + struct crypto_shash *tfm = lrng_hash->tfm; + + shash->tfm = tfm; + return crypto_shash_init(shash); +} + +static int lrng_kcapi_hash_update(struct shash_desc *shash, const u8 *inbuf, + u32 inbuflen) +{ + return crypto_shash_update(shash, inbuf, inbuflen); +} + +static int lrng_kcapi_hash_final(struct shash_desc *shash, u8 *digest) +{ + return crypto_shash_final(shash, digest); +} + +static void lrng_kcapi_hash_zero(struct shash_desc *shash) +{ + shash_desc_zero(shash); +} + +static const struct lrng_hash_cb lrng_kcapi_hash_cb = { + .hash_name = lrng_kcapi_hash_name, + .hash_alloc = lrng_kcapi_hash_name_alloc, + .hash_dealloc = lrng_kcapi_hash_dealloc, + .hash_digestsize = lrng_kcapi_hash_digestsize, + .hash_init = lrng_kcapi_hash_init, + .hash_update = lrng_kcapi_hash_update, + .hash_final = lrng_kcapi_hash_final, + .hash_desc_zero = lrng_kcapi_hash_zero, +}; + +static int __init lrng_kcapi_init(void) +{ + return lrng_set_hash_cb(&lrng_kcapi_hash_cb); +} + +static void __exit lrng_kcapi_exit(void) +{ + lrng_set_hash_cb(NULL); +} + +late_initcall(lrng_kcapi_init); +module_exit(lrng_kcapi_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - Kernel crypto API hash backend"); diff --git a/drivers/char/lrng/lrng_health.c b/drivers/char/lrng/lrng_health.c new file mode 100644 index 000000000000..a60c8378eea5 --- /dev/null +++ b/drivers/char/lrng/lrng_health.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Entropy Source and DRNG Manager (LRNG) Health Testing + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_es_mgr.h" +#include "lrng_health.h" + +/* Stuck Test */ +struct lrng_stuck_test { + u32 last_time; /* Stuck test: time of previous IRQ */ + u32 last_delta; /* Stuck test: delta of previous IRQ */ + u32 last_delta2; /* Stuck test: 2. time derivation of prev IRQ */ +}; + +/* Repetition Count Test */ +struct lrng_rct { + atomic_t rct_count; /* Number of stuck values */ +}; + +/* Adaptive Proportion Test */ +struct lrng_apt { + /* Data window size */ +#define LRNG_APT_WINDOW_SIZE 512 + /* LSB of time stamp to process */ +#define LRNG_APT_LSB 16 +#define LRNG_APT_WORD_MASK (LRNG_APT_LSB - 1) + atomic_t apt_count; /* APT counter */ + atomic_t apt_base; /* APT base reference */ + + atomic_t apt_trigger; + bool apt_base_set; /* Is APT base set? */ +}; + +/* Health data collected for one entropy source */ +struct lrng_health_es_state { + struct lrng_rct rct; + struct lrng_apt apt; + + /* SP800-90B startup health tests */ +#define LRNG_SP80090B_STARTUP_SAMPLES 1024 +#define LRNG_SP80090B_STARTUP_BLOCKS ((LRNG_SP80090B_STARTUP_SAMPLES + \ + LRNG_APT_WINDOW_SIZE - 1) / \ + LRNG_APT_WINDOW_SIZE) + bool sp80090b_startup_done; + atomic_t sp80090b_startup_blocks; +}; + +#define LRNG_HEALTH_ES_INIT(x) \ + x.rct.rct_count = ATOMIC_INIT(0), \ + x.apt.apt_count = ATOMIC_INIT(0), \ + x.apt.apt_base = ATOMIC_INIT(-1), \ + x.apt.apt_trigger = ATOMIC_INIT(LRNG_APT_WINDOW_SIZE), \ + x.apt.apt_base_set = false, \ + x.sp80090b_startup_blocks = ATOMIC_INIT(LRNG_SP80090B_STARTUP_BLOCKS), \ + x.sp80090b_startup_done = false, + +/* The health test code must operate lock-less */ +struct lrng_health { + bool health_test_enabled; + struct lrng_health_es_state es_state[lrng_int_es_last]; +}; + +static struct lrng_health lrng_health = { + .health_test_enabled = true, + +#ifdef CONFIG_LRNG_IRQ + LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_irq]) +#endif +#ifdef CONFIG_LRNG_SCHED + LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_sched]) +#endif +}; + +static DEFINE_PER_CPU(struct lrng_stuck_test[lrng_int_es_last], + lrng_stuck_test_array); + +static bool lrng_sp80090b_health_requested(void) +{ + /* Health tests are only requested in FIPS mode */ + return fips_enabled; +} + +static bool lrng_sp80090b_health_enabled(void) +{ + struct lrng_health *health = &lrng_health; + + return lrng_sp80090b_health_requested() && health->health_test_enabled; +} + +/*************************************************************************** + * SP800-90B Compliance + * + * If the Linux-RNG is booted into FIPS mode, the following interfaces + * provide an SP800-90B compliant noise source: + * + * * /dev/random + * * getrandom(2) + * * get_random_bytes_full + * + * All other interfaces, including /dev/urandom or get_random_bytes without + * the add_random_ready_callback cannot claim to use an SP800-90B compliant + * noise source. + ***************************************************************************/ + +/* + * Perform SP800-90B startup testing + */ +static void lrng_sp80090b_startup(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + if (!es_state->sp80090b_startup_done && + atomic_dec_and_test(&es_state->sp80090b_startup_blocks)) { + es_state->sp80090b_startup_done = true; + pr_info("SP800-90B startup health tests for internal entropy source %u completed\n", + es); + lrng_drng_force_reseed(); + + /* + * We cannot call lrng_es_add_entropy() as this may cause a + * schedule operation while in scheduler context for the + * scheduler ES. + */ + } +} + +/* + * Handle failure of SP800-90B startup testing + */ +static void lrng_sp80090b_startup_failure(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + /* Reset of LRNG and its entropy - NOTE: we are in atomic context */ + lrng_reset(); + + /* + * Reset the SP800-90B startup test. + * + * NOTE SP800-90B section 4.3 bullet 4 does not specify what + * exactly is to be done in case of failure! Thus, we do what + * makes sense, i.e. restarting the health test and thus gating + * the output function of /dev/random and getrandom(2). + */ + atomic_set(&es_state->sp80090b_startup_blocks, + LRNG_SP80090B_STARTUP_BLOCKS); +} + +/* + * Handle failure of SP800-90B runtime testing + */ +static void lrng_sp80090b_runtime_failure(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + lrng_sp80090b_startup_failure(health, es); + es_state->sp80090b_startup_done = false; +} + +static void lrng_sp80090b_failure(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + if (es_state->sp80090b_startup_done) { + pr_err("SP800-90B runtime health test failure for internal entropy source %u - invalidating all existing entropy and initiate SP800-90B startup\n", es); + lrng_sp80090b_runtime_failure(health, es); + } else { + pr_err("SP800-90B startup test failure for internal entropy source %u - resetting\n", es); + lrng_sp80090b_startup_failure(health, es); + } +} + +bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) +{ + struct lrng_health *health = &lrng_health; + struct lrng_health_es_state *es_state = &health->es_state[es]; + + if (!lrng_sp80090b_health_enabled()) + return true; + + return es_state->sp80090b_startup_done; +} + +bool lrng_sp80090b_compliant(enum lrng_internal_es es) +{ + struct lrng_health *health = &lrng_health; + struct lrng_health_es_state *es_state = &health->es_state[es]; + + return lrng_sp80090b_health_enabled() && + es_state->sp80090b_startup_done; +} + +/*************************************************************************** + * Adaptive Proportion Test + * + * This test complies with SP800-90B section 4.4.2. + ***************************************************************************/ + +/* + * Reset the APT counter + * + * @health [in] Reference to health state + */ +static void lrng_apt_reset(struct lrng_apt *apt, unsigned int time_masked) +{ + /* Reset APT */ + atomic_set(&apt->apt_count, 0); + atomic_set(&apt->apt_base, time_masked); +} + +static void lrng_apt_restart(struct lrng_apt *apt) +{ + atomic_set(&apt->apt_trigger, LRNG_APT_WINDOW_SIZE); +} + +/* + * Insert a new entropy event into APT + * + * This function does is void as it does not decide about the fate of a time + * stamp. An APT failure can only happen at the same time of a stuck test + * failure. Thus, the stuck failure will already decide how the time stamp + * is handled. + * + * @health [in] Reference to health state + * @now_time [in] Time stamp to process + */ +static void lrng_apt_insert(struct lrng_health *health, + unsigned int now_time, enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + struct lrng_apt *apt = &es_state->apt; + + if (!lrng_sp80090b_health_requested()) + return; + + now_time &= LRNG_APT_WORD_MASK; + + /* Initialization of APT */ + if (!apt->apt_base_set) { + atomic_set(&apt->apt_base, now_time); + apt->apt_base_set = true; + return; + } + + if (now_time == (unsigned int)atomic_read(&apt->apt_base)) { + u32 apt_val = (u32)atomic_inc_return_relaxed(&apt->apt_count); + + if (apt_val >= CONFIG_LRNG_APT_CUTOFF) + lrng_sp80090b_failure(health, es); + } + + if (atomic_dec_and_test(&apt->apt_trigger)) { + lrng_apt_restart(apt); + lrng_apt_reset(apt, now_time); + lrng_sp80090b_startup(health, es); + } +} + +/*************************************************************************** + * Repetition Count Test + * + * The LRNG uses an enhanced version of the Repetition Count Test + * (RCT) specified in SP800-90B section 4.4.1. Instead of counting identical + * back-to-back values, the input to the RCT is the counting of the stuck + * values while filling the entropy pool. + * + * The RCT is applied with an alpha of 2^-30 compliant to FIPS 140-2 IG 9.8. + * + * During the counting operation, the LRNG always calculates the RCT + * cut-off value of C. If that value exceeds the allowed cut-off value, + * the LRNG will invalidate all entropy for the entropy pool which implies + * that no data can be extracted from the entropy pool unless new entropy + * is received. + ***************************************************************************/ + +/* + * Hot code path - Insert data for Repetition Count Test + * + * @health: Reference to health information + * @stuck: Decision of stuck test + */ +static void lrng_rct(struct lrng_health *health, enum lrng_internal_es es, + int stuck) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + struct lrng_rct *rct = &es_state->rct; + + if (!lrng_sp80090b_health_requested()) + return; + + if (stuck) { + u32 rct_count = atomic_add_return_relaxed(1, &rct->rct_count); + + /* + * The cutoff value is based on the following consideration: + * alpha = 2^-30 as recommended in FIPS 140-2 IG 9.8. + * In addition, we imply an entropy value H of 1 bit as this + * is the minimum entropy required to provide full entropy. + * + * Note, rct_count (which equals to value B in the + * pseudo code of SP800-90B section 4.4.1) starts with zero. + * Hence we need to subtract one from the cutoff value as + * calculated following SP800-90B. + */ + if (rct_count >= CONFIG_LRNG_RCT_CUTOFF) { + atomic_set(&rct->rct_count, 0); + + /* + * APT must start anew as we consider all previously + * recorded data to contain no entropy. + */ + lrng_apt_restart(&es_state->apt); + + lrng_sp80090b_failure(health, es); + } + } else { + atomic_set(&rct->rct_count, 0); + } +} + +/*************************************************************************** + * Stuck Test + * + * Checking the: + * 1st derivative of the event occurrence (time delta) + * 2nd derivative of the event occurrence (delta of time deltas) + * 3rd derivative of the event occurrence (delta of delta of time deltas) + * + * All values must always be non-zero. The stuck test is only valid disabled if + * high-resolution time stamps are identified after initialization. + ***************************************************************************/ + +static u32 lrng_delta(u32 prev, u32 next) +{ + /* + * Note that this (unsigned) subtraction does yield the correct value + * in the wraparound-case, i.e. when next < prev. + */ + return (next - prev); +} + +/* + * Hot code path + * + * @health: Reference to health information + * @now: Event time + * @return: 0 event occurrence not stuck (good time stamp) + * != 0 event occurrence stuck (reject time stamp) + */ +static int lrng_irq_stuck(enum lrng_internal_es es, u32 now_time) +{ + struct lrng_stuck_test *stuck = this_cpu_ptr(lrng_stuck_test_array); + u32 delta = lrng_delta(stuck[es].last_time, now_time); + u32 delta2 = lrng_delta(stuck[es].last_delta, delta); + u32 delta3 = lrng_delta(stuck[es].last_delta2, delta2); + + stuck[es].last_time = now_time; + stuck[es].last_delta = delta; + stuck[es].last_delta2 = delta2; + + if (!delta || !delta2 || !delta3) + return 1; + + return 0; +} + +/*************************************************************************** + * Health test interfaces + ***************************************************************************/ + +/* + * Disable all health tests + */ +void lrng_health_disable(void) +{ + struct lrng_health *health = &lrng_health; + + health->health_test_enabled = false; + + if (lrng_sp80090b_health_requested()) + pr_warn("SP800-90B compliance requested but the Linux RNG is NOT SP800-90B compliant\n"); +} + +/* + * Hot code path - Perform health test on time stamp received from an event + * + * @now_time Time stamp + */ +enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es) +{ + struct lrng_health *health = &lrng_health; + int stuck; + + if (!health->health_test_enabled) + return lrng_health_pass; + + lrng_apt_insert(health, now_time, es); + + stuck = lrng_irq_stuck(es, now_time); + lrng_rct(health, es, stuck); + if (stuck) { + /* SP800-90B disallows using a failing health test time stamp */ + return lrng_sp80090b_health_requested() ? + lrng_health_fail_drop : lrng_health_fail_use; + } + + return lrng_health_pass; +} diff --git a/drivers/char/lrng/lrng_health.h b/drivers/char/lrng/lrng_health.h new file mode 100644 index 000000000000..4f9f5033fc30 --- /dev/null +++ b/drivers/char/lrng/lrng_health.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_HEALTH_H +#define _LRNG_HEALTH_H + +#include "lrng_es_mgr.h" + +enum lrng_health_res { + lrng_health_pass, /* Health test passes on time stamp */ + lrng_health_fail_use, /* Time stamp unhealthy, but mix in */ + lrng_health_fail_drop /* Time stamp unhealthy, drop it */ +}; + +#ifdef CONFIG_LRNG_HEALTH_TESTS +bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es); +bool lrng_sp80090b_compliant(enum lrng_internal_es es); + +enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es); +void lrng_health_disable(void); +#else /* CONFIG_LRNG_HEALTH_TESTS */ +static inline bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) +{ + return true; +} + +static inline bool lrng_sp80090b_compliant(enum lrng_internal_es es) +{ + return false; +} + +static inline enum lrng_health_res +lrng_health_test(u32 now_time, enum lrng_internal_es es) +{ + return lrng_health_pass; +} +static inline void lrng_health_disable(void) { } +#endif /* CONFIG_LRNG_HEALTH_TESTS */ + +#endif /* _LRNG_HEALTH_H */ diff --git a/drivers/char/lrng/lrng_interface_aux.c b/drivers/char/lrng/lrng_interface_aux.c new file mode 100644 index 000000000000..a2236ff3b133 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_aux.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG auxiliary interfaces + * + * Copyright (C) 2022 Stephan Mueller + * Copyright (C) 2017 Jason A. Donenfeld . All + * Rights Reserved. + * Copyright (C) 2016 Jason Cooper + */ + +#include +#include +#include + +#include "lrng_es_mgr.h" +#include "lrng_interface_random_kernel.h" + +struct batched_entropy { + union { + u64 entropy_u64[LRNG_DRNG_BLOCKSIZE / sizeof(u64)]; + u32 entropy_u32[LRNG_DRNG_BLOCKSIZE / sizeof(u32)]; + }; + unsigned int position; + spinlock_t batch_lock; +}; + +/* + * Get a random word for internal kernel use only. The quality of the random + * number is as good as /dev/urandom, but there is no backtrack protection, + * with the goal of being quite fast and not depleting entropy. + */ +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { + .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), +}; + +u64 get_random_u64(void) +{ + u64 ret; + unsigned long flags; + struct batched_entropy *batch; + + lrng_debug_report_seedlevel("get_random_u64"); + + batch = raw_cpu_ptr(&batched_entropy_u64); + spin_lock_irqsave(&batch->batch_lock, flags); + if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { + lrng_get_random_bytes(batch->entropy_u64, LRNG_DRNG_BLOCKSIZE); + batch->position = 0; + } + ret = batch->entropy_u64[batch->position++]; + spin_unlock_irqrestore(&batch->batch_lock, flags); + return ret; +} +EXPORT_SYMBOL(get_random_u64); + +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { + .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), +}; + +u32 get_random_u32(void) +{ + u32 ret; + unsigned long flags; + struct batched_entropy *batch; + + lrng_debug_report_seedlevel("get_random_u32"); + + batch = raw_cpu_ptr(&batched_entropy_u32); + spin_lock_irqsave(&batch->batch_lock, flags); + if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { + lrng_get_random_bytes(batch->entropy_u32, LRNG_DRNG_BLOCKSIZE); + batch->position = 0; + } + ret = batch->entropy_u32[batch->position++]; + spin_unlock_irqrestore(&batch->batch_lock, flags); + return ret; +} +EXPORT_SYMBOL(get_random_u32); + +#ifdef CONFIG_SMP +/* + * This function is called when the CPU is coming up, with entry + * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP. + */ +int random_prepare_cpu(unsigned int cpu) +{ + /* + * When the cpu comes back online, immediately invalidate all batches, + * so that we serve fresh randomness. + */ + per_cpu_ptr(&batched_entropy_u32, cpu)->position = 0; + per_cpu_ptr(&batched_entropy_u64, cpu)->position = 0; + return 0; +} + +int random_online_cpu(unsigned int cpu) +{ + return 0; +} +#endif + +/* + * It's important to invalidate all potential batched entropy that might + * be stored before the crng is initialized, which we can do lazily by + * simply resetting the counter to zero so that it's re-extracted on the + * next usage. + */ +void invalidate_batched_entropy(void) +{ + int cpu; + unsigned long flags; + + for_each_possible_cpu(cpu) { + struct batched_entropy *batched_entropy; + + batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); + spin_lock_irqsave(&batched_entropy->batch_lock, flags); + batched_entropy->position = 0; + spin_unlock(&batched_entropy->batch_lock); + + batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); + spin_lock(&batched_entropy->batch_lock); + batched_entropy->position = 0; + spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); + } +} + +/* + * randomize_page - Generate a random, page aligned address + * @start: The smallest acceptable address the caller will take. + * @range: The size of the area, starting at @start, within which the + * random address must fall. + * + * If @start + @range would overflow, @range is capped. + * + * NOTE: Historical use of randomize_range, which this replaces, presumed that + * @start was already page aligned. We now align it regardless. + * + * Return: A page aligned address within [start, start + range). On error, + * @start is returned. + */ +unsigned long randomize_page(unsigned long start, unsigned long range) +{ + if (!PAGE_ALIGNED(start)) { + range -= PAGE_ALIGN(start) - start; + start = PAGE_ALIGN(start); + } + + if (start > ULONG_MAX - range) + range = ULONG_MAX - start; + + range >>= PAGE_SHIFT; + + if (range == 0) + return start; + + return start + (get_random_long() % range << PAGE_SHIFT); +} diff --git a/drivers/char/lrng/lrng_interface_dev.c b/drivers/char/lrng/lrng_interface_dev.c new file mode 100644 index 000000000000..e60060d402b3 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_dev.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG user space device file interface + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include + +#include "lrng_interface_dev_common.h" + +static const struct file_operations lrng_fops = { + .read = lrng_drng_read_block, + .write = lrng_drng_write, + .poll = lrng_random_poll, + .unlocked_ioctl = lrng_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = lrng_fasync, + .llseek = noop_llseek, +}; + +static struct miscdevice lrng_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "lrng", + .nodename = "lrng", + .fops = &lrng_fops, + .mode = 0666 +}; + +static int __init lrng_dev_if_mod_init(void) +{ + return misc_register(&lrng_miscdev); +} +device_initcall(lrng_dev_if_mod_init); diff --git a/drivers/char/lrng/lrng_interface_dev_common.c b/drivers/char/lrng/lrng_interface_dev_common.c new file mode 100644 index 000000000000..25f77250e65a --- /dev/null +++ b/drivers/char/lrng/lrng_interface_dev_common.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG User and kernel space interfaces + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_interface_dev_common.h" + +DECLARE_WAIT_QUEUE_HEAD(lrng_write_wait); +static struct fasync_struct *fasync; + +static bool lrng_seed_hw = true; /* Allow HW to provide seed */ +static bool lrng_seed_user = true; /* Allow user space to provide seed */ + +/********************************** Helper ***********************************/ + +static u32 lrng_get_aux_ent(void) +{ + return lrng_es[lrng_ext_es_aux]->curr_entropy(0); +} + +/* Is the DRNG seed level too low? */ +bool lrng_need_entropy(void) +{ + return (lrng_get_aux_ent() < lrng_write_wakeup_bits); +} + +void lrng_writer_wakeup(void) +{ + if (lrng_need_entropy() && wq_has_sleeper(&lrng_write_wait)) { + wake_up_interruptible(&lrng_write_wait); + kill_fasync(&fasync, SIGIO, POLL_OUT); + } +} + +void lrng_init_wakeup_dev(void) +{ + kill_fasync(&fasync, SIGIO, POLL_IN); +} + +/* External entropy provider is allowed to provide seed data */ +bool lrng_state_exseed_allow(enum lrng_external_noise_source source) +{ + if (source == lrng_noise_source_hw) + return lrng_seed_hw; + return lrng_seed_user; +} + +/* Enable / disable external entropy provider to furnish seed */ +void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) +{ + /* + * If the LRNG is not yet operational, allow all entropy sources + * to deliver data unconditionally to get fully seeded asap. + */ + if (!lrng_state_operational()) + return; + + if (source == lrng_noise_source_hw) + lrng_seed_hw = type; + else + lrng_seed_user = type; +} + +void lrng_state_exseed_allow_all(void) +{ + lrng_state_exseed_set(lrng_noise_source_hw, true); + lrng_state_exseed_set(lrng_noise_source_user, true); +} + +/************************ LRNG user output interfaces *************************/ + +ssize_t lrng_read_common(char __user *buf, size_t nbytes) +{ + ssize_t ret = 0; + u8 tmpbuf[LRNG_DRNG_BLOCKSIZE] __aligned(LRNG_KCAPI_ALIGN); + u8 *tmp_large = NULL, *tmp = tmpbuf; + u32 tmplen = sizeof(tmpbuf); + + if (nbytes == 0) + return 0; + + /* + * Satisfy large read requests -- as the common case are smaller + * request sizes, such as 16 or 32 bytes, avoid a kmalloc overhead for + * those by using the stack variable of tmpbuf. + */ + if (!CONFIG_BASE_SMALL && (nbytes > sizeof(tmpbuf))) { + tmplen = min_t(u32, nbytes, LRNG_DRNG_MAX_REQSIZE); + tmp_large = kmalloc(tmplen + LRNG_KCAPI_ALIGN, GFP_KERNEL); + if (!tmp_large) + tmplen = sizeof(tmpbuf); + else + tmp = PTR_ALIGN(tmp_large, LRNG_KCAPI_ALIGN); + } + + while (nbytes) { + u32 todo = min_t(u32, nbytes, tmplen); + int rc = 0; + + /* Reschedule if we received a large request. */ + if ((tmp_large) && need_resched()) { + if (signal_pending(current)) { + if (ret == 0) + ret = -ERESTARTSYS; + break; + } + schedule(); + } + + rc = lrng_drng_get_sleep(tmp, todo); + if (rc <= 0) { + if (rc < 0) + ret = rc; + break; + } + if (copy_to_user(buf, tmp, rc)) { + ret = -EFAULT; + break; + } + + nbytes -= rc; + buf += rc; + ret += rc; + } + + /* Wipe data just returned from memory */ + if (tmp_large) + kfree_sensitive(tmp_large); + else + memzero_explicit(tmpbuf, sizeof(tmpbuf)); + + return ret; +} + +ssize_t lrng_read_common_block(int nonblock, char __user *buf, size_t nbytes) +{ + int ret; + + if (nbytes == 0) + return 0; + + ret = lrng_drng_sleep_while_nonoperational(nonblock); + if (ret) + return ret; + + return lrng_read_common(buf, nbytes); +} + +ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, + loff_t *ppos) +{ + return lrng_read_common_block(file->f_flags & O_NONBLOCK, buf, nbytes); +} + +__poll_t lrng_random_poll(struct file *file, poll_table *wait) +{ + __poll_t mask; + + poll_wait(file, &lrng_init_wait, wait); + poll_wait(file, &lrng_write_wait, wait); + mask = 0; + if (lrng_state_operational()) + mask |= EPOLLIN | EPOLLRDNORM; + if (lrng_need_entropy() || + lrng_state_exseed_allow(lrng_noise_source_user)) { + lrng_state_exseed_set(lrng_noise_source_user, false); + mask |= EPOLLOUT | EPOLLWRNORM; + } + return mask; +} + +ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, + u32 entropy_bits) +{ + ssize_t ret = 0; + u8 buf[64] __aligned(LRNG_KCAPI_ALIGN); + const char __user *p = buffer; + u32 orig_entropy_bits = entropy_bits; + + if (!lrng_get_available()) { + ret = lrng_drng_initalize(); + if (!ret) + return ret; + } + + count = min_t(size_t, count, INT_MAX); + while (count > 0) { + size_t bytes = min_t(size_t, count, sizeof(buf)); + u32 ent = min_t(u32, bytes<<3, entropy_bits); + + if (copy_from_user(&buf, p, bytes)) + return -EFAULT; + /* Inject data into entropy pool */ + lrng_pool_insert_aux(buf, bytes, ent); + + count -= bytes; + p += bytes; + ret += bytes; + entropy_bits -= ent; + + cond_resched(); + } + + /* Force reseed of DRNG during next data request. */ + if (!orig_entropy_bits) + lrng_drng_force_reseed(); + + return ret; +} + +ssize_t lrng_drng_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + return lrng_drng_write_common(buffer, count, 0); +} + +long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg) +{ + u32 digestsize_bits; + int size, ent_count_bits; + int __user *p = (int __user *)arg; + + switch (cmd) { + case RNDGETENTCNT: + ent_count_bits = lrng_avail_entropy(); + if (put_user(ent_count_bits, p)) + return -EFAULT; + return 0; + case RNDADDTOENTCNT: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (get_user(ent_count_bits, p)) + return -EFAULT; + ent_count_bits = (int)lrng_get_aux_ent() + ent_count_bits; + if (ent_count_bits < 0) + ent_count_bits = 0; + digestsize_bits = lrng_get_digestsize(); + if (ent_count_bits > digestsize_bits) + ent_count_bits = digestsize_bits; + lrng_pool_set_entropy(ent_count_bits); + return 0; + case RNDADDENTROPY: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (get_user(ent_count_bits, p++)) + return -EFAULT; + if (ent_count_bits < 0) + return -EINVAL; + if (get_user(size, p++)) + return -EFAULT; + if (size < 0) + return -EINVAL; + /* there cannot be more entropy than data */ + ent_count_bits = min(ent_count_bits, size<<3); + return lrng_drng_write_common((const char __user *)p, size, + ent_count_bits); + case RNDZAPENTCNT: + case RNDCLEARPOOL: + /* Clear the entropy pool counter. */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + lrng_pool_set_entropy(0); + return 0; + case RNDRESEEDCRNG: + /* + * We leave the capability check here since it is present + * in the upstream's RNG implementation. Yet, user space + * can trigger a reseed as easy as writing into /dev/random + * or /dev/urandom where no privilege is needed. + */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* Force a reseed of all DRNGs */ + lrng_drng_force_reseed(); + return 0; + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(lrng_ioctl); + +int lrng_fasync(int fd, struct file *filp, int on) +{ + return fasync_helper(fd, filp, on, &fasync); +} diff --git a/drivers/char/lrng/lrng_interface_dev_common.h b/drivers/char/lrng/lrng_interface_dev_common.h new file mode 100644 index 000000000000..87cf504647d3 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_dev_common.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_INTERFACE_DEV_COMMON_H +#define _LRNG_INTERFACE_DEV_COMMON_H + +#include +#include + +/******************* Upstream functions hooked into the LRNG ******************/ +enum lrng_external_noise_source { + lrng_noise_source_hw, + lrng_noise_source_user +}; + +#ifdef CONFIG_LRNG_COMMON_DEV_IF +void lrng_writer_wakeup(void); +void lrng_init_wakeup_dev(void); +void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type); +void lrng_state_exseed_allow_all(void); +#else /* CONFIG_LRNG_COMMON_DEV_IF */ +static inline void lrng_writer_wakeup(void) { } +static inline void lrng_init_wakeup_dev(void) { } +static inline void +lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) { } +static inline void lrng_state_exseed_allow_all(void) { } +#endif /* CONFIG_LRNG_COMMON_DEV_IF */ + +/****** Downstream service functions to actual interface implementations ******/ + +bool lrng_state_exseed_allow(enum lrng_external_noise_source source); +int lrng_fasync(int fd, struct file *filp, int on); +long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg); +ssize_t lrng_drng_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos); +ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, + u32 entropy_bits); +__poll_t lrng_random_poll(struct file *file, poll_table *wait); +ssize_t lrng_read_common_block(int nonblock, char __user *buf, size_t nbytes); +ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, + loff_t *ppos); +ssize_t lrng_read_common(char __user *buf, size_t nbytes); +bool lrng_need_entropy(void); + +extern struct wait_queue_head lrng_write_wait; +extern struct wait_queue_head lrng_init_wait; + +#endif /* _LRNG_INTERFACE_DEV_COMMON_H */ diff --git a/drivers/char/lrng/lrng_interface_hwrand.c b/drivers/char/lrng/lrng_interface_hwrand.c new file mode 100644 index 000000000000..e841eea13348 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_hwrand.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG interface with the HW-Random framework + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include + +static int lrng_hwrand_if_random(struct hwrng *rng, void *buf, size_t max, + bool wait) +{ + /* + * lrng_get_random_bytes_full not called as we cannot block. + * + * Note: We should either adjust .quality below depending on + * rng_is_initialized() or block here, but neither is not supported by + * the hw_rand framework. + */ + lrng_get_random_bytes(buf, max); + return (int)max; +} + +static struct hwrng lrng_hwrand = { + .name = "lrng", + .init = NULL, + .cleanup = NULL, + .read = lrng_hwrand_if_random, + + /* + * We set .quality only in case the LRNG does not provide the common + * interfaces or does not use the legacy RNG as entropy source. This + * shall avoid that the LRNG automatically spawns the hw_rand + * framework's hwrng kernel thread to feed data into + * add_hwgenerator_randomness. When the LRNG implements the common + * interfaces, this function feeds the data directly into the LRNG. + * If the LRNG uses the legacy RNG as entropy source, + * add_hwgenerator_randomness is implemented by the legacy RNG, but + * still eventually feeds the data into the LRNG. We should avoid such + * circular loops. + * + * We can specify full entropy here, because the LRNG is designed + * to provide full entropy. + */ +#if !defined(CONFIG_LRNG_RANDOM_IF) && \ + !defined(CONFIG_LRNG_KERNEL_RNG) + .quality = 1024, +#endif +}; + +static int __init lrng_hwrand_if_mod_init(void) +{ + return hwrng_register(&lrng_hwrand); +} + +static void __exit lrng_hwrand_if_mod_exit(void) +{ + hwrng_unregister(&lrng_hwrand); +} + +module_init(lrng_hwrand_if_mod_init); +module_exit(lrng_hwrand_if_mod_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager HW-Random Interface"); diff --git a/drivers/char/lrng/lrng_interface_kcapi.c b/drivers/char/lrng/lrng_interface_kcapi.c new file mode 100644 index 000000000000..4cb511f8088e --- /dev/null +++ b/drivers/char/lrng/lrng_interface_kcapi.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG interface with the RNG framework of the kernel crypto API + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" + +static int lrng_kcapi_if_init(struct crypto_tfm *tfm) +{ + return 0; +} + +static void lrng_kcapi_if_cleanup(struct crypto_tfm *tfm) { } + +static int lrng_kcapi_if_reseed(const u8 *src, unsigned int slen) +{ + int ret; + + if (!slen) + return 0; + + /* Insert caller-provided data without crediting entropy */ + ret = lrng_pool_insert_aux((u8 *)src, slen, 0); + if (ret) + return ret; + + /* Make sure the new data is immediately available to DRNG */ + lrng_drng_force_reseed(); + + return 0; +} + +static int lrng_kcapi_if_random(struct crypto_rng *tfm, + const u8 *src, unsigned int slen, + u8 *rdata, unsigned int dlen) +{ + int ret = lrng_kcapi_if_reseed(src, slen); + + if (!ret) + lrng_get_random_bytes_full(rdata, dlen); + + return ret; +} + +static int lrng_kcapi_if_reset(struct crypto_rng *tfm, + const u8 *seed, unsigned int slen) +{ + return lrng_kcapi_if_reseed(seed, slen); +} + +static struct rng_alg lrng_alg = { + .generate = lrng_kcapi_if_random, + .seed = lrng_kcapi_if_reset, + .seedsize = 0, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "lrng", + .cra_priority = 500, + .cra_ctxsize = 0, + .cra_module = THIS_MODULE, + .cra_init = lrng_kcapi_if_init, + .cra_exit = lrng_kcapi_if_cleanup, + + } +}; + +#ifdef CONFIG_LRNG_DRNG_ATOMIC +static int lrng_kcapi_if_random_atomic(struct crypto_rng *tfm, + const u8 *src, unsigned int slen, + u8 *rdata, unsigned int dlen) +{ + int ret = lrng_kcapi_if_reseed(src, slen); + + if (!ret) + lrng_get_random_bytes(rdata, dlen); + + return ret; +} + +static struct rng_alg lrng_alg_atomic = { + .generate = lrng_kcapi_if_random_atomic, + .seed = lrng_kcapi_if_reset, + .seedsize = 0, + .base = { + .cra_name = "lrng_atomic", + .cra_driver_name = "lrng_atomic", + .cra_priority = 100, + .cra_ctxsize = 0, + .cra_module = THIS_MODULE, + .cra_init = lrng_kcapi_if_init, + .cra_exit = lrng_kcapi_if_cleanup, + + } +}; +#endif /* CONFIG_LRNG_DRNG_ATOMIC */ + +static int __init lrng_kcapi_if_mod_init(void) +{ + return +#ifdef CONFIG_LRNG_DRNG_ATOMIC + crypto_register_rng(&lrng_alg_atomic) ?: +#endif + crypto_register_rng(&lrng_alg); +} + +static void __exit lrng_kcapi_if_mod_exit(void) +{ + crypto_unregister_rng(&lrng_alg); +#ifdef CONFIG_LRNG_DRNG_ATOMIC + crypto_unregister_rng(&lrng_alg_atomic); +#endif +} + +module_init(lrng_kcapi_if_mod_init); +module_exit(lrng_kcapi_if_mod_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager kernel crypto API RNG framework interface"); +MODULE_ALIAS_CRYPTO("lrng"); +MODULE_ALIAS_CRYPTO("lrng_atomic"); +MODULE_ALIAS_CRYPTO("stdrng"); diff --git a/drivers/char/lrng/lrng_interface_random_kernel.c b/drivers/char/lrng/lrng_interface_random_kernel.c new file mode 100644 index 000000000000..0efbc3a463dd --- /dev/null +++ b/drivers/char/lrng/lrng_interface_random_kernel.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Kernel space interfaces API/ABI compliant to linux/random.h + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#include "lrng_es_aux.h" +#include "lrng_es_irq.h" +#include "lrng_es_mgr.h" +#include "lrng_interface_dev_common.h" +#include "lrng_interface_random_kernel.h" + +static LIST_HEAD(lrng_ready_list); +static DEFINE_SPINLOCK(lrng_ready_list_lock); + +/********************************** Helper ***********************************/ + +int __init rand_initialize(void) +{ + return lrng_rand_initialize(); +} +early_initcall(rand_initialize); + +bool lrng_ready_list_has_sleeper(void) +{ + return list_empty(&lrng_ready_list); +} + +/* + * lrng_process_ready_list() - Ping all kernel internal callers waiting until + * the DRNG is completely initialized to inform that the DRNG reached that + * seed level. + * + * When the SP800-90B testing is enabled, the ping only happens if the SP800-90B + * startup health tests are completed. This implies that kernel internal + * callers always have an SP800-90B compliant noise source when being + * pinged. + */ +void lrng_process_ready_list(void) +{ + unsigned long flags; + struct random_ready_callback *rdy, *tmp; + + if (!lrng_state_operational()) + return; + + spin_lock_irqsave(&lrng_ready_list_lock, flags); + list_for_each_entry_safe(rdy, tmp, &lrng_ready_list, list) { + struct module *owner = rdy->owner; + + list_del_init(&rdy->list); + rdy->func(rdy); + module_put(owner); + } + spin_unlock_irqrestore(&lrng_ready_list_lock, flags); +} + +/************************ LRNG kernel input interfaces ************************/ + +/* + * add_hwgenerator_randomness() - Interface for in-kernel drivers of true + * hardware RNGs. + * + * Those devices may produce endless random bits and will be throttled + * when our pool is full. + * + * @buffer: buffer holding the entropic data from HW noise sources to be used to + * insert into entropy pool. + * @count: length of buffer + * @entropy_bits: amount of entropy in buffer (value is in bits) + */ +void add_hwgenerator_randomness(const char *buffer, size_t count, + size_t entropy_bits) +{ + /* + * Suspend writing if we are fully loaded with entropy. + * We'll be woken up again once below lrng_write_wakeup_thresh, + * or when the calling thread is about to terminate. + */ + wait_event_interruptible(lrng_write_wait, + lrng_need_entropy() || + lrng_state_exseed_allow(lrng_noise_source_hw) || + kthread_should_stop()); + lrng_state_exseed_set(lrng_noise_source_hw, false); + lrng_pool_insert_aux(buffer, count, entropy_bits); +} +EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); + +/* + * add_bootloader_randomness() - Handle random seed passed by bootloader. + * + * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise + * it would be regarded as device data. + * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. + * + * @buf: buffer holding the entropic data from HW noise sources to be used to + * insert into entropy pool. + * @size: length of buffer + */ +void add_bootloader_randomness(const void *buf, unsigned int size) +{ + lrng_pool_insert_aux(buf, size, + IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER) ? + size * 8 : 0); +} +EXPORT_SYMBOL_GPL(add_bootloader_randomness); + +/* + * Callback for HID layer -- use the HID event values to stir the entropy pool + */ +void add_input_randomness(unsigned int type, unsigned int code, + unsigned int value) +{ + static unsigned char last_value; + + /* ignore autorepeat and the like */ + if (value == last_value) + return; + + last_value = value; + + lrng_irq_array_add_u32((type << 4) ^ code ^ (code >> 4) ^ value); +} +EXPORT_SYMBOL_GPL(add_input_randomness); + +/* + * add_device_randomness() - Add device- or boot-specific data to the entropy + * pool to help initialize it. + * + * None of this adds any entropy; it is meant to avoid the problem of + * the entropy pool having similar initial state across largely + * identical devices. + * + * @buf: buffer holding the entropic data from HW noise sources to be used to + * insert into entropy pool. + * @size: length of buffer + */ +void add_device_randomness(const void *buf, unsigned int size) +{ + lrng_pool_insert_aux((u8 *)buf, size, 0); +} +EXPORT_SYMBOL(add_device_randomness); + +#ifdef CONFIG_BLOCK +void rand_initialize_disk(struct gendisk *disk) { } +void add_disk_randomness(struct gendisk *disk) { } +EXPORT_SYMBOL(add_disk_randomness); +#endif + +#ifndef CONFIG_LRNG_IRQ +void add_interrupt_randomness(int irq, int irq_flg) { } +EXPORT_SYMBOL(add_interrupt_randomness); +#endif + +/* + * unregister_random_ready_notifier() - Delete a previously registered readiness + * callback function. + * + * @rdy: callback definition that was registered initially + */ +void del_random_ready_callback(struct random_ready_callback *rdy) +{ + unsigned long flags; + struct module *owner = NULL; + + spin_lock_irqsave(&lrng_ready_list_lock, flags); + if (!list_empty(&rdy->list)) { + list_del_init(&rdy->list); + owner = rdy->owner; + } + spin_unlock_irqrestore(&lrng_ready_list_lock, flags); + + module_put(owner); +} +EXPORT_SYMBOL(del_random_ready_callback); + +/* + * add_random_ready_callback() - Add a callback function that will be + * invoked when the DRNG is fully initialized and seeded. + * + * @rdy: callback definition to be invoked when the LRNG is seeded + * + * Return: + * * 0 if callback is successfully added + * * -EALREADY if pool is already initialised (callback not called) + * * -ENOENT if module for callback is not alive + */ +int add_random_ready_callback(struct random_ready_callback *rdy) +{ + struct module *owner; + unsigned long flags; + int err = -EALREADY; + + if (likely(lrng_state_operational())) + return err; + + owner = rdy->owner; + if (!try_module_get(owner)) + return -ENOENT; + + spin_lock_irqsave(&lrng_ready_list_lock, flags); + if (lrng_state_operational()) + goto out; + + owner = NULL; + + list_add(&rdy->list, &lrng_ready_list); + err = 0; + +out: + spin_unlock_irqrestore(&lrng_ready_list_lock, flags); + + module_put(owner); + + return err; +} +EXPORT_SYMBOL(add_random_ready_callback); + +/*********************** LRNG kernel output interfaces ************************/ + +/* + * get_random_bytes() - Provider of cryptographic strong random numbers for + * kernel-internal usage. + * + * This function is appropriate for all in-kernel use cases. However, + * it will always use the ChaCha20 DRNG. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +void get_random_bytes(void *buf, int nbytes) +{ + lrng_get_random_bytes(buf, nbytes); +} +EXPORT_SYMBOL(get_random_bytes); + +/* + * wait_for_random_bytes() - Wait for the LRNG to be seeded and thus + * guaranteed to supply cryptographically secure random numbers. + * + * This applies to: the /dev/urandom device, the get_random_bytes function, + * and the get_random_{u32,u64,int,long} family of functions. Using any of + * these functions without first calling this function forfeits the guarantee + * of security. + * + * Return: + * * 0 if the LRNG has been seeded. + * * -ERESTARTSYS if the function was interrupted by a signal. + */ +int wait_for_random_bytes(void) +{ + return lrng_drng_sleep_while_non_min_seeded(); +} +EXPORT_SYMBOL(wait_for_random_bytes); + +/* + * get_random_bytes_arch() - This function will use the architecture-specific + * hardware random number generator if it is available. + * + * The arch-specific hw RNG will almost certainly be faster than what we can + * do in software, but it is impossible to verify that it is implemented + * securely (as opposed, to, say, the AES encryption of a sequence number using + * a key known by the NSA). So it's useful if we need the speed, but only if + * we're willing to trust the hardware manufacturer not to have put in a back + * door. + * + * @buf: buffer allocated by caller to store the random data in + * @nbytes: length of outbuf + * + * Return: number of bytes filled in. + */ +int __must_check get_random_bytes_arch(void *buf, int nbytes) +{ + u8 *p = buf; + + while (nbytes) { + unsigned long v; + int chunk = min_t(int, nbytes, sizeof(unsigned long)); + + if (!arch_get_random_long(&v)) + break; + + memcpy(p, &v, chunk); + p += chunk; + nbytes -= chunk; + } + + if (nbytes) + lrng_get_random_bytes(p, nbytes); + + return nbytes; +} +EXPORT_SYMBOL(get_random_bytes_arch); + +/* + * Returns whether or not the LRNG has been seeded. + * + * Returns: true if the urandom pool has been seeded. + * false if the urandom pool has not been seeded. + */ +bool rng_is_initialized(void) +{ + return lrng_state_operational(); +} +EXPORT_SYMBOL(rng_is_initialized); diff --git a/drivers/char/lrng/lrng_interface_random_kernel.h b/drivers/char/lrng/lrng_interface_random_kernel.h new file mode 100644 index 000000000000..14c9baf216d3 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_random_kernel.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_INTERFACE_RANDOM_H +#define _LRNG_INTERFACE_RANDOM_H + +#ifdef CONFIG_LRNG_RANDOM_IF +void lrng_process_ready_list(void); +bool lrng_ready_list_has_sleeper(void); +void invalidate_batched_entropy(void); +#else /* CONFIG_LRNG_RANDOM_IF */ +static inline bool lrng_ready_list_has_sleeper(void) { return false; } +static inline void lrng_process_ready_list(void) { } +static inline void invalidate_batched_entropy(void) { } +#endif /* CONFIG_LRNG_RANDOM_IF */ + +#endif /* _LRNG_INTERFACE_RANDOM_H */ diff --git a/drivers/char/lrng/lrng_interface_random_user.c b/drivers/char/lrng/lrng_interface_random_user.c new file mode 100644 index 000000000000..7e45f41b7a4d --- /dev/null +++ b/drivers/char/lrng/lrng_interface_random_user.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Common user space interfaces compliant to random(4), random(7) and + * getrandom(2) man pages. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_es_mgr.h" +#include "lrng_interface_dev_common.h" + +static ssize_t lrng_drng_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + if (!lrng_state_min_seeded()) + pr_notice_ratelimited("%s - use of insufficiently seeded DRNG (%zu bytes read)\n", + current->comm, nbytes); + else if (!lrng_state_operational()) + pr_debug_ratelimited("%s - use of not fully seeded DRNG (%zu bytes read)\n", + current->comm, nbytes); + + return lrng_read_common(buf, nbytes); +} + +const struct file_operations random_fops = { + .read = lrng_drng_read_block, + .write = lrng_drng_write, + .poll = lrng_random_poll, + .unlocked_ioctl = lrng_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = lrng_fasync, + .llseek = noop_llseek, +}; + +const struct file_operations urandom_fops = { + .read = lrng_drng_read, + .write = lrng_drng_write, + .unlocked_ioctl = lrng_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = lrng_fasync, + .llseek = noop_llseek, +}; + +SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, + unsigned int, flags) +{ + if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE)) + return -EINVAL; + + /* + * Requesting insecure and blocking randomness at the same time makes + * no sense. + */ + if ((flags & + (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM)) + return -EINVAL; + + if (count > INT_MAX) + count = INT_MAX; + + if (flags & GRND_INSECURE) + return lrng_drng_read(NULL, buf, count, NULL); + + return lrng_read_common_block(flags & GRND_NONBLOCK, buf, count); +} diff --git a/drivers/char/lrng/lrng_internal.h b/drivers/char/lrng/lrng_internal.h new file mode 100644 index 000000000000..69e4ffb0cf57 --- /dev/null +++ b/drivers/char/lrng/lrng_internal.h @@ -0,0 +1,489 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2018 - 2021, Stephan Mueller + */ + +#ifndef _LRNG_INTERNAL_H +#define _LRNG_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include + +#define arch_get_random_seed_long_early arch_get_random_seed_long +#define arch_get_random_long_early arch_get_random_long + +#define kfree_sensitive kzfree + +/*************************** General LRNG parameter ***************************/ + +/* Security strength of LRNG -- this must match DRNG security strength */ +#define LRNG_DRNG_SECURITY_STRENGTH_BYTES 32 +#define LRNG_DRNG_SECURITY_STRENGTH_BITS (LRNG_DRNG_SECURITY_STRENGTH_BYTES * 8) +#define LRNG_DRNG_BLOCKSIZE 64 /* Maximum of DRNG block sizes */ +#define LRNG_DRNG_INIT_SEED_SIZE_BITS (LRNG_DRNG_SECURITY_STRENGTH_BITS + \ + CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS) +#define LRNG_DRNG_INIT_SEED_SIZE_BYTES (LRNG_DRNG_INIT_SEED_SIZE_BITS >> 3) + +/* + * SP800-90A defines a maximum request size of 1<<16 bytes. The given value is + * considered a safer margin. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_MAX_REQSIZE (1<<12) + +/* + * SP800-90A defines a maximum number of requests between reseeds of 2^48. + * The given value is considered a much safer margin, balancing requests for + * frequent reseeds with the need to conserve entropy. This value MUST NOT be + * larger than INT_MAX because it is used in an atomic_t. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_RESEED_THRESH (1<<20) + +/* + * Maximum DRNG generation operations without reseed having full entropy + * This value defines the absolute maximum value of DRNG generation operations + * without a reseed holding full entropy. LRNG_DRNG_RESEED_THRESH is the + * threshold when a new reseed is attempted. But it is possible that this fails + * to deliver full entropy. In this case the DRNG will continue to provide data + * even though it was not reseeded with full entropy. To avoid in the extreme + * case that no reseed is performed for too long, this threshold is enforced. + * If that absolute low value is reached, the LRNG is marked as not operational. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_MAX_WITHOUT_RESEED (1<<30) + +/* + * Min required seed entropy is 128 bits covering the minimum entropy + * requirement of SP800-131A and the German BSI's TR02102. + * + * This value is allowed to be changed. + */ +#define LRNG_FULL_SEED_ENTROPY_BITS LRNG_DRNG_SECURITY_STRENGTH_BITS +#define LRNG_MIN_SEED_ENTROPY_BITS 128 +#define LRNG_INIT_ENTROPY_BITS 32 + +/* + * Wakeup value + * + * This value is allowed to be changed but must not be larger than the + * digest size of the hash operation used update the aux_pool. + */ +#ifdef CONFIG_CRYPTO_LIB_SHA256 +# define LRNG_ATOMIC_DIGEST_SIZE SHA256_DIGEST_SIZE +#else +# define LRNG_ATOMIC_DIGEST_SIZE SHA1_DIGEST_SIZE +#endif +#define LRNG_WRITE_WAKEUP_ENTROPY LRNG_ATOMIC_DIGEST_SIZE + +/* + * If the switching support is configured, we must provide support up to + * the largest digest size. Without switching support, we know it is only + * the built-in digest size. + */ +#ifdef CONFIG_LRNG_DRNG_SWITCH +# define LRNG_MAX_DIGESTSIZE 64 +#else +# define LRNG_MAX_DIGESTSIZE LRNG_ATOMIC_DIGEST_SIZE +#endif + +/* + * Oversampling factor of IRQ events to obtain + * LRNG_DRNG_SECURITY_STRENGTH_BYTES. This factor is used when a + * high-resolution time stamp is not available. In this case, jiffies and + * register contents are used to fill the entropy pool. These noise sources + * are much less entropic than the high-resolution timer. The entropy content + * is the entropy content assumed with LRNG_IRQ_ENTROPY_BITS divided by + * LRNG_IRQ_OVERSAMPLING_FACTOR. + * + * This value is allowed to be changed. + */ +#define LRNG_IRQ_OVERSAMPLING_FACTOR 10 + +/* Alignmask that is intended to be identical to CRYPTO_MINALIGN */ +#define LRNG_KCAPI_ALIGN ARCH_KMALLOC_MINALIGN + +/* + * This definition must provide a buffer that is equal to SHASH_DESC_ON_STACK + * as it will be casted into a struct shash_desc. + */ +#define LRNG_POOL_SIZE (sizeof(struct shash_desc) + HASH_MAX_DESCSIZE) + +/************************ Default DRNG implementation *************************/ + +extern struct chacha20_state chacha20; +extern const struct lrng_crypto_cb lrng_cc20_crypto_cb; +void lrng_cc20_init_state(struct chacha20_state *state); + +/********************************** /proc *************************************/ + +#ifdef CONFIG_SYSCTL +void lrng_pool_inc_numa_node(void); +void lrng_proc_update_max_write_thresh(u32 new_digestsize); +#else +static inline void lrng_pool_inc_numa_node(void) { } +static inline void lrng_proc_update_max_write_thresh(u32 new_digestsize) { } +#endif + +/****************************** LRNG interfaces *******************************/ + +extern u32 lrng_write_wakeup_bits; +extern int lrng_drng_reseed_max_time; + +void lrng_writer_wakeup(void); +void lrng_init_wakeup(void); +void lrng_debug_report_seedlevel(const char *name); +void lrng_process_ready_list(void); + +/* External interface to use of the switchable DRBG inside the kernel */ +void get_random_bytes_full(void *buf, int nbytes); + +/************************* Jitter RNG Entropy Source **************************/ + +#ifdef CONFIG_LRNG_JENT +u32 lrng_get_jent(u8 *outbuf, u32 requested_bits); +u32 lrng_jent_entropylevel(u32 requested_bits); +void lrng_jent_es_state(unsigned char *buf, size_t buflen); +#else /* CONFIG_LRNG_JENT */ +static inline u32 lrng_get_jent(u8 *outbuf, u32 requested_bits) { return 0; } +static inline u32 lrng_jent_entropylevel(u32 requested_bits) { return 0; } +static inline void lrng_jent_es_state(unsigned char *buf, size_t buflen) { } +#endif /* CONFIG_LRNG_JENT */ + +/************************** CPU-based Entropy Source **************************/ + +static inline u32 lrng_fast_noise_entropylevel(u32 ent_bits, u32 requested_bits) +{ + /* Obtain entropy statement */ + ent_bits = ent_bits * requested_bits / LRNG_DRNG_SECURITY_STRENGTH_BITS; + /* Cap entropy to buffer size in bits */ + ent_bits = min_t(u32, ent_bits, requested_bits); + return ent_bits; +} + +#ifdef CONFIG_LRNG_CPU +u32 lrng_get_arch(u8 *outbuf, u32 requested_bits); +u32 lrng_archrandom_entropylevel(u32 requested_bits); +void lrng_arch_es_state(unsigned char *buf, size_t buflen); +#else /* CONFIG_LRNG_CPU */ +static inline u32 lrng_get_arch(u8 *outbuf, u32 requested_bits) { return 0; } +static inline u32 lrng_archrandom_entropylevel(u32 requested_bits) { return 0; } +static inline void lrng_arch_es_state(unsigned char *buf, size_t buflen) { } +#endif /* CONFIG_LRNG_CPU */ + +/************************** Interrupt Entropy Source **************************/ + +#ifdef CONFIG_LRNG_IRQ +void lrng_pcpu_reset(void); +u32 lrng_pcpu_avail_pool_size(void); +u32 lrng_pcpu_avail_entropy(void); +int lrng_pcpu_switch_hash(int node, + const struct lrng_crypto_cb *new_cb, void *new_hash, + const struct lrng_crypto_cb *old_cb); +u32 lrng_pcpu_pool_hash(u8 *outbuf, u32 requested_bits, bool fully_seeded); +void lrng_pcpu_array_add_u32(u32 data); +u32 lrng_gcd_analyze(u32 *history, size_t nelem); +void lrng_irq_es_state(unsigned char *buf, size_t buflen); +#else /* CONFIG_LRNG_IRQ */ +static inline void lrng_pcpu_reset(void) { } +static inline u32 lrng_pcpu_avail_pool_size(void) { return 0; } +static inline u32 lrng_pcpu_avail_entropy(void) { return 0; } +static inline int lrng_pcpu_switch_hash(int node, + const struct lrng_crypto_cb *new_cb, void *new_hash, + const struct lrng_crypto_cb *old_cb) +{ + return 0; +} +static inline u32 lrng_pcpu_pool_hash(u8 *outbuf, u32 requested_bits, + bool fully_seeded) +{ + return 0; +} +static inline void lrng_pcpu_array_add_u32(u32 data) { } +static inline void lrng_irq_es_state(unsigned char *buf, size_t buflen) { } +#endif /* CONFIG_LRNG_IRQ */ + +/****************************** DRNG processing *******************************/ + +/* DRNG state handle */ +struct lrng_drng { + void *drng; /* DRNG handle */ + void *hash; /* Hash handle */ + const struct lrng_crypto_cb *crypto_cb; /* Crypto callbacks */ + atomic_t requests; /* Number of DRNG requests */ + atomic_t requests_since_fully_seeded; /* Number DRNG requests since + last fully seeded */ + unsigned long last_seeded; /* Last time it was seeded */ + bool fully_seeded; /* Is DRNG fully seeded? */ + bool force_reseed; /* Force a reseed */ + + /* Lock write operations on DRNG state, DRNG replacement of crypto_cb */ + struct mutex lock; + spinlock_t spin_lock; + /* Lock *hash replacement - always take before DRNG lock */ + rwlock_t hash_lock; +}; + +extern struct mutex lrng_crypto_cb_update; + +struct lrng_drng *lrng_drng_init_instance(void); +struct lrng_drng *lrng_drng_atomic_instance(void); + +static __always_inline bool lrng_drng_is_atomic(struct lrng_drng *drng) +{ + return (drng->drng == lrng_drng_atomic_instance()->drng); +} + +/* Lock the DRNG */ +static __always_inline void lrng_drng_lock(struct lrng_drng *drng, + unsigned long *flags) + __acquires(&drng->spin_lock) +{ + /* Use spin lock in case the atomic DRNG context is used */ + if (lrng_drng_is_atomic(drng)) { + spin_lock_irqsave(&drng->spin_lock, *flags); + + /* + * In case a lock transition happened while we were spinning, + * catch this case and use the new lock type. + */ + if (!lrng_drng_is_atomic(drng)) { + spin_unlock_irqrestore(&drng->spin_lock, *flags); + __acquire(&drng->spin_lock); + mutex_lock(&drng->lock); + } + } else { + __acquire(&drng->spin_lock); + mutex_lock(&drng->lock); + } +} + +/* Unlock the DRNG */ +static __always_inline void lrng_drng_unlock(struct lrng_drng *drng, + unsigned long *flags) + __releases(&drng->spin_lock) +{ + if (lrng_drng_is_atomic(drng)) { + spin_unlock_irqrestore(&drng->spin_lock, *flags); + } else { + mutex_unlock(&drng->lock); + __release(&drng->spin_lock); + } +} + +void lrng_reset(void); +void lrng_drngs_init_cc20(bool force_seed); +bool lrng_sp80090c_compliant(void); + +static inline u32 lrng_compress_osr(void) +{ + return lrng_sp80090c_compliant() ? CONFIG_LRNG_OVERSAMPLE_ES_BITS : 0; +} + +static inline u32 lrng_reduce_by_osr(u32 entropy_bits) +{ + u32 osr_bits = lrng_compress_osr(); + return (entropy_bits >= osr_bits) ? (entropy_bits - osr_bits) : 0; +} + +bool lrng_get_available(void); +void lrng_set_available(void); +void lrng_drng_reset(struct lrng_drng *drng); +int lrng_drng_get_atomic(u8 *outbuf, u32 outbuflen); +int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen); +void lrng_drng_force_reseed(void); +void lrng_drng_seed_work(struct work_struct *dummy); + +#ifdef CONFIG_NUMA +struct lrng_drng **lrng_drng_instances(void); +void lrng_drngs_numa_alloc(void); +#else /* CONFIG_NUMA */ +static inline struct lrng_drng **lrng_drng_instances(void) { return NULL; } +static inline void lrng_drngs_numa_alloc(void) { return; } +#endif /* CONFIG_NUMA */ + +/************************* Entropy sources management *************************/ + +enum lrng_external_noise_source { + lrng_noise_source_hw, + lrng_noise_source_user +}; + +void lrng_set_entropy_thresh(u32 new); +u32 lrng_avail_entropy(void); +void lrng_reset_state(void); + +bool lrng_state_exseed_allow(enum lrng_external_noise_source source); +void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type); +bool lrng_state_min_seeded(void); +bool lrng_state_fully_seeded(void); +bool lrng_state_operational(void); + +int lrng_pool_trylock(void); +void lrng_pool_unlock(void); +void lrng_pool_all_numa_nodes_seeded(bool set); +void lrng_pool_add_entropy(void); + +struct entropy_buf { + u8 a[LRNG_DRNG_INIT_SEED_SIZE_BYTES]; + u8 b[LRNG_DRNG_INIT_SEED_SIZE_BYTES]; + u8 c[LRNG_DRNG_INIT_SEED_SIZE_BYTES]; + u8 d[LRNG_DRNG_INIT_SEED_SIZE_BYTES]; + u32 now, a_bits, b_bits, c_bits, d_bits; +}; + +bool lrng_fully_seeded(bool fully_seeded, struct entropy_buf *eb); +void lrng_unset_fully_seeded(struct lrng_drng *drng); +void lrng_fill_seed_buffer(struct entropy_buf *entropy_buf, u32 requested_bits); +void lrng_init_ops(struct entropy_buf *eb); + +/*********************** Auxiliary Pool Entropy Source ************************/ + +u32 lrng_avail_aux_entropy(void); +void lrng_aux_es_state(unsigned char *buf, size_t buflen); +u32 lrng_get_digestsize(void); +void lrng_pool_set_entropy(u32 entropy_bits); +int lrng_aux_switch_hash(const struct lrng_crypto_cb *new_cb, void *new_hash, + const struct lrng_crypto_cb *old_cb); +int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits); +void lrng_get_backtrack_aux(struct entropy_buf *entropy_buf, + u32 requested_bits); + +/* Obtain the security strength of the LRNG in bits */ +static inline u32 lrng_security_strength(void) +{ + /* + * We use a hash to read the entropy in the entropy pool. According to + * SP800-90B table 1, the entropy can be at most the digest size. + * Considering this together with the last sentence in section 3.1.5.1.2 + * the security strength of a (approved) hash is equal to its output + * size. On the other hand the entropy cannot be larger than the + * security strength of the used DRBG. + */ + return min_t(u32, LRNG_FULL_SEED_ENTROPY_BITS, lrng_get_digestsize()); +} + +static inline u32 lrng_get_seed_entropy_osr(bool fully_seeded) +{ + u32 requested_bits = lrng_security_strength(); + + /* Apply oversampling during initialization according to SP800-90C */ + if (lrng_sp80090c_compliant() && !fully_seeded) + requested_bits += CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS; + return requested_bits; +} + +/************************** Health Test linking code **************************/ + +enum lrng_health_res { + lrng_health_pass, /* Health test passes on time stamp */ + lrng_health_fail_use, /* Time stamp unhealthy, but mix in */ + lrng_health_fail_drop /* Time stamp unhealthy, drop it */ +}; + +#ifdef CONFIG_LRNG_HEALTH_TESTS +bool lrng_sp80090b_startup_complete(void); +bool lrng_sp80090b_compliant(void); + +enum lrng_health_res lrng_health_test(u32 now_time); +void lrng_health_disable(void); + +#else /* CONFIG_LRNG_HEALTH_TESTS */ +static inline bool lrng_sp80090b_startup_complete(void) { return true; } +static inline bool lrng_sp80090b_compliant(void) { return false; } + +static inline enum lrng_health_res +lrng_health_test(u32 now_time) { return lrng_health_pass; } +static inline void lrng_health_disable(void) { } +#endif /* CONFIG_LRNG_HEALTH_TESTS */ + +/****************************** Helper code ***********************************/ + +static inline u32 atomic_read_u32(atomic_t *v) +{ + return (u32)atomic_read(v); +} + +/******************** Crypto Primitive Switching Support **********************/ + +#ifdef CONFIG_LRNG_DRNG_SWITCH +static inline void lrng_hash_lock(struct lrng_drng *drng, unsigned long *flags) +{ + read_lock_irqsave(&drng->hash_lock, *flags); +} + +static inline void lrng_hash_unlock(struct lrng_drng *drng, unsigned long flags) +{ + read_unlock_irqrestore(&drng->hash_lock, flags); +} +#else /* CONFIG_LRNG_DRNG_SWITCH */ +static inline void lrng_hash_lock(struct lrng_drng *drng, unsigned long *flags) +{ } + +static inline void lrng_hash_unlock(struct lrng_drng *drng, unsigned long flags) +{ } +#endif /* CONFIG_LRNG_DRNG_SWITCH */ + +/*************************** Auxiliary functions ******************************/ + +void invalidate_batched_entropy(void); + +/***************************** Testing code ***********************************/ + +#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY +bool lrng_raw_hires_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ +static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY +bool lrng_raw_jiffies_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ +static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY +bool lrng_raw_irq_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ +static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_IRQFLAGS_ENTROPY +bool lrng_raw_irqflags_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_IRQFLAGS_ENTROPY */ +static inline bool lrng_raw_irqflags_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_IRQFLAGS_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY +bool lrng_raw_retip_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ +static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY +bool lrng_raw_regs_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ +static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_ARRAY +bool lrng_raw_array_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_ARRAY */ +static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_ARRAY */ + +#ifdef CONFIG_LRNG_IRQ_PERF +bool lrng_perf_time(u32 start); +#else /* CONFIG_LRNG_IRQ_PERF */ +static inline bool lrng_perf_time(u32 start) { return false; } +#endif /*CONFIG_LRNG_IRQ_PERF */ + +#endif /* _LRNG_INTERNAL_H */ diff --git a/drivers/char/lrng/lrng_numa.c b/drivers/char/lrng/lrng_numa.c new file mode 100644 index 000000000000..d74dd8df2843 --- /dev/null +++ b/drivers/char/lrng/lrng_numa.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG NUMA support + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_irq.h" +#include "lrng_es_mgr.h" +#include "lrng_numa.h" +#include "lrng_proc.h" + +static struct lrng_drng **lrng_drng __read_mostly = NULL; + +struct lrng_drng **lrng_drng_instances(void) +{ + /* counterpart to cmpxchg_release in _lrng_drngs_numa_alloc */ + return READ_ONCE(lrng_drng); +} + +/* Allocate the data structures for the per-NUMA node DRNGs */ +static void _lrng_drngs_numa_alloc(struct work_struct *work) +{ + struct lrng_drng **drngs; + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + u32 node; + bool init_drng_used = false; + + mutex_lock(&lrng_crypto_cb_update); + + /* per-NUMA-node DRNGs are already present */ + if (lrng_drng) + goto unlock; + + /* Make sure the initial DRNG is initialized and its drng_cb is set */ + if (lrng_drng_initalize()) + goto err; + + drngs = kcalloc(nr_node_ids, sizeof(void *), GFP_KERNEL|__GFP_NOFAIL); + for_each_online_node(node) { + struct lrng_drng *drng; + + if (!init_drng_used) { + drngs[node] = lrng_drng_init; + init_drng_used = true; + continue; + } + + drng = kmalloc_node(sizeof(struct lrng_drng), + GFP_KERNEL|__GFP_NOFAIL, node); + memset(drng, 0, sizeof(lrng_drng)); + + if (lrng_drng_alloc_common(drng, lrng_drng_init->drng_cb)) { + kfree(drng); + goto err; + } + + drng->hash_cb = lrng_drng_init->hash_cb; + drng->hash = lrng_drng_init->hash_cb->hash_alloc(); + if (IS_ERR(drng->hash)) { + lrng_drng_init->drng_cb->drng_dealloc(drng->drng); + kfree(drng); + goto err; + } + + mutex_init(&drng->lock); + rwlock_init(&drng->hash_lock); + + /* + * No reseeding of NUMA DRNGs from previous DRNGs as this + * would complicate the code. Let it simply reseed. + */ + drngs[node] = drng; + + lrng_pool_inc_numa_node(); + pr_info("DRNG and entropy pool read hash for NUMA node %d allocated\n", + node); + } + + /* counterpart to READ_ONCE in lrng_drng_instances */ + if (!cmpxchg_release(&lrng_drng, NULL, drngs)) { + lrng_pool_all_numa_nodes_seeded(false); + goto unlock; + } + +err: + for_each_online_node(node) { + struct lrng_drng *drng = drngs[node]; + + if (drng == lrng_drng_init) + continue; + + if (drng) { + drng->hash_cb->hash_dealloc(drng->hash); + drng->drng_cb->drng_dealloc(drng->drng); + kfree(drng); + } + } + kfree(drngs); + +unlock: + mutex_unlock(&lrng_crypto_cb_update); +} + +static DECLARE_WORK(lrng_drngs_numa_alloc_work, _lrng_drngs_numa_alloc); + +static void lrng_drngs_numa_alloc(void) +{ + schedule_work(&lrng_drngs_numa_alloc_work); +} + +static int __init lrng_numa_init(void) +{ + lrng_drngs_numa_alloc(); + return 0; +} + +late_initcall(lrng_numa_init); diff --git a/drivers/char/lrng/lrng_numa.h b/drivers/char/lrng/lrng_numa.h new file mode 100644 index 000000000000..dc8dff9816ee --- /dev/null +++ b/drivers/char/lrng/lrng_numa.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_NUMA_H +#define _LRNG_NUMA_H + +#ifdef CONFIG_NUMA +struct lrng_drng **lrng_drng_instances(void); +#else /* CONFIG_NUMA */ +static inline struct lrng_drng **lrng_drng_instances(void) { return NULL; } +#endif /* CONFIG_NUMA */ + +#endif /* _LRNG_NUMA_H */ diff --git a/drivers/char/lrng/lrng_proc.c b/drivers/char/lrng/lrng_proc.c new file mode 100644 index 000000000000..6e3e0699a840 --- /dev/null +++ b/drivers/char/lrng/lrng_proc.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG proc interfaces + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_proc.h" + +/* Number of online DRNGs */ +static u32 numa_drngs = 1; + +void lrng_pool_inc_numa_node(void) +{ + numa_drngs++; +} + +static int lrng_proc_type_show(struct seq_file *m, void *v) +{ + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + unsigned char buf[250]; + u32 i; + + mutex_lock(&lrng_drng_init->lock); + snprintf(buf, sizeof(buf), + "DRNG name: %s\n" + "LRNG security strength in bits: %d\n" + "Number of DRNG instances: %u\n" + "Standards compliance: %s\n" + "LRNG minimally seeded: %s\n" + "LRNG fully seeded: %s\n", + lrng_drng_init->drng_cb->drng_name(), + lrng_security_strength(), + numa_drngs, + lrng_sp80090c_compliant() ? "SP800-90C " : "", + lrng_state_min_seeded() ? "true" : "false", + lrng_state_fully_seeded() ? "true" : "false"); + seq_write(m, buf, strlen(buf)); + + for_each_lrng_es(i) { + snprintf(buf, sizeof(buf), + "Entropy Source %u properties:\n" + " Name: %s\n", + i, lrng_es[i]->name); + seq_write(m, buf, strlen(buf)); + + buf[0] = '\0'; + lrng_es[i]->state(buf, sizeof(buf)); + seq_write(m, buf, strlen(buf)); + } + + mutex_unlock(&lrng_drng_init->lock); + + return 0; +} + +static int __init lrng_proc_type_init(void) +{ + proc_create_single("lrng_type", 0444, NULL, &lrng_proc_type_show); + return 0; +} + +module_init(lrng_proc_type_init); diff --git a/drivers/char/lrng/lrng_proc.h b/drivers/char/lrng/lrng_proc.h new file mode 100644 index 000000000000..c653274f1954 --- /dev/null +++ b/drivers/char/lrng/lrng_proc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_PROC_H +#define _LRNG_PROC_H + +#ifdef CONFIG_SYSCTL +void lrng_pool_inc_numa_node(void); +#else +static inline void lrng_pool_inc_numa_node(void) { } +#endif + +#endif /* _LRNG_PROC_H */ diff --git a/drivers/char/lrng/lrng_selftest.c b/drivers/char/lrng/lrng_selftest.c new file mode 100644 index 000000000000..15f1e4a2a719 --- /dev/null +++ b/drivers/char/lrng/lrng_selftest.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG power-on and on-demand self-test + * + * Copyright (C) 2022, Stephan Mueller + */ + +/* + * In addition to the self-tests below, the following LRNG components + * are covered with self-tests during regular operation: + * + * * power-on self-test: SP800-90A DRBG provided by the Linux kernel crypto API + * * power-on self-test: PRNG provided by the Linux kernel crypto API + * * runtime test: Raw noise source data testing including SP800-90B compliant + * tests when enabling CONFIG_LRNG_HEALTH_TESTS + * + * Additional developer tests present with LRNG code: + * * SP800-90B APT and RCT test enforcement validation when enabling + * CONFIG_LRNG_APT_BROKEN or CONFIG_LRNG_RCT_BROKEN. + * * Collection of raw entropy from the interrupt noise source when enabling + * CONFIG_LRNG_TESTING and pulling the data from the kernel with the provided + * interface. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "lrng_drng_chacha20.h" +#include "lrng_sha.h" + +#define LRNG_SELFTEST_PASSED 0 +#define LRNG_SEFLTEST_ERROR_TIME (1 << 0) +#define LRNG_SEFLTEST_ERROR_CHACHA20 (1 << 1) +#define LRNG_SEFLTEST_ERROR_HASH (1 << 2) +#define LRNG_SEFLTEST_ERROR_GCD (1 << 3) +#define LRNG_SELFTEST_NOT_EXECUTED 0xffffffff + +#ifdef CONFIG_LRNG_TIMER_COMMON + +#include "lrng_es_timer_common.h" + +static u32 lrng_data_selftest_ptr = 0; +static u32 lrng_data_selftest[LRNG_DATA_ARRAY_SIZE]; + +static void lrng_data_process_selftest_insert(u32 time) +{ + u32 ptr = lrng_data_selftest_ptr++ & LRNG_DATA_WORD_MASK; + unsigned int array = lrng_data_idx2array(ptr); + unsigned int slot = lrng_data_idx2slot(ptr); + + /* zeroization of slot to ensure the following OR adds the data */ + lrng_data_selftest[array] &= + ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, + slot)); + lrng_data_selftest[array] |= + lrng_data_slot_val(time & LRNG_DATA_SLOTSIZE_MASK, slot); +} + +static void lrng_data_process_selftest_u32(u32 data) +{ + u32 pre_ptr, ptr, mask; + unsigned int pre_array; + + /* Increment pointer by number of slots taken for input value */ + lrng_data_selftest_ptr += LRNG_DATA_SLOTS_PER_UINT; + + /* ptr to current unit */ + ptr = lrng_data_selftest_ptr; + + lrng_data_split_u32(&ptr, &pre_ptr, &mask); + + /* MSB of data go into previous unit */ + pre_array = lrng_data_idx2array(pre_ptr); + /* zeroization of slot to ensure the following OR adds the data */ + lrng_data_selftest[pre_array] &= ~(0xffffffff & ~mask); + lrng_data_selftest[pre_array] |= data & ~mask; + + /* LSB of data go into current unit */ + lrng_data_selftest[lrng_data_idx2array(ptr)] = data & mask; +} + +static unsigned int lrng_data_process_selftest(void) +{ + u32 time; + u32 idx_zero_compare = (0 << 0) | (1 << 8) | (2 << 16) | (3 << 24); + u32 idx_one_compare = (4 << 0) | (5 << 8) | (6 << 16) | (7 << 24); + u32 idx_last_compare = + (((LRNG_DATA_NUM_VALUES - 4) & LRNG_DATA_SLOTSIZE_MASK) << 0) | + (((LRNG_DATA_NUM_VALUES - 3) & LRNG_DATA_SLOTSIZE_MASK) << 8) | + (((LRNG_DATA_NUM_VALUES - 2) & LRNG_DATA_SLOTSIZE_MASK) << 16) | + (((LRNG_DATA_NUM_VALUES - 1) & LRNG_DATA_SLOTSIZE_MASK) << 24); + + (void)idx_one_compare; + + /* "poison" the array to verify the operation of the zeroization */ + lrng_data_selftest[0] = 0xffffffff; + lrng_data_selftest[1] = 0xffffffff; + + lrng_data_process_selftest_insert(0); + /* + * Note, when using lrng_data_process_u32() on unaligned ptr, + * the first slots will go into next word, and the last slots go + * into the previous word. + */ + lrng_data_process_selftest_u32((4 << 0) | (1 << 8) | (2 << 16) | + (3 << 24)); + lrng_data_process_selftest_insert(5); + lrng_data_process_selftest_insert(6); + lrng_data_process_selftest_insert(7); + + if ((lrng_data_selftest[0] != idx_zero_compare) || + (lrng_data_selftest[1] != idx_one_compare)) + goto err; + + /* Reset for next test */ + lrng_data_selftest[0] = 0; + lrng_data_selftest[1] = 0; + lrng_data_selftest_ptr = 0; + + for (time = 0; time < LRNG_DATA_NUM_VALUES; time++) + lrng_data_process_selftest_insert(time); + + if ((lrng_data_selftest[0] != idx_zero_compare) || + (lrng_data_selftest[1] != idx_one_compare) || + (lrng_data_selftest[LRNG_DATA_ARRAY_SIZE - 1] != idx_last_compare)) + goto err; + + return LRNG_SELFTEST_PASSED; + +err: + pr_err("LRNG data array self-test FAILED\n"); + return LRNG_SEFLTEST_ERROR_TIME; +} + +static unsigned int lrng_gcd_selftest(void) +{ + u32 history[10]; + unsigned int i; + +#define LRNG_GCD_SELFTEST 3 + for (i = 0; i < ARRAY_SIZE(history); i++) + history[i] = i * LRNG_GCD_SELFTEST; + + if (lrng_gcd_analyze(history, ARRAY_SIZE(history)) == LRNG_GCD_SELFTEST) + return LRNG_SELFTEST_PASSED; + + pr_err("LRNG GCD self-test FAILED\n"); + return LRNG_SEFLTEST_ERROR_GCD; +} + +#else /* CONFIG_LRNG_TIMER_COMMON */ + +static unsigned int lrng_data_process_selftest(void) +{ + return LRNG_SELFTEST_PASSED; +} + +static unsigned int lrng_gcd_selftest(void) +{ + return LRNG_SELFTEST_PASSED; +} + +#endif /* CONFIG_LRNG_TIMER_COMMON */ + +/* The test vectors are taken from crypto/testmgr.h */ +static unsigned int lrng_hash_selftest(void) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; + static const u8 lrng_hash_selftest_result[] = +#ifdef CONFIG_CRYPTO_LIB_SHA256 + { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, + 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, + 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad }; +#else /* CONFIG_CRYPTO_LIB_SHA256 */ + { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e, + 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d }; +#endif /* CONFIG_CRYPTO_LIB_SHA256 */ + static const u8 hash_input[] = { 0x61, 0x62, 0x63 }; /* "abc" */ + u8 digest[sizeof(lrng_hash_selftest_result)] __aligned(sizeof(u32)); + + if (sizeof(digest) != hash_cb->hash_digestsize(NULL)) + return LRNG_SEFLTEST_ERROR_HASH; + + if (!hash_cb->hash_init(shash, NULL) && + !hash_cb->hash_update(shash, hash_input, + sizeof(hash_input)) && + !hash_cb->hash_final(shash, digest) && + !memcmp(digest, lrng_hash_selftest_result, sizeof(digest))) + return 0; + + pr_err("LRNG %s Hash self-test FAILED\n", hash_cb->hash_name()); + return LRNG_SEFLTEST_ERROR_HASH; +} + +#ifdef CONFIG_LRNG_DRNG_CHACHA20 + +static void lrng_selftest_bswap32(u32 *ptr, u32 words) +{ + u32 i; + + /* Byte-swap data which is an LE representation */ + for (i = 0; i < words; i++) { + __le32 *p = (__le32 *)ptr; + + *p = cpu_to_le32(*ptr); + ptr++; + } +} + +/* + * The test vectors were generated using the ChaCha20 DRNG from + * https://www.chronox.de/chacha20.html + */ +static unsigned int lrng_chacha20_drng_selftest(void) +{ + const struct lrng_drng_cb *drng_cb = &lrng_cc20_drng_cb; + u8 seed[CHACHA_KEY_SIZE * 2] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + }; + struct chacha20_block chacha20; + int ret; + u8 outbuf[CHACHA_KEY_SIZE * 2] __aligned(sizeof(u32)); + + /* + * Expected result when ChaCha20 DRNG state is zero: + * * constants are set to "expand 32-byte k" + * * remaining state is 0 + * and pulling one half ChaCha20 DRNG block. + */ + static const u8 expected_halfblock[CHACHA_KEY_SIZE] = { + 0x76, 0xb8, 0xe0, 0xad, 0xa0, 0xf1, 0x3d, 0x90, + 0x40, 0x5d, 0x6a, 0xe5, 0x53, 0x86, 0xbd, 0x28, + 0xbd, 0xd2, 0x19, 0xb8, 0xa0, 0x8d, 0xed, 0x1a, + 0xa8, 0x36, 0xef, 0xcc, 0x8b, 0x77, 0x0d, 0xc7 }; + + /* + * Expected result when ChaCha20 DRNG state is zero: + * * constants are set to "expand 32-byte k" + * * remaining state is 0 + * followed by a reseed with two keyblocks + * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + * 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + * 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + * 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + * 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f + * and pulling one ChaCha20 DRNG block. + */ + static const u8 expected_oneblock[CHACHA_KEY_SIZE * 2] = { + 0xe3, 0xb0, 0x8a, 0xcc, 0x34, 0xc3, 0x17, 0x0e, + 0xc3, 0xd8, 0xc3, 0x40, 0xe7, 0x73, 0xe9, 0x0d, + 0xd1, 0x62, 0xa3, 0x5d, 0x7d, 0xf2, 0xf1, 0x4a, + 0x24, 0x42, 0xb7, 0x1e, 0xb0, 0x05, 0x17, 0x07, + 0xb9, 0x35, 0x10, 0x69, 0x8b, 0x46, 0xfb, 0x51, + 0xe9, 0x91, 0x3f, 0x46, 0xf2, 0x4d, 0xea, 0xd0, + 0x81, 0xc1, 0x1b, 0xa9, 0x5d, 0x52, 0x91, 0x5f, + 0xcd, 0xdc, 0xc6, 0xd6, 0xc3, 0x7c, 0x50, 0x23 }; + + /* + * Expected result when ChaCha20 DRNG state is zero: + * * constants are set to "expand 32-byte k" + * * remaining state is 0 + * followed by a reseed with one key block plus one byte + * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + * 0x20 + * and pulling less than one ChaCha20 DRNG block. + */ + static const u8 expected_block_nonalinged[CHACHA_KEY_SIZE + 4] = { + 0x9c, 0xfc, 0x5e, 0x31, 0x21, 0x62, 0x11, 0x85, + 0xd3, 0x77, 0xd3, 0x69, 0x0f, 0xa8, 0x16, 0x55, + 0xb4, 0x4c, 0xf6, 0x52, 0xf3, 0xa8, 0x37, 0x99, + 0x38, 0x76, 0xa0, 0x66, 0xec, 0xbb, 0xce, 0xa9, + 0x9c, 0x95, 0xa1, 0xfd }; + + BUILD_BUG_ON(sizeof(seed) % sizeof(u32)); + + memset(&chacha20, 0, sizeof(chacha20)); + lrng_cc20_init_rfc7539(&chacha20); + lrng_selftest_bswap32((u32 *)seed, sizeof(seed) / sizeof(u32)); + + /* Generate with zero state */ + ret = drng_cb->drng_generate(&chacha20, outbuf, + sizeof(expected_halfblock)); + if (ret != sizeof(expected_halfblock)) + goto err; + if (memcmp(outbuf, expected_halfblock, sizeof(expected_halfblock))) + goto err; + + /* Clear state of DRNG */ + memset(&chacha20.key.u[0], 0, 48); + + /* Reseed with 2 key blocks */ + ret = drng_cb->drng_seed(&chacha20, seed, sizeof(expected_oneblock)); + if (ret < 0) + goto err; + ret = drng_cb->drng_generate(&chacha20, outbuf, + sizeof(expected_oneblock)); + if (ret != sizeof(expected_oneblock)) + goto err; + if (memcmp(outbuf, expected_oneblock, sizeof(expected_oneblock))) + goto err; + + /* Clear state of DRNG */ + memset(&chacha20.key.u[0], 0, 48); + + /* Reseed with 1 key block and one byte */ + ret = drng_cb->drng_seed(&chacha20, seed, + sizeof(expected_block_nonalinged)); + if (ret < 0) + goto err; + ret = drng_cb->drng_generate(&chacha20, outbuf, + sizeof(expected_block_nonalinged)); + if (ret != sizeof(expected_block_nonalinged)) + goto err; + if (memcmp(outbuf, expected_block_nonalinged, + sizeof(expected_block_nonalinged))) + goto err; + + return LRNG_SELFTEST_PASSED; + +err: + pr_err("LRNG ChaCha20 DRNG self-test FAILED\n"); + return LRNG_SEFLTEST_ERROR_CHACHA20; +} + +#else /* CONFIG_LRNG_DRNG_CHACHA20 */ + +static unsigned int lrng_chacha20_drng_selftest(void) +{ + return LRNG_SELFTEST_PASSED; +} + +#endif /* CONFIG_LRNG_DRNG_CHACHA20 */ + +static unsigned int lrng_selftest_status = LRNG_SELFTEST_NOT_EXECUTED; + +static int lrng_selftest(void) +{ + unsigned int ret = lrng_data_process_selftest(); + + ret |= lrng_chacha20_drng_selftest(); + ret |= lrng_hash_selftest(); + ret |= lrng_gcd_selftest(); + + if (ret) { + if (IS_ENABLED(CONFIG_LRNG_SELFTEST_PANIC)) + panic("LRNG self-tests failed: %u\n", ret); + } else { + pr_info("LRNG self-tests passed\n"); + } + + lrng_selftest_status = ret; + + if (lrng_selftest_status) + return -EFAULT; + return 0; +} + +#ifdef CONFIG_SYSFS +/* Re-perform self-test when any value is written to the sysfs file. */ +static int lrng_selftest_sysfs_set(const char *val, + const struct kernel_param *kp) +{ + return lrng_selftest(); +} + +static const struct kernel_param_ops lrng_selftest_sysfs = { + .set = lrng_selftest_sysfs_set, + .get = param_get_uint, +}; +module_param_cb(selftest_status, &lrng_selftest_sysfs, &lrng_selftest_status, + 0644); +#endif /* CONFIG_SYSFS */ + +static int __init lrng_selftest_init(void) +{ + return lrng_selftest(); +} + +module_init(lrng_selftest_init); diff --git a/drivers/char/lrng/lrng_sha.h b/drivers/char/lrng/lrng_sha.h new file mode 100644 index 000000000000..d2f134f54773 --- /dev/null +++ b/drivers/char/lrng/lrng_sha.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG SHA definition usable in atomic contexts right from the start of the + * kernel. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_SHA_H +#define _LRNG_SHA_H + +extern const struct lrng_hash_cb lrng_sha_hash_cb; + +#endif /* _LRNG_SHA_H */ diff --git a/drivers/char/lrng/lrng_sha1.c b/drivers/char/lrng/lrng_sha1.c new file mode 100644 index 000000000000..0e4909b0e64a --- /dev/null +++ b/drivers/char/lrng/lrng_sha1.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the SHA-1 implementation that can be used + * without the kernel crypto API available including during early boot and in + * atomic contexts. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "lrng_sha.h" + +/* + * If the SHA-256 support is not compiled, we fall back to SHA-1 that is always + * compiled and present in the kernel. + */ +static u32 lrng_sha1_hash_digestsize(void *hash) +{ + return SHA1_DIGEST_SIZE; +} + +static void lrng_sha1_block_fn(struct sha1_state *sctx, const u8 *src, + int blocks) +{ + u32 temp[SHA1_WORKSPACE_WORDS]; + + while (blocks--) { + sha_transform(sctx->state, src, temp); + src += SHA1_BLOCK_SIZE; + } + memzero_explicit(temp, sizeof(temp)); +} + +static int lrng_sha1_hash_init(struct shash_desc *shash, void *hash) +{ + /* + * We do not need a TFM - we only need sufficient space for + * struct sha1_state on the stack. + */ + sha1_base_init(shash); + return 0; +} + +static int lrng_sha1_hash_update(struct shash_desc *shash, + const u8 *inbuf, u32 inbuflen) +{ + return sha1_base_do_update(shash, inbuf, inbuflen, lrng_sha1_block_fn); +} + +static int lrng_sha1_hash_final(struct shash_desc *shash, u8 *digest) +{ + return sha1_base_do_finalize(shash, lrng_sha1_block_fn) ?: + sha1_base_finish(shash, digest); +} + +static const char *lrng_sha1_hash_name(void) +{ + return "SHA-1"; +} + +static void lrng_sha1_hash_desc_zero(struct shash_desc *shash) +{ + memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha1_state)); +} + +static void *lrng_sha1_hash_alloc(void) +{ + pr_info("Hash %s allocated\n", lrng_sha1_hash_name()); + return NULL; +} + +static void lrng_sha1_hash_dealloc(void *hash) { } + +const struct lrng_hash_cb lrng_sha_hash_cb = { + .hash_name = lrng_sha1_hash_name, + .hash_alloc = lrng_sha1_hash_alloc, + .hash_dealloc = lrng_sha1_hash_dealloc, + .hash_digestsize = lrng_sha1_hash_digestsize, + .hash_init = lrng_sha1_hash_init, + .hash_update = lrng_sha1_hash_update, + .hash_final = lrng_sha1_hash_final, + .hash_desc_zero = lrng_sha1_hash_desc_zero, +}; diff --git a/drivers/char/lrng/lrng_sha256.c b/drivers/char/lrng/lrng_sha256.c new file mode 100644 index 000000000000..6a694b1ccf0d --- /dev/null +++ b/drivers/char/lrng/lrng_sha256.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the SHA-256 implementation that can be used + * without the kernel crypto API available including during early boot and in + * atomic contexts. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_sha.h" + +static u32 lrng_sha256_hash_digestsize(void *hash) +{ + return SHA256_DIGEST_SIZE; +} + +static int lrng_sha256_hash_init(struct shash_desc *shash, void *hash) +{ + /* + * We do not need a TFM - we only need sufficient space for + * struct sha256_state on the stack. + */ + sha256_init(shash_desc_ctx(shash)); + return 0; +} + +static int lrng_sha256_hash_update(struct shash_desc *shash, + const u8 *inbuf, u32 inbuflen) +{ + sha256_update(shash_desc_ctx(shash), inbuf, inbuflen); + return 0; +} + +static int lrng_sha256_hash_final(struct shash_desc *shash, u8 *digest) +{ + sha256_final(shash_desc_ctx(shash), digest); + return 0; +} + +static const char *lrng_sha256_hash_name(void) +{ + return "SHA-256"; +} + +static void lrng_sha256_hash_desc_zero(struct shash_desc *shash) +{ + memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha256_state)); +} + +static void *lrng_sha256_hash_alloc(void) +{ + pr_info("Hash %s allocated\n", lrng_sha256_hash_name()); + return NULL; +} + +static void lrng_sha256_hash_dealloc(void *hash) { } + +const struct lrng_hash_cb lrng_sha_hash_cb = { + .hash_name = lrng_sha256_hash_name, + .hash_alloc = lrng_sha256_hash_alloc, + .hash_dealloc = lrng_sha256_hash_dealloc, + .hash_digestsize = lrng_sha256_hash_digestsize, + .hash_init = lrng_sha256_hash_init, + .hash_update = lrng_sha256_hash_update, + .hash_final = lrng_sha256_hash_final, + .hash_desc_zero = lrng_sha256_hash_desc_zero, +}; diff --git a/drivers/char/lrng/lrng_switch.c b/drivers/char/lrng/lrng_switch.c new file mode 100644 index 000000000000..5ffc0286a4a4 --- /dev/null +++ b/drivers/char/lrng/lrng_switch.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG DRNG switching support + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include + +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_numa.h" + +static int __maybe_unused +lrng_hash_switch(struct lrng_drng *drng_store, const void *cb, int node) +{ + const struct lrng_hash_cb *new_cb = (const struct lrng_hash_cb *)cb; + const struct lrng_hash_cb *old_cb = drng_store->hash_cb; + unsigned long flags; + u32 i; + void *new_hash = new_cb->hash_alloc(); + void *old_hash = drng_store->hash; + int ret; + + if (IS_ERR(new_hash)) { + pr_warn("could not allocate new LRNG pool hash (%ld)\n", + PTR_ERR(new_hash)); + return PTR_ERR(new_hash); + } + + if (new_cb->hash_digestsize(new_hash) > LRNG_MAX_DIGESTSIZE) { + pr_warn("digest size of newly requested hash too large\n"); + new_cb->hash_dealloc(new_hash); + return -EINVAL; + } + + write_lock_irqsave(&drng_store->hash_lock, flags); + + /* Trigger the switch for each entropy source */ + for_each_lrng_es(i) { + if (!lrng_es[i]->switch_hash) + continue; + ret = lrng_es[i]->switch_hash(drng_store, node, new_cb, + new_hash, old_cb); + if (ret) { + u32 j; + + /* Revert all already executed operations */ + for (j = 0; j < i; j++) { + if (!lrng_es[j]->switch_hash) + continue; + WARN_ON(lrng_es[j]->switch_hash(drng_store, + node, old_cb, + old_hash, + new_cb)); + } + goto err; + } + } + + drng_store->hash = new_hash; + drng_store->hash_cb = new_cb; + old_cb->hash_dealloc(old_hash); + pr_info("Conditioning function allocated for DRNG for NUMA node %d\n", + node); + +err: + write_unlock_irqrestore(&drng_store->hash_lock, flags); + return ret; +} + +static int __maybe_unused +lrng_drng_switch(struct lrng_drng *drng_store, const void *cb, int node) +{ + const struct lrng_drng_cb *new_cb = (const struct lrng_drng_cb *)cb; + const struct lrng_drng_cb *old_cb = drng_store->drng_cb; + int ret; + u8 seed[LRNG_DRNG_SECURITY_STRENGTH_BYTES]; + void *new_drng = new_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); + void *old_drng = drng_store->drng; + u32 current_security_strength; + bool reset_drng = !lrng_get_available(); + + if (IS_ERR(new_drng)) { + pr_warn("could not allocate new DRNG for NUMA node %d (%ld)\n", + node, PTR_ERR(new_drng)); + return PTR_ERR(new_drng); + } + + current_security_strength = lrng_security_strength(); + mutex_lock(&drng_store->lock); + + /* + * Pull from existing DRNG to seed new DRNG regardless of seed status + * of old DRNG -- the entropy state for the DRNG is left unchanged which + * implies that als the new DRNG is reseeded when deemed necessary. This + * seeding of the new DRNG shall only ensure that the new DRNG has the + * same entropy as the old DRNG. + */ + ret = old_cb->drng_generate(old_drng, seed, sizeof(seed)); + mutex_unlock(&drng_store->lock); + + if (ret < 0) { + reset_drng = true; + pr_warn("getting random data from DRNG failed for NUMA node %d (%d)\n", + node, ret); + } else { + /* seed new DRNG with data */ + ret = new_cb->drng_seed(new_drng, seed, ret); + memzero_explicit(seed, sizeof(seed)); + if (ret < 0) { + reset_drng = true; + pr_warn("seeding of new DRNG failed for NUMA node %d (%d)\n", + node, ret); + } else { + pr_debug("seeded new DRNG of NUMA node %d instance from old DRNG instance\n", + node); + } + } + + mutex_lock(&drng_store->lock); + + if (reset_drng) + lrng_drng_reset(drng_store); + + drng_store->drng = new_drng; + drng_store->drng_cb = new_cb; + + /* Reseed if previous LRNG security strength was insufficient */ + if (current_security_strength < lrng_security_strength()) + drng_store->force_reseed = true; + + /* Force oversampling seeding as we initialize DRNG */ + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) + lrng_unset_fully_seeded(drng_store); + + if (lrng_state_min_seeded()) + lrng_set_entropy_thresh(lrng_get_seed_entropy_osr( + drng_store->fully_seeded)); + + old_cb->drng_dealloc(old_drng); + + pr_info("DRNG of NUMA node %d switched\n", node); + + mutex_unlock(&drng_store->lock); + return ret; +} + +/* + * Switch the existing DRNG and hash instances with new using the new crypto + * callbacks. The caller must hold the lrng_crypto_cb_update lock. + */ +static int lrng_switch(const void *cb, + int(*switcher)(struct lrng_drng *drng_store, + const void *cb, int node)) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + int ret = 0; + + if (lrng_drng) { + u32 node; + + for_each_online_node(node) { + if (lrng_drng[node]) + ret = switcher(lrng_drng[node], cb, node); + } + } else { + ret = switcher(lrng_drng_init, cb, 0); + } + + return 0; +} + +/* + * lrng_set_drng_cb - Register new cryptographic callback functions for DRNG + * The registering implies that all old DRNG states are replaced with new + * DRNG states. + * + * drng_cb: Callback functions to be registered -- if NULL, use the default + * callbacks defined at compile time. + * + * Return: + * * 0 on success + * * < 0 on error + */ +int lrng_set_drng_cb(const struct lrng_drng_cb *drng_cb) +{ + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + int ret; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH_DRNG)) + return -EOPNOTSUPP; + + if (!drng_cb) + drng_cb = lrng_default_drng_cb; + + mutex_lock(&lrng_crypto_cb_update); + + /* + * If a callback other than the default is set, allow it only to be + * set back to the default callback. This ensures that multiple + * different callbacks can be registered at the same time. If a + * callback different from the current callback and the default + * callback shall be set, the current callback must be deregistered + * (e.g. the kernel module providing it must be unloaded) and the new + * implementation can be registered. + */ + if ((drng_cb != lrng_default_drng_cb) && + (lrng_drng_init->drng_cb != lrng_default_drng_cb)) { + pr_warn("disallow setting new DRNG callbacks, unload the old callbacks first!\n"); + ret = -EINVAL; + goto out; + } + + ret = lrng_switch(drng_cb, lrng_drng_switch); + /* The swtich may imply new entropy due to larger DRNG sec strength. */ + if (!ret) + lrng_es_add_entropy(); + +out: + mutex_unlock(&lrng_crypto_cb_update); + return ret; +} +EXPORT_SYMBOL(lrng_set_drng_cb); + +/* + * lrng_set_hash_cb - Register new cryptographic callback functions for hash + * The registering implies that all old hash states are replaced with new + * hash states. + * + * @hash_cb: Callback functions to be registered -- if NULL, use the default + * callbacks defined at compile time. + * + * Return: + * * 0 on success + * * < 0 on error + */ +int lrng_set_hash_cb(const struct lrng_hash_cb *hash_cb) +{ + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + int ret; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH_HASH)) + return -EOPNOTSUPP; + + if (!hash_cb) + hash_cb = lrng_default_hash_cb; + + mutex_lock(&lrng_crypto_cb_update); + + /* Comment from lrng_set_drng_cb applies. */ + if ((hash_cb != lrng_default_hash_cb) && + (lrng_drng_init->hash_cb != lrng_default_hash_cb)) { + pr_warn("disallow setting new hash callbacks, unload the old callbacks first!\n"); + ret = -EINVAL; + goto out; + } + + ret = lrng_switch(hash_cb, lrng_hash_switch); + /* The swtich may imply new entropy due to larger digest size. */ + if (!ret) + lrng_es_add_entropy(); + +out: + mutex_unlock(&lrng_crypto_cb_update); + return ret; +} +EXPORT_SYMBOL(lrng_set_hash_cb); diff --git a/drivers/char/lrng/lrng_sysctl.c b/drivers/char/lrng/lrng_sysctl.c new file mode 100644 index 000000000000..d9432378d007 --- /dev/null +++ b/drivers/char/lrng/lrng_sysctl.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG sysctl interfaces + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_mgr.h" +#include "lrng_sysctl.h" + +/* + * This function is used to return both the bootid UUID, and random + * UUID. The difference is in whether table->data is NULL; if it is, + * then a new UUID is generated and returned to the user. + * + * If the user accesses this via the proc interface, the UUID will be + * returned as an ASCII string in the standard UUID format; if via the + * sysctl system call, as 16 bytes of binary data. + */ +static int lrng_sysctl_do_uuid(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table fake_table; + unsigned char buf[64], tmp_uuid[16], *uuid; + + uuid = table->data; + if (!uuid) { + uuid = tmp_uuid; + generate_random_uuid(uuid); + } else { + static DEFINE_SPINLOCK(bootid_spinlock); + + spin_lock(&bootid_spinlock); + if (!uuid[8]) + generate_random_uuid(uuid); + spin_unlock(&bootid_spinlock); + } + + sprintf(buf, "%pU", uuid); + + fake_table.data = buf; + fake_table.maxlen = sizeof(buf); + + return proc_dostring(&fake_table, write, buffer, lenp, ppos); +} + +static int lrng_sysctl_do_entropy(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table fake_table; + int entropy_count; + + entropy_count = lrng_avail_entropy(); + + fake_table.data = &entropy_count; + fake_table.maxlen = sizeof(entropy_count); + + return proc_dointvec(&fake_table, write, buffer, lenp, ppos); +} + +static int lrng_sysctl_do_poolsize(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table fake_table; + u32 i, entropy_count = 0; + + for_each_lrng_es(i) + entropy_count += lrng_es[i]->max_entropy(); + + fake_table.data = &entropy_count; + fake_table.maxlen = sizeof(entropy_count); + + return proc_dointvec(&fake_table, write, buffer, lenp, ppos); +} + +static int lrng_min_write_thresh; +static int lrng_max_write_thresh = (LRNG_WRITE_WAKEUP_ENTROPY << 3); +static char lrng_sysctl_bootid[16]; +static int lrng_drng_reseed_max_min; + +void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) +{ + lrng_max_write_thresh = (int)new_digestsize; + /* Ensure that changes to the global variable are visible */ + mb(); +} + +struct ctl_table random_table[] = { + { + .procname = "poolsize", + .maxlen = sizeof(int), + .mode = 0444, + .proc_handler = lrng_sysctl_do_poolsize, + }, + { + .procname = "entropy_avail", + .maxlen = sizeof(int), + .mode = 0444, + .proc_handler = lrng_sysctl_do_entropy, + }, + { + .procname = "write_wakeup_threshold", + .data = &lrng_write_wakeup_bits, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &lrng_min_write_thresh, + .extra2 = &lrng_max_write_thresh, + }, + { + .procname = "boot_id", + .data = &lrng_sysctl_bootid, + .maxlen = 16, + .mode = 0444, + .proc_handler = lrng_sysctl_do_uuid, + }, + { + .procname = "uuid", + .maxlen = 16, + .mode = 0444, + .proc_handler = lrng_sysctl_do_uuid, + }, + { + .procname = "urandom_min_reseed_secs", + .data = &lrng_drng_reseed_max_time, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + .extra1 = &lrng_drng_reseed_max_min, + }, + { } +}; + diff --git a/drivers/char/lrng/lrng_sysctl.h b/drivers/char/lrng/lrng_sysctl.h new file mode 100644 index 000000000000..4b487e5077ed --- /dev/null +++ b/drivers/char/lrng/lrng_sysctl.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_SYSCTL_H +#define _LRNG_SYSCTL_H + +#ifdef CONFIG_LRNG_SYSCTL +void lrng_sysctl_update_max_write_thresh(u32 new_digestsize); +#else +static inline void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) { } +#endif + +#endif /* _LRNG_SYSCTL_H */ diff --git a/drivers/char/lrng/lrng_testing.c b/drivers/char/lrng/lrng_testing.c new file mode 100644 index 000000000000..101140085d81 --- /dev/null +++ b/drivers/char/lrng/lrng_testing.c @@ -0,0 +1,901 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG testing interfaces to obtain raw entropy + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lrng_definitions.h" +#include "lrng_drng_chacha20.h" +#include "lrng_sha.h" +#include "lrng_testing.h" + +#if defined(CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY) || \ + defined(CONFIG_LRNG_RAW_SCHED_PID_ENTROPY) || \ + defined(CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY) || \ + defined(CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY) || \ + defined(CONFIG_LRNG_SCHED_PERF) +#define LRNG_TESTING_USE_BUSYLOOP +#endif + +#ifdef CONFIG_LRNG_TESTING_RECORDING + +#define LRNG_TESTING_RINGBUFFER_SIZE 1024 +#define LRNG_TESTING_RINGBUFFER_MASK (LRNG_TESTING_RINGBUFFER_SIZE - 1) + +struct lrng_testing { + u32 lrng_testing_rb[LRNG_TESTING_RINGBUFFER_SIZE]; + u32 rb_reader; + atomic_t rb_writer; + atomic_t lrng_testing_enabled; + spinlock_t lock; + wait_queue_head_t read_wait; +}; + +/*************************** Generic Data Handling ****************************/ + +/* + * boot variable: + * 0 ==> No boot test, gathering of runtime data allowed + * 1 ==> Boot test enabled and ready for collecting data, gathering runtime + * data is disabled + * 2 ==> Boot test completed and disabled, gathering of runtime data is + * disabled + */ + +static void lrng_testing_reset(struct lrng_testing *data) +{ + unsigned long flags; + + spin_lock_irqsave(&data->lock, flags); + data->rb_reader = 0; + atomic_set(&data->rb_writer, 0); + spin_unlock_irqrestore(&data->lock, flags); +} + +static void lrng_testing_init(struct lrng_testing *data, u32 boot) +{ + /* + * The boot time testing implies we have a running test. If the + * caller wants to clear it, he has to unset the boot_test flag + * at runtime via sysfs to enable regular runtime testing + */ + if (boot) + return; + + lrng_testing_reset(data); + atomic_set(&data->lrng_testing_enabled, 1); + pr_warn("Enabling data collection\n"); +} + +static void lrng_testing_fini(struct lrng_testing *data, u32 boot) +{ + /* If we have boot data, we do not reset yet to allow data to be read */ + if (boot) + return; + + atomic_set(&data->lrng_testing_enabled, 0); + lrng_testing_reset(data); + pr_warn("Disabling data collection\n"); +} + +static bool lrng_testing_store(struct lrng_testing *data, u32 value, + u32 *boot) +{ + unsigned long flags; + + if (!atomic_read(&data->lrng_testing_enabled) && (*boot != 1)) + return false; + + spin_lock_irqsave(&data->lock, flags); + + /* + * Disable entropy testing for boot time testing after ring buffer + * is filled. + */ + if (*boot) { + if (((u32)atomic_read(&data->rb_writer)) > + LRNG_TESTING_RINGBUFFER_SIZE) { + *boot = 2; + pr_warn_once("One time data collection test disabled\n"); + spin_unlock_irqrestore(&data->lock, flags); + return false; + } + + if (atomic_read(&data->rb_writer) == 1) + pr_warn("One time data collection test enabled\n"); + } + + data->lrng_testing_rb[((u32)atomic_read(&data->rb_writer)) & + LRNG_TESTING_RINGBUFFER_MASK] = value; + atomic_inc(&data->rb_writer); + + spin_unlock_irqrestore(&data->lock, flags); + +#ifndef LRNG_TESTING_USE_BUSYLOOP + if (wq_has_sleeper(&data->read_wait)) + wake_up_interruptible(&data->read_wait); +#endif + + return true; +} + +static bool lrng_testing_have_data(struct lrng_testing *data) +{ + return ((((u32)atomic_read(&data->rb_writer)) & + LRNG_TESTING_RINGBUFFER_MASK) != + (data->rb_reader & LRNG_TESTING_RINGBUFFER_MASK)); +} + +static int lrng_testing_reader(struct lrng_testing *data, u32 *boot, + u8 *outbuf, u32 outbuflen) +{ + unsigned long flags; + int collected_data = 0; + + lrng_testing_init(data, *boot); + + while (outbuflen) { + u32 writer = (u32)atomic_read(&data->rb_writer); + + spin_lock_irqsave(&data->lock, flags); + + /* We have no data or reached the writer. */ + if (!writer || (writer == data->rb_reader)) { + + spin_unlock_irqrestore(&data->lock, flags); + + /* + * Now we gathered all boot data, enable regular data + * collection. + */ + if (*boot) { + *boot = 0; + goto out; + } + +#ifdef LRNG_TESTING_USE_BUSYLOOP + while (!lrng_testing_have_data(data)) + ; +#else + wait_event_interruptible(data->read_wait, + lrng_testing_have_data(data)); +#endif + if (signal_pending(current)) { + collected_data = -ERESTARTSYS; + goto out; + } + + continue; + } + + /* We copy out word-wise */ + if (outbuflen < sizeof(u32)) { + spin_unlock_irqrestore(&data->lock, flags); + goto out; + } + + memcpy(outbuf, &data->lrng_testing_rb[data->rb_reader], + sizeof(u32)); + data->rb_reader++; + + spin_unlock_irqrestore(&data->lock, flags); + + outbuf += sizeof(u32); + outbuflen -= sizeof(u32); + collected_data += sizeof(u32); + } + +out: + lrng_testing_fini(data, *boot); + return collected_data; +} + +static int lrng_testing_extract_user(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos, + int (*reader)(u8 *outbuf, u32 outbuflen)) +{ + u8 *tmp, *tmp_aligned; + int ret = 0, large_request = (nbytes > 256); + + if (!nbytes) + return 0; + + /* + * The intention of this interface is for collecting at least + * 1000 samples due to the SP800-90B requirements. So, we make no + * effort in avoiding allocating more memory that actually needed + * by the user. Hence, we allocate sufficient memory to always hold + * that amount of data. + */ + tmp = kmalloc(LRNG_TESTING_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); + + while (nbytes) { + int i; + + if (large_request && need_resched()) { + if (signal_pending(current)) { + if (ret == 0) + ret = -ERESTARTSYS; + break; + } + schedule(); + } + + i = min_t(int, nbytes, LRNG_TESTING_RINGBUFFER_SIZE); + i = reader(tmp_aligned, i); + if (i <= 0) { + if (i < 0) + ret = i; + break; + } + if (copy_to_user(buf, tmp_aligned, i)) { + ret = -EFAULT; + break; + } + + nbytes -= i; + buf += i; + ret += i; + } + + kfree_sensitive(tmp); + + if (ret > 0) + *ppos += ret; + + return ret; +} + +#endif /* CONFIG_LRNG_TESTING_RECORDING */ + +/************* Raw High-Resolution IRQ Timer Entropy Data Handling ************/ + +#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY + +static u32 boot_raw_hires_test = 0; +module_param(boot_raw_hires_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_hires_test, "Enable gathering boot time high resolution timer entropy of the first IRQ entropy events"); + +static struct lrng_testing lrng_raw_hires = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_hires.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_hires.read_wait) +}; + +bool lrng_raw_hires_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_hires, value, &boot_raw_hires_test); +} + +static int lrng_raw_hires_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_hires, &boot_raw_hires_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_hires_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_hires_entropy_reader); +} + +static const struct file_operations lrng_raw_hires_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_hires_read, +}; + +#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ + +/********************* Raw Jiffies Entropy Data Handling **********************/ + +#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY + +static u32 boot_raw_jiffies_test = 0; +module_param(boot_raw_jiffies_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_jiffies_test, "Enable gathering boot time high resolution timer entropy of the first entropy events"); + +static struct lrng_testing lrng_raw_jiffies = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_jiffies.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_jiffies.read_wait) +}; + +bool lrng_raw_jiffies_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_jiffies, value, + &boot_raw_jiffies_test); +} + +static int lrng_raw_jiffies_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_jiffies, &boot_raw_jiffies_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_jiffies_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_jiffies_entropy_reader); +} + +static const struct file_operations lrng_raw_jiffies_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_jiffies_read, +}; + +#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ + +/************************** Raw IRQ Data Handling ****************************/ + +#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY + +static u32 boot_raw_irq_test = 0; +module_param(boot_raw_irq_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_irq_test, "Enable gathering boot time entropy of the first IRQ entropy events"); + +static struct lrng_testing lrng_raw_irq = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_irq.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_irq.read_wait) +}; + +bool lrng_raw_irq_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_irq, value, &boot_raw_irq_test); +} + +static int lrng_raw_irq_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_irq, &boot_raw_irq_test, outbuf, + outbuflen); +} + +static ssize_t lrng_raw_irq_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_irq_entropy_reader); +} + +static const struct file_operations lrng_raw_irq_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_irq_read, +}; + +#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ + +/************************ Raw _RET_IP_ Data Handling **************************/ + +#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY + +static u32 boot_raw_retip_test = 0; +module_param(boot_raw_retip_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_retip_test, "Enable gathering boot time entropy of the first return instruction pointer entropy events"); + +static struct lrng_testing lrng_raw_retip = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_retip.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_retip.read_wait) +}; + +bool lrng_raw_retip_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_retip, value, &boot_raw_retip_test); +} + +static int lrng_raw_retip_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_retip, &boot_raw_retip_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_retip_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_retip_entropy_reader); +} + +static const struct file_operations lrng_raw_retip_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_retip_read, +}; + +#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ + +/********************** Raw IRQ register Data Handling ************************/ + +#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY + +static u32 boot_raw_regs_test = 0; +module_param(boot_raw_regs_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_regs_test, "Enable gathering boot time entropy of the first interrupt register entropy events"); + +static struct lrng_testing lrng_raw_regs = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_regs.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_regs.read_wait) +}; + +bool lrng_raw_regs_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_regs, value, &boot_raw_regs_test); +} + +static int lrng_raw_regs_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_regs, &boot_raw_regs_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_regs_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_regs_entropy_reader); +} + +static const struct file_operations lrng_raw_regs_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_regs_read, +}; + +#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ + +/********************** Raw Entropy Array Data Handling ***********************/ + +#ifdef CONFIG_LRNG_RAW_ARRAY + +static u32 boot_raw_array = 0; +module_param(boot_raw_array, uint, 0644); +MODULE_PARM_DESC(boot_raw_array, "Enable gathering boot time raw noise array data of the first entropy events"); + +static struct lrng_testing lrng_raw_array = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_array.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_array.read_wait) +}; + +bool lrng_raw_array_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_array, value, &boot_raw_array); +} + +static int lrng_raw_array_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_array, &boot_raw_array, outbuf, + outbuflen); +} + +static ssize_t lrng_raw_array_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_array_entropy_reader); +} + +static const struct file_operations lrng_raw_array_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_array_read, +}; + +#endif /* CONFIG_LRNG_RAW_ARRAY */ + +/******************** Interrupt Performance Data Handling *********************/ + +#ifdef CONFIG_LRNG_IRQ_PERF + +static u32 boot_irq_perf = 0; +module_param(boot_irq_perf, uint, 0644); +MODULE_PARM_DESC(boot_irq_perf, "Enable gathering interrupt entropy source performance data"); + +static struct lrng_testing lrng_irq_perf = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_irq_perf.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_irq_perf.read_wait) +}; + +bool lrng_perf_time(u32 start) +{ + return lrng_testing_store(&lrng_irq_perf, random_get_entropy() - start, + &boot_irq_perf); +} + +static int lrng_irq_perf_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_irq_perf, &boot_irq_perf, outbuf, + outbuflen); +} + +static ssize_t lrng_irq_perf_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_irq_perf_reader); +} + +static const struct file_operations lrng_irq_perf_fops = { + .owner = THIS_MODULE, + .read = lrng_irq_perf_read, +}; + +#endif /* CONFIG_LRNG_IRQ_PERF */ + +/****** Raw High-Resolution Scheduler-based Timer Entropy Data Handling *******/ + +#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY + +static u32 boot_raw_sched_hires_test = 0; +module_param(boot_raw_sched_hires_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_hires_test, "Enable gathering boot time high resolution timer entropy of the first Scheduler-based entropy events"); + +static struct lrng_testing lrng_raw_sched_hires = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_hires.lock), + .read_wait = + __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_hires.read_wait) +}; + +bool lrng_raw_sched_hires_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_hires, value, + &boot_raw_sched_hires_test); +} + +static int lrng_raw_sched_hires_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_hires, + &boot_raw_sched_hires_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_hires_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_hires_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_hires_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_hires_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ + +/******************** Interrupt Performance Data Handling *********************/ + +#ifdef CONFIG_LRNG_SCHED_PERF + +static u32 boot_sched_perf = 0; +module_param(boot_sched_perf, uint, 0644); +MODULE_PARM_DESC(boot_sched_perf, "Enable gathering scheduler-based entropy source performance data"); + +static struct lrng_testing lrng_sched_perf = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_sched_perf.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_sched_perf.read_wait) +}; + +bool lrng_sched_perf_time(u32 start) +{ + return lrng_testing_store(&lrng_sched_perf, random_get_entropy() - start, + &boot_sched_perf); +} + +static int lrng_sched_perf_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_sched_perf, &boot_sched_perf, outbuf, + outbuflen); +} + +static ssize_t lrng_sched_perf_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_sched_perf_reader); +} + +static const struct file_operations lrng_sched_perf_fops = { + .owner = THIS_MODULE, + .read = lrng_sched_perf_read, +}; + +#endif /* CONFIG_LRNG_SCHED_PERF */ + +/*************** Raw Scheduler task_struct->pid Data Handling *****************/ + +#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY + +static u32 boot_raw_sched_pid_test = 0; +module_param(boot_raw_sched_pid_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_pid_test, "Enable gathering boot time entropy of the first PIDs collected by the scheduler entropy source"); + +static struct lrng_testing lrng_raw_sched_pid = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_pid.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_pid.read_wait) +}; + +bool lrng_raw_sched_pid_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_pid, value, + &boot_raw_sched_pid_test); +} + +static int lrng_raw_sched_pid_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_pid, + &boot_raw_sched_pid_test, outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_pid_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_pid_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_pid_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_pid_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ + + +/*********** Raw Scheduler task_struct->start_time Data Handling **************/ + +#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY + +static u32 boot_raw_sched_starttime_test = 0; +module_param(boot_raw_sched_starttime_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_starttime_test, "Enable gathering boot time entropy of the first task start times collected by the scheduler entropy source"); + +static struct lrng_testing lrng_raw_sched_starttime = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_starttime.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_starttime.read_wait) +}; + +bool lrng_raw_sched_starttime_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_starttime, value, + &boot_raw_sched_starttime_test); +} + +static int lrng_raw_sched_starttime_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_starttime, + &boot_raw_sched_starttime_test, outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_starttime_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_starttime_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_starttime_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_starttime_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ + +/************** Raw Scheduler task_struct->nvcsw Data Handling ****************/ + +#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY + +static u32 boot_raw_sched_nvcsw_test = 0; +module_param(boot_raw_sched_nvcsw_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_nvcsw_test, "Enable gathering boot time entropy of the first task context switch numbers collected by the scheduler entropy source"); + +static struct lrng_testing lrng_raw_sched_nvcsw = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_nvcsw.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_nvcsw.read_wait) +}; + +bool lrng_raw_sched_nvcsw_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_nvcsw, value, + &boot_raw_sched_nvcsw_test); +} + +static int lrng_raw_sched_nvcsw_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_nvcsw, + &boot_raw_sched_nvcsw_test, outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_nvcsw_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_nvcsw_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_nvcsw_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_nvcsw_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ + +/*********************************** ACVT ************************************/ + +#ifdef CONFIG_LRNG_ACVT_HASH + +/* maximum amount of data to be hashed as defined by ACVP */ +#define LRNG_ACVT_MAX_SHA_MSG (65536 >> 3) + +/* + * As we use static variables to store the data, it is clear that the + * test interface is only able to handle single threaded testing. This is + * considered to be sufficient for testing. If multi-threaded use of the + * ACVT test interface would be performed, the caller would get garbage + * but the kernel operation is unaffected by this. + */ +static u8 lrng_acvt_hash_data[LRNG_ACVT_MAX_SHA_MSG] + __aligned(LRNG_KCAPI_ALIGN); +static atomic_t lrng_acvt_hash_data_size = ATOMIC_INIT(0); +static u8 lrng_acvt_hash_digest[LRNG_ATOMIC_DIGEST_SIZE]; + +static ssize_t lrng_acvt_hash_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + if (nbytes > LRNG_ACVT_MAX_SHA_MSG) + return -EINVAL; + + atomic_set(&lrng_acvt_hash_data_size, (int)nbytes); + + return simple_write_to_buffer(lrng_acvt_hash_data, + LRNG_ACVT_MAX_SHA_MSG, ppos, buf, nbytes); +} + +static ssize_t lrng_acvt_hash_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; + ssize_t ret; + + if (count > LRNG_ATOMIC_DIGEST_SIZE) + return -EINVAL; + + ret = hash_cb->hash_init(shash, NULL) ?: + hash_cb->hash_update(shash, lrng_acvt_hash_data, + atomic_read_u32(&lrng_acvt_hash_data_size)) ?: + hash_cb->hash_final(shash, lrng_acvt_hash_digest); + if (ret) + return ret; + + return simple_read_from_buffer(to, count, ppos, lrng_acvt_hash_digest, + sizeof(lrng_acvt_hash_digest)); +} + +static const struct file_operations lrng_acvt_hash_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .llseek = default_llseek, + .read = lrng_acvt_hash_read, + .write = lrng_acvt_hash_write, +}; + +#endif /* CONFIG_LRNG_ACVT_DRNG */ + +/************************************************************************** + * Debugfs interface + **************************************************************************/ + +static int __init lrng_raw_init(void) +{ + struct dentry *lrng_raw_debugfs_root; + + lrng_raw_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); + +#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY + debugfs_create_file_unsafe("lrng_raw_hires", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_hires_fops); +#endif +#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY + debugfs_create_file_unsafe("lrng_raw_jiffies", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_jiffies_fops); +#endif +#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY + debugfs_create_file_unsafe("lrng_raw_irq", 0400, lrng_raw_debugfs_root, + NULL, &lrng_raw_irq_fops); +#endif +#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY + debugfs_create_file_unsafe("lrng_raw_retip", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_retip_fops); +#endif +#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY + debugfs_create_file_unsafe("lrng_raw_regs", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_regs_fops); +#endif +#ifdef CONFIG_LRNG_RAW_ARRAY + debugfs_create_file_unsafe("lrng_raw_array", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_array_fops); +#endif +#ifdef CONFIG_LRNG_IRQ_PERF + debugfs_create_file_unsafe("lrng_irq_perf", 0400, lrng_raw_debugfs_root, + NULL, &lrng_irq_perf_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_hires", 0400, + lrng_raw_debugfs_root, + NULL, &lrng_raw_sched_hires_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_pid", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_sched_pid_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_starttime", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_sched_starttime_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_nvcsw", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_sched_nvcsw_fops); +#endif +#ifdef CONFIG_LRNG_SCHED_PERF + debugfs_create_file_unsafe("lrng_sched_perf", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_sched_perf_fops); +#endif +#ifdef CONFIG_LRNG_ACVT_HASH + debugfs_create_file_unsafe("lrng_acvt_hash", 0600, + lrng_raw_debugfs_root, NULL, + &lrng_acvt_hash_fops); +#endif + + return 0; +} + +module_init(lrng_raw_init); diff --git a/drivers/char/lrng/lrng_testing.h b/drivers/char/lrng/lrng_testing.h new file mode 100644 index 000000000000..672a7fca4595 --- /dev/null +++ b/drivers/char/lrng/lrng_testing.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_TESTING_H +#define _LRNG_TESTING_H + +#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY +bool lrng_raw_hires_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ +static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY +bool lrng_raw_jiffies_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ +static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY +bool lrng_raw_irq_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ +static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY +bool lrng_raw_retip_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ +static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY +bool lrng_raw_regs_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ +static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_ARRAY +bool lrng_raw_array_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_ARRAY */ +static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_ARRAY */ + +#ifdef CONFIG_LRNG_IRQ_PERF +bool lrng_perf_time(u32 start); +#else /* CONFIG_LRNG_IRQ_PERF */ +static inline bool lrng_perf_time(u32 start) { return false; } +#endif /*CONFIG_LRNG_IRQ_PERF */ + +#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY +bool lrng_raw_sched_hires_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ +static inline bool +lrng_raw_sched_hires_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY +bool lrng_raw_sched_pid_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ +static inline bool +lrng_raw_sched_pid_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY +bool lrng_raw_sched_starttime_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ +static inline bool +lrng_raw_sched_starttime_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY +bool lrng_raw_sched_nvcsw_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ +static inline bool +lrng_raw_sched_nvcsw_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ + +#ifdef CONFIG_LRNG_SCHED_PERF +bool lrng_sched_perf_time(u32 start); +#else /* CONFIG_LRNG_SCHED_PERF */ +static inline bool lrng_sched_perf_time(u32 start) { return false; } +#endif /*CONFIG_LRNG_SCHED_PERF */ + +#endif /* _LRNG_TESTING_H */ diff --git a/drivers/char/random.c b/drivers/char/random.c index 00d4aac56197..b6a4bba16953 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -461,7 +461,7 @@ static struct crng_state primary_crng = { * its value (from 0->1->2). */ static int crng_init = 0; -#define crng_ready() (likely(crng_init > 1)) +#define crng_ready() (true) static int crng_init_cnt = 0; static unsigned long crng_global_init_time = 0; #define CRNG_INIT_CNT_THRESH (2*CHACHA_KEY_SIZE) @@ -781,27 +781,55 @@ static int __init parse_trust_cpu(char *arg) } early_param("random.trust_cpu", parse_trust_cpu); -static void crng_initialize(struct crng_state *crng) +static bool crng_init_try_arch(struct crng_state *crng) { int i; - int arch_init = 1; + bool arch_init = true; unsigned long rv; - memcpy(&crng->state[0], "expand 32-byte k", 16); - if (crng == &primary_crng) - _extract_entropy(&input_pool, &crng->state[4], - sizeof(__u32) * 12, 0); - else - _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); for (i = 4; i < 16; i++) { if (!arch_get_random_seed_long(&rv) && !arch_get_random_long(&rv)) { rv = random_get_entropy(); - arch_init = 0; + arch_init = false; + } + crng->state[i] ^= rv; + } + + return arch_init; +} + +static bool __init crng_init_try_arch_early(struct crng_state *crng) +{ + int i; + bool arch_init = true; + unsigned long rv; + + for (i = 4; i < 16; i++) { + if (!arch_get_random_seed_long_early(&rv) && + !arch_get_random_long_early(&rv)) { + rv = random_get_entropy(); + arch_init = false; } crng->state[i] ^= rv; } - if (trust_cpu && arch_init && crng == &primary_crng) { + + return arch_init; +} + +static void __maybe_unused crng_initialize_secondary(struct crng_state *crng) +{ + chacha_init_consts(crng->state); + _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); + crng_init_try_arch(crng); + crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; +} + +static void __init crng_initialize_primary(struct crng_state *crng) +{ + chacha_init_consts(crng->state); + _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); + if (crng_init_try_arch_early(crng) && trust_cpu) { invalidate_batched_entropy(); numa_crng_init(); crng_init = 2; @@ -822,7 +850,7 @@ static void do_numa_crng_init(struct work_struct *work) crng = kmalloc_node(sizeof(struct crng_state), GFP_KERNEL | __GFP_NOFAIL, i); spin_lock_init(&crng->lock); - crng_initialize(crng); + crng_initialize_secondary(crng); pool[i] = crng; } /* pairs with READ_ONCE() in select_crng() */ @@ -1775,7 +1803,7 @@ static void __init init_std_data(struct entropy_store *r) int __init rand_initialize(void) { init_std_data(&input_pool); - crng_initialize(&primary_crng); + crng_initialize_primary(&primary_crng); crng_global_init_time = jiffies; if (ratelimit_disable) { urandom_warning.interval = 0; @@ -1962,7 +1990,7 @@ static int random_fasync(int fd, struct file *filp, int on) } const struct file_operations random_fops = { - .read = random_read, + .read = urandom_read, .write = random_write, .poll = random_poll, .unlocked_ioctl = random_ioctl, @@ -2300,4 +2328,4 @@ void add_bootloader_randomness(const void *buf, unsigned int size) else add_device_randomness(buf, size); } -EXPORT_SYMBOL_GPL(add_bootloader_randomness); \ No newline at end of file +EXPORT_SYMBOL_GPL(add_bootloader_randomness); diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index e71c6b24aed1..ce497b0691d7 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -717,7 +717,16 @@ ssize_t tpm2_get_tpm_pt(struct tpm_chip *chip, u32 property_id, u32 *value, if (!rc) { out = (struct tpm2_get_cap_out *) &buf.data[TPM_HEADER_SIZE]; - *value = be32_to_cpu(out->value); + /* + * To prevent failing boot up of some systems, Infineon TPM2.0 + * returns SUCCESS on TPM2_Startup in field upgrade mode. Also + * the TPM2_Getcapability command returns a zero length list + * in field upgrade mode. + */ + if (be32_to_cpu(out->property_cnt) > 0) + *value = be32_to_cpu(out->value); + else + rc = -ENODATA; } tpm_buf_destroy(&buf); return rc; diff --git a/drivers/char/tpm/tpm_ibmvtpm.c b/drivers/char/tpm/tpm_ibmvtpm.c index 3ba67bc6baba..80647eb071fd 100644 --- a/drivers/char/tpm/tpm_ibmvtpm.c +++ b/drivers/char/tpm/tpm_ibmvtpm.c @@ -692,6 +692,7 @@ static int tpm_ibmvtpm_probe(struct vio_dev *vio_dev, if (!wait_event_timeout(ibmvtpm->crq_queue.wq, ibmvtpm->rtce_buf != NULL, HZ)) { + rc = -ENODEV; dev_err(dev, "CRQ response timed out\n"); goto init_irq_cleanup; } diff --git a/drivers/clocksource/riscv_timer.c b/drivers/clocksource/riscv_timer.c index 4e8b347e43e2..0d5b99ca3bbd 100644 --- a/drivers/clocksource/riscv_timer.c +++ b/drivers/clocksource/riscv_timer.c @@ -33,7 +33,7 @@ static int riscv_clock_next_event(unsigned long delta, static DEFINE_PER_CPU(struct clock_event_device, riscv_clock_event) = { .name = "riscv_timer_clockevent", - .features = CLOCK_EVT_FEAT_ONESHOT, + .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP, .rating = 100, .set_next_event = riscv_clock_next_event, }; diff --git a/drivers/clocksource/timer-oxnas-rps.c b/drivers/clocksource/timer-oxnas-rps.c index 30c6f4ce672b..cfcd54e66c57 100644 --- a/drivers/clocksource/timer-oxnas-rps.c +++ b/drivers/clocksource/timer-oxnas-rps.c @@ -247,7 +247,7 @@ static int __init oxnas_rps_timer_init(struct device_node *np) } rps->irq = irq_of_parse_and_map(np, 0); - if (rps->irq < 0) { + if (!rps->irq) { ret = -EINVAL; goto err_iomap; } diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index e01222ea888f..738c50e916f7 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -228,6 +228,11 @@ static int __init sp804_of_init(struct device_node *np) struct clk *clk1, *clk2; const char *name = of_get_property(np, "compatible", NULL); + if (initialized) { + pr_debug("%pOF: skipping further SP804 timer device\n", np); + return 0; + } + base = of_iomap(np, 0); if (!base) return -ENXIO; @@ -236,11 +241,6 @@ static int __init sp804_of_init(struct device_node *np) writel(0, base + TIMER_CTRL); writel(0, base + TIMER_2_BASE + TIMER_CTRL); - if (initialized || !of_device_is_available(np)) { - ret = -EINVAL; - goto err; - } - clk1 = of_clk_get(np, 0); if (IS_ERR(clk1)) clk1 = NULL; diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index 0ae84ec9e21c..c9b905efc448 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -625,7 +625,6 @@ struct skcipher_alg mv_cesa_ecb_des3_ede_alg = { .decrypt = mv_cesa_ecb_des3_ede_decrypt, .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, .base = { .cra_name = "ecb(des3_ede)", .cra_driver_name = "mv-ecb-des3-ede", diff --git a/drivers/devfreq/arm-memlat-mon.c b/drivers/devfreq/arm-memlat-mon.c index e8efed9fd79f..9b2b97d0c414 100644 --- a/drivers/devfreq/arm-memlat-mon.c +++ b/drivers/devfreq/arm-memlat-mon.c @@ -26,7 +26,6 @@ #include #include #include -#include enum common_ev_idx { INST_IDX, @@ -129,6 +128,7 @@ struct memlat_cpu_grp { cpumask_t cpus; unsigned int common_ev_ids[NUM_COMMON_EVS]; struct cpu_data *cpus_data; + int read_event_cpu; ktime_t last_update_ts; unsigned long last_ts_delta_us; @@ -185,14 +185,20 @@ static void update_counts(struct memlat_cpu_grp *cpu_grp) struct cpu_data *cpu_data = to_cpu_data(cpu_grp, cpu); struct event_data *common_evs = cpu_data->common_evs; - for (i = 0; i < NUM_COMMON_EVS; i++) + for (i = 0; i < NUM_COMMON_EVS; i++) { + cpu_grp->read_event_cpu = cpu; read_event(&common_evs[i]); + cpu_grp->read_event_cpu = -1; + } if (!common_evs[STALL_IDX].pevent) common_evs[STALL_IDX].last_delta = common_evs[CYC_IDX].last_delta; - cpu_data->freq = common_evs[CYC_IDX].last_delta / delta; + if (delta != 0) + cpu_data->freq = common_evs[CYC_IDX].last_delta / delta; + else + cpu_data->freq = common_evs[CYC_IDX].last_delta; cpu_data->stall_pct = mult_frac(100, common_evs[STALL_IDX].last_delta, common_evs[CYC_IDX].last_delta); @@ -207,12 +213,15 @@ static void update_counts(struct memlat_cpu_grp *cpu_grp) for_each_cpu(cpu, &mon->cpus) { unsigned int mon_idx = cpu - cpumask_first(&mon->cpus); + cpu_grp->read_event_cpu = cpu; read_event(&mon->miss_ev[mon_idx]); if (mon->wb_ev_id && mon->access_ev_id) { read_event(&mon->wb_ev[mon_idx]); read_event(&mon->access_ev[mon_idx]); } + + cpu_grp->read_event_cpu = -1; } } } @@ -223,13 +232,6 @@ static unsigned long get_cnt(struct memlat_hwmon *hw) struct memlat_cpu_grp *cpu_grp = mon->cpu_grp; unsigned int cpu; - /* - * Some of SCM call is very heavy(+20ms) so perf IPI could - * be stuck on the CPU which contributes long latency. - */ - if (under_scm_call()) - return 0; - for_each_cpu(cpu, &mon->cpus) { struct cpu_data *cpu_data = to_cpu_data(cpu_grp, cpu); struct event_data *common_evs = cpu_data->common_evs; @@ -572,6 +574,8 @@ static int memlat_cpu_grp_probe(struct platform_device *pdev) return -ENODEV; } + cpu_grp->read_event_cpu = -1; + num_mons = of_get_available_child_count(dev->of_node); if (!num_mons) { diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index f0e2af4fc05d..12b1d723e879 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -1258,7 +1258,7 @@ static ssize_t min_freq_show(struct device *dev, struct device_attribute *attr, return sprintf(buf, "%lu\n", MAX(df->scaling_min_freq, df->min_freq)); } -static ssize_t max_freq_store(struct device *dev, struct device_attribute *attr, +static ssize_t __maybe_unused max_freq_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct devfreq *df = to_devfreq(dev); @@ -1304,7 +1304,7 @@ static ssize_t max_freq_show(struct device *dev, struct device_attribute *attr, return sprintf(buf, "%lu\n", MIN(df->scaling_max_freq, df->max_freq)); } -static DEVICE_ATTR_RW(max_freq); +static DEVICE_ATTR_RO(max_freq); static ssize_t available_frequencies_show(struct device *d, struct device_attribute *attr, @@ -1403,7 +1403,8 @@ static int __init devfreq_init(void) return PTR_ERR(devfreq_class); } - devfreq_wq = create_freezable_workqueue("devfreq_wq"); + devfreq_wq = alloc_workqueue("devfreq_wq", WQ_HIGHPRI | WQ_FREEZABLE | + WQ_UNBOUND | WQ_MEM_RECLAIM, 1); if (!devfreq_wq) { class_destroy(devfreq_class); pr_err("%s: couldn't create workqueue\n", __FILE__); diff --git a/drivers/devfreq/governor_memlat.c b/drivers/devfreq/governor_memlat.c index 4948a144ab87..4477b99b94e7 100644 --- a/drivers/devfreq/governor_memlat.c +++ b/drivers/devfreq/governor_memlat.c @@ -215,10 +215,12 @@ static int gov_start(struct devfreq *df) static int gov_suspend(struct devfreq *df) { struct memlat_node *node = df->data; + struct memlat_hwmon *hw = node->hw; unsigned long prev_freq = df->previous_freq; node->mon_started = false; - devfreq_monitor_suspend(df); + if (!hw->should_ignore_df_monitor) + devfreq_monitor_suspend(df); mutex_lock(&df->lock); update_devfreq(df); @@ -232,6 +234,7 @@ static int gov_suspend(struct devfreq *df) static int gov_resume(struct devfreq *df) { struct memlat_node *node = df->data; + struct memlat_hwmon *hw = node->hw; mutex_lock(&df->lock); update_devfreq(df); @@ -239,7 +242,8 @@ static int gov_resume(struct devfreq *df) node->resume_freq = 0; - devfreq_monitor_resume(df); + if (!hw->should_ignore_df_monitor) + devfreq_monitor_resume(df); node->mon_started = true; return 0; @@ -325,7 +329,7 @@ static int devfreq_memlat_get_freq(struct devfreq *df, return 0; } -gov_attr(ratio_ceil, 1U, 20000U); +gov_attr(ratio_ceil, 1U, 50000U); gov_attr(stall_floor, 0U, 100U); gov_attr(wb_pct_thres, 0U, 100U); gov_attr(wb_filter_ratio, 0U, 50000U); diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c index e795ad2b3f6b..eefda6edc89c 100644 --- a/drivers/devfreq/rk3399_dmc.c +++ b/drivers/devfreq/rk3399_dmc.c @@ -411,6 +411,8 @@ static int rk3399_dmcfreq_remove(struct platform_device *pdev) { struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(&pdev->dev); + devfreq_event_disable_edev(dmcfreq->edev); + /* * Before remove the opp table we need to unregister the opp notifier. */ diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index 8585fed84e83..3259c450544c 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -50,7 +50,6 @@ STM32_MDMA_SHIFT(mask)) #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */ -#define STM32_MDMA_GISR1 0x0004 /* MDMA Int Status Reg 2 */ /* MDMA Channel x interrupt/status register */ #define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */ @@ -206,7 +205,7 @@ #define STM32_MDMA_MAX_BUF_LEN 128 #define STM32_MDMA_MAX_BLOCK_LEN 65536 -#define STM32_MDMA_MAX_CHANNELS 63 +#define STM32_MDMA_MAX_CHANNELS 32 #define STM32_MDMA_MAX_REQUESTS 256 #define STM32_MDMA_MAX_BURST 128 #define STM32_MDMA_VERY_HIGH_PRIORITY 0x11 @@ -1361,21 +1360,11 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) /* Find out which channel generates the interrupt */ status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0); - if (status) { - id = __ffs(status); - } else { - status = readl_relaxed(dmadev->base + STM32_MDMA_GISR1); - if (!status) { - dev_dbg(mdma2dev(dmadev), "spurious it\n"); - return IRQ_NONE; - } - id = __ffs(status); - /* - * As GISR0 provides status for channel id from 0 to 31, - * so GISR1 provides status for channel id from 32 to 62 - */ - id += 32; + if (!status) { + dev_dbg(mdma2dev(dmadev), "spurious it\n"); + return IRQ_NONE; } + id = __ffs(status); chan = &dmadev->chan[id]; if (!chan) { diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c index 4b68aa9ac9ca..17ec0d3dbba5 100644 --- a/drivers/extcon/extcon.c +++ b/drivers/extcon/extcon.c @@ -1303,23 +1303,17 @@ int extcon_dev_register(struct extcon_dev *edev) edev->dev.type = &edev->extcon_dev_type; } - ret = device_register(&edev->dev); - if (ret) { - put_device(&edev->dev); - goto err_dev; - } - spin_lock_init(&edev->lock); - edev->nh = devm_kcalloc(&edev->dev, edev->max_supported, - sizeof(*edev->nh), GFP_KERNEL); - if (!edev->nh) { - ret = -ENOMEM; - device_unregister(&edev->dev); - goto err_dev; + if (edev->max_supported) { + edev->nh = kcalloc(edev->max_supported, sizeof(*edev->nh), + GFP_KERNEL); + if (!edev->nh) { + ret = -ENOMEM; + goto err_alloc_nh; + } } - edev->bnh = devm_kzalloc(&edev->dev, - sizeof(*edev->bnh) * edev->max_supported, GFP_KERNEL); + edev->bnh = kzalloc(sizeof(*edev->bnh) * edev->max_supported, GFP_KERNEL); if (!edev->bnh) { ret = -ENOMEM; goto err_dev; @@ -1335,13 +1329,24 @@ int extcon_dev_register(struct extcon_dev *edev) dev_set_drvdata(&edev->dev, edev); edev->state = 0; + ret = device_register(&edev->dev); + if (ret) { + put_device(&edev->dev); + goto err_reg; + } + mutex_lock(&extcon_dev_list_lock); list_add(&edev->entry, &extcon_dev_list); mutex_unlock(&extcon_dev_list_lock); return 0; +err_reg: + kfree(edev->bnh); err_dev: + if (edev->max_supported) + kfree(edev->nh); +err_alloc_nh: if (edev->max_supported) kfree(edev->extcon_dev_type.groups); err_alloc_groups: @@ -1402,7 +1407,9 @@ void extcon_dev_unregister(struct extcon_dev *edev) if (edev->max_supported) { kfree(edev->extcon_dev_type.groups); kfree(edev->cables); + kfree(edev->nh); } + kfree(edev->bnh); put_device(&edev->dev); } diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c index 204390297f4b..95d892db0dff 100644 --- a/drivers/firmware/arm_scmi/base.c +++ b/drivers/firmware/arm_scmi/base.c @@ -164,7 +164,7 @@ static int scmi_base_implementation_list_get(const struct scmi_handle *handle, break; loop_num_ret = le32_to_cpu(*num_ret); - if (tot_num_ret + loop_num_ret > MAX_PROTOCOLS_IMP) { + if (loop_num_ret > MAX_PROTOCOLS_IMP - tot_num_ret) { dev_err(dev, "No. of Protocol > MAX_PROTOCOLS_IMP"); break; } diff --git a/drivers/firmware/dmi-sysfs.c b/drivers/firmware/dmi-sysfs.c index ecf2eeb5f6f9..5d6b497d54d0 100644 --- a/drivers/firmware/dmi-sysfs.c +++ b/drivers/firmware/dmi-sysfs.c @@ -602,7 +602,7 @@ static void __init dmi_sysfs_register_handle(const struct dmi_header *dh, "%d-%d", dh->type, entry->instance); if (*ret) { - kfree(entry); + kobject_put(&entry->kobj); return; } diff --git a/drivers/firmware/qcom/tz_log.c b/drivers/firmware/qcom/tz_log.c index d346d5576807..e8697d7b2bf6 100644 --- a/drivers/firmware/qcom/tz_log.c +++ b/drivers/firmware/qcom/tz_log.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include @@ -1238,8 +1239,15 @@ static ssize_t tzdbgfs_read(struct file *file, char __user *buf, struct seq_file *seq = file->private_data; int tz_id = TZDBG_STATS_MAX; - if (seq) - tz_id = *(int *)(seq->private); + if (seq) { + if (seq->private) + tz_id = *(int *)(seq->private); + else { + pr_err("%s: Seq data private null unable to proceed\n", + __func__); + return 0; + } + } else { pr_err("%s: Seq data null unable to proceed\n", __func__); return 0; diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index fccc4e51108b..547fabb681ce 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -422,3 +422,7 @@ config DRM_PANEL_ORIENTATION_QUIRKS config DRM_LIB_RANDOM bool default n + +config MI_DRM_OPT + bool "Enable xiaomi early wakeup optimization" + default n diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 023309296bfc..e667bcf64bc7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -110,7 +110,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs int ret; if (cs->in.num_chunks == 0) - return 0; + return -EINVAL; chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); if (!chunk_array) diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c index cb79a93c2eb7..91504eccc60c 100644 --- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c @@ -1610,19 +1610,7 @@ static int kv_update_samu_dpm(struct amdgpu_device *adev, bool gate) static u8 kv_get_acp_boot_level(struct amdgpu_device *adev) { - u8 i; - struct amdgpu_clock_voltage_dependency_table *table = - &adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table; - - for (i = 0; i < table->count; i++) { - if (table->entries[i].clk >= 0) /* XXX */ - break; - } - - if (i >= table->count) - i = table->count - 1; - - return i; + return 0; } static void kv_update_acp_boot_level(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c index 1de96995e690..9f811051ceb0 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c @@ -7247,17 +7247,15 @@ static int si_parse_power_table(struct amdgpu_device *adev) if (!adev->pm.dpm.ps) return -ENOMEM; power_state_offset = (u8 *)state_array->states; - for (i = 0; i < state_array->ucNumEntries; i++) { + for (adev->pm.dpm.num_ps = 0, i = 0; i < state_array->ucNumEntries; i++) { u8 *idx; power_state = (union pplib_power_state *)power_state_offset; non_clock_array_index = power_state->v2.nonClockInfoIndex; non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) &non_clock_info_array->nonClockInfo[non_clock_array_index]; ps = kzalloc(sizeof(struct si_ps), GFP_KERNEL); - if (ps == NULL) { - kfree(adev->pm.dpm.ps); + if (ps == NULL) return -ENOMEM; - } adev->pm.dpm.ps[i].ps_priv = ps; si_parse_pplib_non_clock_info(adev, &adev->pm.dpm.ps[i], non_clock_info, @@ -7279,8 +7277,8 @@ static int si_parse_power_table(struct amdgpu_device *adev) k++; } power_state_offset += 2 + power_state->v2.ucNumDPMLevels; + adev->pm.dpm.num_ps++; } - adev->pm.dpm.num_ps = state_array->ucNumEntries; /* fill in the vce power states */ for (i = 0; i < adev->pm.dpm.num_of_vce_states; i++) { diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c index e7ddd3e3db92..b6e7cc9082ca 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c +++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c @@ -1225,6 +1225,7 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id) return 0; err_unregister_cec: + cec_unregister_adapter(adv7511->cec_adap); i2c_unregister_device(adv7511->i2c_cec); if (adv7511->cec_clk) clk_disable_unprepare(adv7511->cec_clk); diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c index 84abf5d6f760..e21c7673cd5b 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c +++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c @@ -1514,8 +1514,19 @@ static ssize_t analogix_dpaux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) { struct analogix_dp_device *dp = to_dp(aux); + int ret; + + pm_runtime_get_sync(dp->dev); + + ret = analogix_dp_detect_hpd(dp); + if (ret) + goto out; - return analogix_dp_transfer(dp, msg); + ret = analogix_dp_transfer(dp, msg); +out: + pm_runtime_put(dp->dev); + + return ret; } struct analogix_dp_device * diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c index d8aca426a157..1dc9a027f26d 100644 --- a/drivers/gpu/drm/drm_bridge.c +++ b/drivers/gpu/drm/drm_bridge.c @@ -271,14 +271,18 @@ void drm_bridge_post_disable(struct drm_bridge *bridge) if (!bridge) return; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (bridge->is_dsi_drm_bridge) mutex_lock(&bridge->lock); +#endif if (bridge->funcs->post_disable) bridge->funcs->post_disable(bridge); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (bridge->is_dsi_drm_bridge) mutex_unlock(&bridge->lock); +#endif drm_bridge_post_disable(bridge->next); } @@ -328,14 +332,18 @@ void drm_bridge_pre_enable(struct drm_bridge *bridge) drm_bridge_pre_enable(bridge->next); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (bridge->is_dsi_drm_bridge) mutex_lock(&bridge->lock); +#endif if (bridge->funcs->pre_enable) bridge->funcs->pre_enable(bridge); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (bridge->is_dsi_drm_bridge) mutex_unlock(&bridge->lock); +#endif } EXPORT_SYMBOL(drm_bridge_pre_enable); diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index bdd1b6cb7e12..fa2d9392bbcc 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -1724,9 +1724,6 @@ struct edid *drm_do_get_edid(struct drm_connector *connector, connector_bad_edid(connector, edid, edid[0x7e] + 1); - edid[EDID_LENGTH-1] += edid[0x7e] - valid_extensions; - edid[0x7e] = valid_extensions; - new = kmalloc_array(valid_extensions + 1, EDID_LENGTH, GFP_KERNEL); if (!new) @@ -1743,6 +1740,9 @@ struct edid *drm_do_get_edid(struct drm_connector *connector, base += EDID_LENGTH; } + new[EDID_LENGTH - 1] += new[0x7e] - valid_extensions; + new[0x7e] = valid_extensions; + kfree(edid); edid = new; } diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c index 2411b6de055e..425e76e39b3b 100644 --- a/drivers/gpu/drm/drm_plane.c +++ b/drivers/gpu/drm/drm_plane.c @@ -177,6 +177,13 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, if (WARN_ON(config->num_total_plane >= 32)) return -EINVAL; + /* + * First driver to need more than 64 formats needs to fix this. Each + * format is encoded as a bit and the current code only supports a u64. + */ + if (WARN_ON(format_count > 64)) + return -EINVAL; + WARN_ON(drm_drv_uses_atomic_modeset(dev) && (!funcs->atomic_destroy_state || !funcs->atomic_duplicate_state)); @@ -198,13 +205,6 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, return -ENOMEM; } - /* - * First driver to need more than 64 formats needs to fix this. Each - * format is encoded as a bit and the current code only supports a u64. - */ - if (WARN_ON(format_count > 64)) - return -EINVAL; - if (format_modifiers) { const uint64_t *temp_modifiers = format_modifiers; while (*temp_modifiers++ != DRM_FORMAT_MOD_INVALID) diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c index 8762efaef283..987f202c70ff 100644 --- a/drivers/gpu/drm/gma500/psb_intel_display.c +++ b/drivers/gpu/drm/gma500/psb_intel_display.c @@ -543,14 +543,15 @@ void psb_intel_crtc_init(struct drm_device *dev, int pipe, struct drm_crtc *psb_intel_get_crtc_from_pipe(struct drm_device *dev, int pipe) { - struct drm_crtc *crtc = NULL; + struct drm_crtc *crtc; list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { struct gma_crtc *gma_crtc = to_gma_crtc(crtc); + if (gma_crtc->pipe == pipe) - break; + return crtc; } - return crtc; + return NULL; } int gma_connector_clones(struct drm_device *dev, int type_mask) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 1847faa45d37..af8865281b47 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -2814,7 +2814,7 @@ hsw_compute_linetime_wm(const struct intel_crtc_state *cstate) } static void intel_read_wm_latency(struct drm_i915_private *dev_priv, - uint16_t wm[8]) + uint16_t wm[]) { if (INTEL_GEN(dev_priv) >= 9) { uint32_t val; diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c index ff34f9bb55a1..824c90dca730 100644 --- a/drivers/gpu/drm/imx/ipuv3-crtc.c +++ b/drivers/gpu/drm/imx/ipuv3-crtc.c @@ -71,7 +71,7 @@ static void ipu_crtc_disable_planes(struct ipu_crtc *ipu_crtc, drm_atomic_crtc_state_for_each_plane(plane, old_crtc_state) { if (plane == &ipu_crtc->plane[0]->base) disable_full = true; - if (&ipu_crtc->plane[1] && plane == &ipu_crtc->plane[1]->base) + if (ipu_crtc->plane[1] && plane == &ipu_crtc->plane[1]->base) disable_partial = true; } diff --git a/drivers/gpu/drm/mediatek/mtk_cec.c b/drivers/gpu/drm/mediatek/mtk_cec.c index 5ce84d0dbf81..acbf878083d7 100644 --- a/drivers/gpu/drm/mediatek/mtk_cec.c +++ b/drivers/gpu/drm/mediatek/mtk_cec.c @@ -92,7 +92,7 @@ static void mtk_cec_mask(struct mtk_cec *cec, unsigned int offset, u32 tmp = readl(cec->regs + offset) & ~mask; tmp |= val & mask; - writel(val, cec->regs + offset); + writel(tmp, cec->regs + offset); } void mtk_cec_set_hpd_event(struct device *dev, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index 52474dcd2573..c88bb92282df 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -657,8 +657,10 @@ static void _dpu_kms_hw_destroy(struct dpu_kms *dpu_kms) for (i = 0; i < dpu_kms->catalog->vbif_count; i++) { u32 vbif_idx = dpu_kms->catalog->vbif[i].id; - if ((vbif_idx < VBIF_MAX) && dpu_kms->hw_vbif[vbif_idx]) + if ((vbif_idx < VBIF_MAX) && dpu_kms->hw_vbif[vbif_idx]) { dpu_hw_vbif_destroy(dpu_kms->hw_vbif[vbif_idx]); + dpu_kms->hw_vbif[vbif_idx] = NULL; + } } } diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c index aa28a43ff842..b8a01c95b831 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c @@ -537,9 +537,15 @@ int mdp5_crtc_setup_pipeline(struct drm_crtc *crtc, if (ret) return ret; - mdp5_mixer_release(new_crtc_state->state, old_mixer); + ret = mdp5_mixer_release(new_crtc_state->state, old_mixer); + if (ret) + return ret; + if (old_r_mixer) { - mdp5_mixer_release(new_crtc_state->state, old_r_mixer); + ret = mdp5_mixer_release(new_crtc_state->state, old_r_mixer); + if (ret) + return ret; + if (!need_right_mixer) pipeline->r_mixer = NULL; } @@ -905,8 +911,10 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc, ret = msm_gem_get_iova(cursor_bo, kms->aspace, &mdp5_crtc->cursor.iova); - if (ret) + if (ret) { + drm_gem_object_put(cursor_bo); return -EINVAL; + } pm_runtime_get_sync(&pdev->dev); diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c index 113e6b569562..9c8275300b55 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c @@ -127,21 +127,28 @@ int mdp5_mixer_assign(struct drm_atomic_state *s, struct drm_crtc *crtc, return 0; } -void mdp5_mixer_release(struct drm_atomic_state *s, struct mdp5_hw_mixer *mixer) +int mdp5_mixer_release(struct drm_atomic_state *s, struct mdp5_hw_mixer *mixer) { struct mdp5_global_state *global_state = mdp5_get_global_state(s); - struct mdp5_hw_mixer_state *new_state = &global_state->hwmixer; + struct mdp5_hw_mixer_state *new_state; if (!mixer) - return; + return 0; + + if (IS_ERR(global_state)) + return PTR_ERR(global_state); + + new_state = &global_state->hwmixer; if (WARN_ON(!new_state->hwmixer_to_crtc[mixer->idx])) - return; + return -EINVAL; DBG("%s: release from crtc %s", mixer->name, new_state->hwmixer_to_crtc[mixer->idx]->name); new_state->hwmixer_to_crtc[mixer->idx] = NULL; + + return 0; } void mdp5_mixer_destroy(struct mdp5_hw_mixer *mixer) diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h index 9be94f567fbd..6bcb0b544665 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h @@ -41,7 +41,7 @@ void mdp5_mixer_destroy(struct mdp5_hw_mixer *lm); int mdp5_mixer_assign(struct drm_atomic_state *s, struct drm_crtc *crtc, uint32_t caps, struct mdp5_hw_mixer **mixer, struct mdp5_hw_mixer **r_mixer); -void mdp5_mixer_release(struct drm_atomic_state *s, - struct mdp5_hw_mixer *mixer); +int mdp5_mixer_release(struct drm_atomic_state *s, + struct mdp5_hw_mixer *mixer); #endif /* __MDP5_LM_H__ */ diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c index 1ef26bc63163..88de12225582 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c @@ -130,18 +130,23 @@ int mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane, return 0; } -void mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe) +int mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe) { struct msm_drm_private *priv = s->dev->dev_private; struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms)); struct mdp5_global_state *state = mdp5_get_global_state(s); - struct mdp5_hw_pipe_state *new_state = &state->hwpipe; + struct mdp5_hw_pipe_state *new_state; if (!hwpipe) - return; + return 0; + + if (IS_ERR(state)) + return PTR_ERR(state); + + new_state = &state->hwpipe; if (WARN_ON(!new_state->hwpipe_to_plane[hwpipe->idx])) - return; + return -EINVAL; DBG("%s: release from plane %s", hwpipe->name, new_state->hwpipe_to_plane[hwpipe->idx]->name); @@ -152,6 +157,8 @@ void mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe) } new_state->hwpipe_to_plane[hwpipe->idx] = NULL; + + return 0; } void mdp5_pipe_destroy(struct mdp5_hw_pipe *hwpipe) diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h index bb2b0ac7aa2b..072db6a99b85 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h @@ -48,7 +48,7 @@ int mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane, uint32_t caps, uint32_t blkcfg, struct mdp5_hw_pipe **hwpipe, struct mdp5_hw_pipe **r_hwpipe); -void mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe); +int mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe); struct mdp5_hw_pipe *mdp5_pipe_init(enum mdp5_pipe pipe, uint32_t reg_offset, uint32_t caps); diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c index 3d8eaa25bea0..501d7989b9a5 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c @@ -402,12 +402,24 @@ static int mdp5_plane_atomic_check_with_state(struct drm_crtc_state *crtc_state, mdp5_state->r_hwpipe = NULL; - mdp5_pipe_release(state->state, old_hwpipe); - mdp5_pipe_release(state->state, old_right_hwpipe); + ret = mdp5_pipe_release(state->state, old_hwpipe); + if (ret) + return ret; + + ret = mdp5_pipe_release(state->state, old_right_hwpipe); + if (ret) + return ret; + } } else { - mdp5_pipe_release(state->state, mdp5_state->hwpipe); - mdp5_pipe_release(state->state, mdp5_state->r_hwpipe); + ret = mdp5_pipe_release(state->state, mdp5_state->hwpipe); + if (ret) + return ret; + + ret = mdp5_pipe_release(state->state, mdp5_state->r_hwpipe); + if (ret) + return ret; + mdp5_state->hwpipe = mdp5_state->r_hwpipe = NULL; } diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index 9abfb19ea7ed..56cfa0a03fd5 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -1354,10 +1354,10 @@ static int dsi_cmds2buf_tx(struct msm_dsi_host *msm_host, dsi_get_bpp(msm_host->format) / 8; len = dsi_cmd_dma_add(msm_host, msg); - if (!len) { + if (len < 0) { pr_err("%s: failed to add cmd type = 0x%x\n", __func__, msg->type); - return -EINVAL; + return len; } /* for video mode, do not send cmds more than @@ -1376,10 +1376,14 @@ static int dsi_cmds2buf_tx(struct msm_dsi_host *msm_host, } ret = dsi_cmd_dma_tx(msm_host, len); - if (ret < len) { - pr_err("%s: cmd dma tx failed, type=0x%x, data0=0x%x, len=%d\n", - __func__, msg->type, (*(u8 *)(msg->tx_buf)), len); - return -ECOMM; + if (ret < 0) { + pr_err("%s: cmd dma tx failed, type=0x%x, data0=0x%x, len=%d, ret=%d\n", + __func__, msg->type, (*(u8 *)(msg->tx_buf)), len, ret); + return ret; + } else if (ret < len) { + pr_err("%s: cmd dma tx failed, type=0x%x, data0=0x%x, ret=%d len=%d\n", + __func__, msg->type, (*(u8 *)(msg->tx_buf)), ret, len); + return -EIO; } return len; @@ -2105,9 +2109,12 @@ int msm_dsi_host_cmd_rx(struct mipi_dsi_host *host, } ret = dsi_cmds2buf_tx(msm_host, msg); - if (ret < msg->tx_len) { + if (ret < 0) { pr_err("%s: Read cmd Tx failed, %d\n", __func__, ret); return ret; + } else if (ret < msg->tx_len) { + pr_err("%s: Read cmd Tx failed, too short: %d\n", __func__, ret); + return -ECOMM; } /* diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c index 33e083f71a17..0b7cb90d2826 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi.c @@ -148,6 +148,10 @@ static struct hdmi *msm_hdmi_init(struct platform_device *pdev) /* HDCP needs physical address of hdmi register */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, config->mmio_name); + if (!res) { + ret = -EINVAL; + goto fail; + } hdmi->mmio_phy_addr = res->start; hdmi->qfprom_mmio = msm_ioremap(pdev, diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c index 13403c6da6c7..7e4664968106 100644 --- a/drivers/gpu/drm/msm/msm_gem_prime.c +++ b/drivers/gpu/drm/msm/msm_gem_prime.c @@ -26,7 +26,7 @@ struct sg_table *msm_gem_prime_get_sg_table(struct drm_gem_object *obj) int npages = obj->size >> PAGE_SHIFT; if (WARN_ON(!msm_obj->pages)) /* should have already pinned! */ - return NULL; + return ERR_PTR(-ENOMEM); return drm_prime_pages_to_sg(msm_obj->pages, npages); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c index ba6a868d4c95..28cccac39c19 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c @@ -134,10 +134,10 @@ nvkm_cstate_find_best(struct nvkm_clk *clk, struct nvkm_pstate *pstate, list_for_each_entry_from_reverse(cstate, &pstate->list, head) { if (nvkm_cstate_valid(clk, cstate, max_volt, clk->temp)) - break; + return cstate; } - return cstate; + return NULL; } static struct nvkm_cstate * @@ -168,6 +168,8 @@ nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei) if (!list_empty(&pstate->list)) { cstate = nvkm_cstate_get(clk, pstate, cstatei); cstate = nvkm_cstate_find_best(clk, pstate, cstate); + if (!cstate) + return -EINVAL; } else { cstate = &pstate->base; } diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c index b9927101e845..47240983f360 100644 --- a/drivers/gpu/drm/radeon/radeon_connectors.c +++ b/drivers/gpu/drm/radeon/radeon_connectors.c @@ -476,6 +476,8 @@ static struct drm_display_mode *radeon_fp_native_mode(struct drm_encoder *encode native_mode->vdisplay != 0 && native_mode->clock != 0) { mode = drm_mode_duplicate(dev, native_mode); + if (!mode) + return NULL; mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER; drm_mode_set_name(mode); @@ -490,6 +492,8 @@ static struct drm_display_mode *radeon_fp_native_mode(struct drm_encoder *encode * simpler. */ mode = drm_cvt_mode(dev, native_mode->hdisplay, native_mode->vdisplay, 60, true, false, false); + if (!mode) + return NULL; mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER; DRM_DEBUG_KMS("Adding cvt approximation of native panel mode %s\n", mode->name); } diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c index 873624a11ce8..c0b647435974 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c @@ -1595,10 +1595,10 @@ static int vop_bind(struct device *dev, struct device *master, void *data) vop_win_init(vop); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - vop->len = resource_size(res); vop->regs = devm_ioremap_resource(dev, res); if (IS_ERR(vop->regs)) return PTR_ERR(vop->regs); + vop->len = resource_size(res); vop->regsbak = devm_kzalloc(dev, vop->len, GFP_KERNEL); if (!vop->regsbak) diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c index 6e23c50168f9..3b56558a5d65 100644 --- a/drivers/gpu/drm/vc4/vc4_txp.c +++ b/drivers/gpu/drm/vc4/vc4_txp.c @@ -311,12 +311,18 @@ static void vc4_txp_connector_atomic_commit(struct drm_connector *conn, if (WARN_ON(i == ARRAY_SIZE(drm_fmts))) return; - ctrl = TXP_GO | TXP_VSTART_AT_EOF | TXP_EI | + ctrl = TXP_GO | TXP_EI | VC4_SET_FIELD(0xf, TXP_BYTE_ENABLE) | VC4_SET_FIELD(txp_fmts[i], TXP_FORMAT); if (fb->format->has_alpha) ctrl |= TXP_ALPHA_ENABLE; + else + /* + * If TXP_ALPHA_ENABLE isn't set and TXP_ALPHA_INVERT is, the + * hardware will force the output padding to be 0xff. + */ + ctrl |= TXP_ALPHA_INVERT; gem = drm_fb_cma_get_gem_obj(fb, 0); TXP_WRITE(TXP_DST_PTR, gem->paddr + fb->offsets[0]); diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c index 9bef800d9073..158ff98e0d79 100644 --- a/drivers/gpu/drm/virtio/virtgpu_display.c +++ b/drivers/gpu/drm/virtio/virtgpu_display.c @@ -187,6 +187,8 @@ static int virtio_gpu_conn_get_modes(struct drm_connector *connector) DRM_DEBUG("add mode: %dx%d\n", width, height); mode = drm_cvt_mode(connector->dev, width, height, 60, false, false, false); + if (!mode) + return count; mode->type |= DRM_MODE_TYPE_PREFERRED; drm_mode_probed_add(connector, mode); count++; diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index 13e00067a876..9665837ffdea 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -2677,15 +2677,6 @@ static int kgsl_setup_anon_useraddr(struct kgsl_pagetable *pagetable, } #ifdef CONFIG_DMA_SHARED_BUFFER -static int match_file(const void *p, struct file *file, unsigned int fd) -{ - /* - * We must return fd + 1 because iterate_fd stops searching on - * non-zero return, but 0 is a valid fd. - */ - return (p == file) ? (fd + 1) : 0; -} - static void _setup_cache_mode(struct kgsl_mem_entry *entry, struct vm_area_struct *vma) { @@ -2723,8 +2714,6 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, vma = find_vma(current->mm, hostptr); if (vma && vma->vm_file) { - int fd; - ret = check_vma_flags(vma, entry->memdesc.flags); if (ret) { up_read(¤t->mm->mmap_sem); @@ -2740,10 +2729,13 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, return -EFAULT; } - /* Look for the fd that matches this the vma file */ - fd = iterate_fd(current->files, 0, match_file, vma->vm_file); - if (fd != 0) - dmabuf = dma_buf_get(fd - 1); + /* + * Take a refcount because dma_buf_put() decrements the + * refcount + */ + get_file(vma->vm_file); + + dmabuf = vma->vm_file->private_data; } if (IS_ERR_OR_NULL(dmabuf)) { diff --git a/drivers/hid/hid-elan.c b/drivers/hid/hid-elan.c index 7139227edb28..63077fb23026 100644 --- a/drivers/hid/hid-elan.c +++ b/drivers/hid/hid-elan.c @@ -192,7 +192,6 @@ static int elan_input_configured(struct hid_device *hdev, struct hid_input *hi) ret = input_mt_init_slots(input, ELAN_MAX_FINGERS, INPUT_MT_POINTER); if (ret) { hid_err(hdev, "Failed to init elan MT slots: %d\n", ret); - input_free_device(input); return ret; } @@ -204,7 +203,6 @@ static int elan_input_configured(struct hid_device *hdev, struct hid_input *hi) hid_err(hdev, "Failed to register elan input device: %d\n", ret); input_mt_destroy_slots(input); - input_free_device(input); return ret; } diff --git a/drivers/hid/hid-led.c b/drivers/hid/hid-led.c index d3e1ab162f7c..7fc5982a0ca4 100644 --- a/drivers/hid/hid-led.c +++ b/drivers/hid/hid-led.c @@ -369,7 +369,7 @@ static const struct hidled_config hidled_configs[] = { .type = DREAM_CHEEKY, .name = "Dream Cheeky Webmail Notifier", .short_name = "dream_cheeky", - .max_brightness = 31, + .max_brightness = 63, .num_leds = 1, .report_size = 9, .report_type = RAW_REQUEST, diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index c20945ed1dc1..e99286258f62 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -2111,6 +2111,9 @@ static const struct hid_device_id mt_devices[] = { { .driver_data = MT_CLS_GOOGLE, HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_TOUCH_ROSE) }, + { .driver_data = MT_CLS_GOOGLE, + HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, USB_VENDOR_ID_GOOGLE, + USB_DEVICE_ID_GOOGLE_WHISKERS) }, /* Generic MT device */ { HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH, HID_ANY_ID, HID_ANY_ID) }, diff --git a/drivers/hwtracing/coresight/coresight-cpu-debug.c b/drivers/hwtracing/coresight/coresight-cpu-debug.c index e8819d750938..a4eba09691b4 100644 --- a/drivers/hwtracing/coresight/coresight-cpu-debug.c +++ b/drivers/hwtracing/coresight/coresight-cpu-debug.c @@ -379,9 +379,10 @@ static int debug_notifier_call(struct notifier_block *self, int cpu; struct debug_drvdata *drvdata; - mutex_lock(&debug_lock); + /* Bail out if we can't acquire the mutex or the functionality is off */ + if (!mutex_trylock(&debug_lock)) + return NOTIFY_DONE; - /* Bail out if the functionality is disabled */ if (!debug_enable) goto skip_dump; @@ -400,7 +401,7 @@ static int debug_notifier_call(struct notifier_block *self, skip_dump: mutex_unlock(&debug_lock); - return 0; + return NOTIFY_DONE; } static struct notifier_block debug_notifier = { diff --git a/drivers/i2c/busses/i2c-at91.c b/drivers/i2c/busses/i2c-at91.c index d51bf536bdf7..29947eb905fb 100644 --- a/drivers/i2c/busses/i2c-at91.c +++ b/drivers/i2c/busses/i2c-at91.c @@ -757,6 +757,7 @@ static int at91_twi_xfer(struct i2c_adapter *adap, struct i2c_msg *msg, int num) unsigned int_addr_flag = 0; struct i2c_msg *m_start = msg; bool is_read; + u8 *dma_buf = NULL; dev_dbg(&adap->dev, "at91_xfer: processing %d messages:\n", num); @@ -804,7 +805,17 @@ static int at91_twi_xfer(struct i2c_adapter *adap, struct i2c_msg *msg, int num) dev->msg = m_start; dev->recv_len_abort = false; + if (dev->use_dma) { + dma_buf = i2c_get_dma_safe_msg_buf(m_start, 1); + if (!dma_buf) { + ret = -ENOMEM; + goto out; + } + dev->buf = dma_buf; + } + ret = at91_do_twi_transfer(dev); + i2c_put_dma_safe_msg_buf(dma_buf, m_start, !ret); ret = (ret < 0) ? ret : num; out: diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index c5475bb4fae6..2150afdcc083 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -511,7 +511,7 @@ static void cdns_i2c_master_reset(struct i2c_adapter *adap) static int cdns_i2c_process_msg(struct cdns_i2c *id, struct i2c_msg *msg, struct i2c_adapter *adap) { - unsigned long time_left; + unsigned long time_left, msg_timeout; u32 reg; id->p_msg = msg; @@ -536,8 +536,16 @@ static int cdns_i2c_process_msg(struct cdns_i2c *id, struct i2c_msg *msg, else cdns_i2c_msend(id); + /* Minimal time to execute this message */ + msg_timeout = msecs_to_jiffies((1000 * msg->len * BITS_PER_BYTE) / id->i2c_clk); + /* Plus some wiggle room */ + msg_timeout += msecs_to_jiffies(500); + + if (msg_timeout < adap->timeout) + msg_timeout = adap->timeout; + /* Wait for the signal of completion */ - time_left = wait_for_completion_timeout(&id->xfer_done, adap->timeout); + time_left = wait_for_completion_timeout(&id->xfer_done, msg_timeout); if (time_left == 0) { cdns_i2c_master_reset(adap); dev_err(id->adap.dev.parent, diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c index 0d1c3ec8cb40..80796061102f 100644 --- a/drivers/i2c/busses/i2c-ismt.c +++ b/drivers/i2c/busses/i2c-ismt.c @@ -80,6 +80,7 @@ #define ISMT_DESC_ENTRIES 2 /* number of descriptor entries */ #define ISMT_MAX_RETRIES 3 /* number of SMBus retries to attempt */ +#define ISMT_LOG_ENTRIES 3 /* number of interrupt cause log entries */ /* Hardware Descriptor Constants - Control Field */ #define ISMT_DESC_CWRL 0x01 /* Command/Write Length */ @@ -173,6 +174,8 @@ struct ismt_priv { u8 head; /* ring buffer head pointer */ struct completion cmp; /* interrupt completion */ u8 buffer[I2C_SMBUS_BLOCK_MAX + 16]; /* temp R/W data buffer */ + dma_addr_t log_dma; + u32 *log; }; /** @@ -406,6 +409,9 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr, memset(desc, 0, sizeof(struct ismt_desc)); desc->tgtaddr_rw = ISMT_DESC_ADDR_RW(addr, read_write); + /* Always clear the log entries */ + memset(priv->log, 0, ISMT_LOG_ENTRIES * sizeof(u32)); + /* Initialize common control bits */ if (likely(pci_dev_msi_enabled(priv->pci_dev))) desc->control = ISMT_DESC_INT | ISMT_DESC_FAIR; @@ -695,6 +701,8 @@ static void ismt_hw_init(struct ismt_priv *priv) /* initialize the Master Descriptor Base Address (MDBA) */ writeq(priv->io_rng_dma, priv->smba + ISMT_MSTR_MDBA); + writeq(priv->log_dma, priv->smba + ISMT_GR_SMTICL); + /* initialize the Master Control Register (MCTRL) */ writel(ISMT_MCTRL_MEIE, priv->smba + ISMT_MSTR_MCTRL); @@ -784,6 +792,12 @@ static int ismt_dev_init(struct ismt_priv *priv) priv->head = 0; init_completion(&priv->cmp); + priv->log = dmam_alloc_coherent(&priv->pci_dev->dev, + ISMT_LOG_ENTRIES * sizeof(u32), + &priv->log_dma, GFP_KERNEL); + if (!priv->log) + return -ENOMEM; + return 0; } diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c index 19f8eec38717..107aeb8b54da 100644 --- a/drivers/i2c/busses/i2c-thunderx-pcidrv.c +++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c @@ -208,6 +208,7 @@ static int thunder_i2c_probe_pci(struct pci_dev *pdev, i2c->adap.bus_recovery_info = &octeon_i2c_recovery_info; i2c->adap.dev.parent = dev; i2c->adap.dev.of_node = pdev->dev.of_node; + i2c->adap.dev.fwnode = dev->fwnode; snprintf(i2c->adap.name, sizeof(i2c->adap.name), "Cavium ThunderX i2c adapter at %s", dev_name(dev)); i2c_set_adapdata(&i2c->adap, i2c); diff --git a/drivers/iio/adc/sc27xx_adc.c b/drivers/iio/adc/sc27xx_adc.c index 2b60efea0c39..4095e1f7d756 100644 --- a/drivers/iio/adc/sc27xx_adc.c +++ b/drivers/iio/adc/sc27xx_adc.c @@ -35,8 +35,8 @@ /* Bits and mask definition for SC27XX_ADC_CH_CFG register */ #define SC27XX_ADC_CHN_ID_MASK GENMASK(4, 0) -#define SC27XX_ADC_SCALE_MASK GENMASK(10, 8) -#define SC27XX_ADC_SCALE_SHIFT 8 +#define SC27XX_ADC_SCALE_MASK GENMASK(10, 9) +#define SC27XX_ADC_SCALE_SHIFT 9 /* Bits definitions for SC27XX_ADC_INT_EN registers */ #define SC27XX_ADC_IRQ_EN BIT(0) diff --git a/drivers/iio/dummy/iio_simple_dummy.c b/drivers/iio/dummy/iio_simple_dummy.c index 62052479c349..f970b4388dda 100644 --- a/drivers/iio/dummy/iio_simple_dummy.c +++ b/drivers/iio/dummy/iio_simple_dummy.c @@ -571,10 +571,9 @@ static struct iio_sw_device *iio_dummy_probe(const char *name) struct iio_sw_device *swd; swd = kzalloc(sizeof(*swd), GFP_KERNEL); - if (!swd) { - ret = -ENOMEM; - goto error_kzalloc; - } + if (!swd) + return ERR_PTR(-ENOMEM); + /* * Allocate an IIO device. * @@ -586,7 +585,7 @@ static struct iio_sw_device *iio_dummy_probe(const char *name) indio_dev = iio_device_alloc(sizeof(*st)); if (!indio_dev) { ret = -ENOMEM; - goto error_ret; + goto error_free_swd; } st = iio_priv(indio_dev); @@ -617,6 +616,10 @@ static struct iio_sw_device *iio_dummy_probe(const char *name) * indio_dev->name = spi_get_device_id(spi)->name; */ indio_dev->name = kstrdup(name, GFP_KERNEL); + if (!indio_dev->name) { + ret = -ENOMEM; + goto error_free_device; + } /* Provide description of available channels */ indio_dev->channels = iio_dummy_channels; @@ -633,7 +636,7 @@ static struct iio_sw_device *iio_dummy_probe(const char *name) ret = iio_simple_dummy_events_register(indio_dev); if (ret < 0) - goto error_free_device; + goto error_free_name; ret = iio_simple_dummy_configure_buffer(indio_dev); if (ret < 0) @@ -650,11 +653,12 @@ static struct iio_sw_device *iio_dummy_probe(const char *name) iio_simple_dummy_unconfigure_buffer(indio_dev); error_unregister_events: iio_simple_dummy_events_unregister(indio_dev); +error_free_name: + kfree(indio_dev->name); error_free_device: iio_device_free(indio_dev); -error_ret: +error_free_swd: kfree(swd); -error_kzalloc: return ERR_PTR(ret); } diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index adeb259458de..64ee11542a56 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -308,6 +308,8 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from) unsigned long dim = from->nr_segs; int idx; + if (!HFI1_CAP_IS_KSET(SDMA)) + return -EINVAL; idx = srcu_read_lock(&fd->pq_srcu); pq = srcu_dereference(fd->pq, &fd->pq_srcu); if (!cq || !pq) { diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index d9890ca1d70a..c3abf7811746 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -535,7 +535,7 @@ void set_link_ipg(struct hfi1_pportdata *ppd) u16 shift, mult; u64 src; u32 current_egress_rate; /* Mbits /sec */ - u32 max_pkt_time; + u64 max_pkt_time; /* * max_pkt_time is the maximum packet egress time in units * of the fabric clock period 1/(805 MHz). diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c index 38258de75a94..33ff9eca28f6 100644 --- a/drivers/infiniband/hw/hfi1/sdma.c +++ b/drivers/infiniband/hw/hfi1/sdma.c @@ -1313,11 +1313,13 @@ void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) kvfree(sde->tx_ring); sde->tx_ring = NULL; } - spin_lock_irq(&dd->sde_map_lock); - sdma_map_free(rcu_access_pointer(dd->sdma_map)); - RCU_INIT_POINTER(dd->sdma_map, NULL); - spin_unlock_irq(&dd->sde_map_lock); - synchronize_rcu(); + if (rcu_access_pointer(dd->sdma_map)) { + spin_lock_irq(&dd->sde_map_lock); + sdma_map_free(rcu_access_pointer(dd->sdma_map)); + RCU_INIT_POINTER(dd->sdma_map, NULL); + spin_unlock_irq(&dd->sde_map_lock); + synchronize_rcu(); + } kfree(dd->per_sdma); dd->per_sdma = NULL; diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 63db49144f62..4008ab2da052 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -680,7 +680,7 @@ int rxe_requester(void *arg) opcode = next_opcode(qp, wqe, wqe->wr.opcode); if (unlikely(opcode < 0)) { wqe->status = IB_WC_LOC_QP_OP_ERR; - goto exit; + goto err; } mask = rxe_opcode[opcode].mask; diff --git a/drivers/input/fingerprint/fpc_tee/fpc1020_tee.c b/drivers/input/fingerprint/fpc_tee/fpc1020_tee.c index c32c4d67bfbc..59a6ee981063 100644 --- a/drivers/input/fingerprint/fpc_tee/fpc1020_tee.c +++ b/drivers/input/fingerprint/fpc_tee/fpc1020_tee.c @@ -24,8 +24,9 @@ * modify it under the terms of the GNU General Public License Version 2 * as published by the Free Software Foundation. */ - -#define FPC_DRM_INTERFACE_WA +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +#define FPC_DRM_INTERFACE +#endif #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -43,8 +44,8 @@ #include #include #include +#if IS_ENABLED(CONFIG_MI_DRM_OPT) #include -#ifndef FPC_DRM_INTERFACE_WA #include #endif @@ -52,7 +53,10 @@ #define FPC_GPIO_NO_DEFINED -2 #define FPC_GPIO_REQUEST_FAIL -3 #define FPC_TTW_HOLD_TIME 2000 + +#if IS_ENABLED(CONFIG_MI_DRM_OPT) #define FP_UNLOCK_REJECTION_TIMEOUT (FPC_TTW_HOLD_TIME - 500) +#endif #define RESET_LOW_SLEEP_MIN_US 5000 #define RESET_LOW_SLEEP_MAX_US (RESET_LOW_SLEEP_MIN_US + 100) @@ -118,12 +122,12 @@ struct fpc1020_data { atomic_t wakeup_enabled; /* Used both in ISR and non-ISR */ int irqf; -#ifndef FPC_DRM_INTERFACE_WA +#if IS_ENABLED(CONFIG_MI_DRM_OPT) struct notifier_block fb_notifier; -#endif bool fb_black; bool wait_finger_down; struct work_struct work; +#endif }; static int reset_gpio_res(struct fpc1020_data *fpc1020); @@ -743,6 +747,7 @@ static ssize_t irq_ack(struct device *dev, static DEVICE_ATTR(irq, S_IRUSR | S_IWUSR, irq_get, irq_ack); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) static ssize_t fingerdown_wait_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -762,6 +767,7 @@ static ssize_t fingerdown_wait_set(struct device *dev, } static DEVICE_ATTR(fingerdown_wait, S_IWUSR, NULL, fingerdown_wait_set); +#endif static ssize_t vendor_update(struct device *dev, struct device_attribute *attr, @@ -835,7 +841,9 @@ static struct attribute *attributes[] = { &dev_attr_irq_enable.attr, &dev_attr_irq.attr, &dev_attr_vendor.attr, +#if IS_ENABLED(CONFIG_MI_DRM_OPT) &dev_attr_fingerdown_wait.attr, +#endif &dev_attr_power_cfg.attr, NULL }; @@ -844,7 +852,7 @@ static const struct attribute_group attribute_group = { .attrs = attributes, }; -#ifndef FPC_DRM_INTERFACE_WA +#ifdef FPC_DRM_INTERFACE static void notification_work(struct work_struct *work) { pr_info("%s: unblank\n", __func__); @@ -864,13 +872,13 @@ static irqreturn_t fpc1020_irq_handler(int irq, void *handle) } sysfs_notify(&fpc1020->dev->kobj, NULL, dev_attr_irq.attr.name); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (fpc1020->wait_finger_down && fpc1020->fb_black && fpc1020->prepared) { pr_info("%s enter fingerdown & fb_black then schedule_work\n", __func__); fpc1020->wait_finger_down = false; -#ifndef FPC_DRM_INTERFACE_WA schedule_work(&fpc1020->work); -#endif } +#endif return IRQ_HANDLED; } @@ -903,7 +911,7 @@ static int fpc1020_request_named_gpio(struct fpc1020_data *fpc1020, return 0; } -#ifndef FPC_DRM_INTERFACE_WA +#ifdef FPC_DRM_INTERFACE static int fpc_fb_notif_callback(struct notifier_block *nb, unsigned long val, void *data) { @@ -1032,9 +1040,9 @@ static int fpc1020_probe(struct platform_device *pdev) (void)device_prepare(fpc1020, true); } +#if IS_ENABLED(CONFIG_MI_DRM_OPT) fpc1020->fb_black = false; fpc1020->wait_finger_down = false; -#ifndef FPC_DRM_INTERFACE_WA INIT_WORK(&fpc1020->work, notification_work); fpc1020->fb_notifier = fpc_notif_block; mi_drm_register_client(&fpc1020->fb_notifier); @@ -1053,7 +1061,7 @@ static int fpc1020_remove(struct platform_device *pdev) { struct fpc1020_data *fpc1020 = platform_get_drvdata(pdev); -#ifndef FPC_DRM_INTERFACE_WA +#ifdef FPC_DRM_INTERFACE mi_drm_unregister_client(&fpc1020->fb_notifier); #endif sysfs_remove_group(&pdev->dev.kobj, &attribute_group); diff --git a/drivers/input/fingerprint/fpc_tee/fpc1020_tee_apollo.c b/drivers/input/fingerprint/fpc_tee/fpc1020_tee_apollo.c index c1b12f0badba..6f5ceb49f6b8 100644 --- a/drivers/input/fingerprint/fpc_tee/fpc1020_tee_apollo.c +++ b/drivers/input/fingerprint/fpc_tee/fpc1020_tee_apollo.c @@ -25,6 +25,10 @@ * as published by the Free Software Foundation. */ +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +#define FPC_DRM_INTERFACE +#endif + #define CONFIG_FINGERPRINT_FP_VREG_CONTROL #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -42,12 +46,20 @@ #include #include #include +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +#include +#include +#endif #define FPC_GPIO_NO_DEFAULT -1 #define FPC_GPIO_NO_DEFINED -2 #define FPC_GPIO_REQUEST_FAIL -3 #define FPC_TTW_HOLD_TIME 2000 +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +#define FP_UNLOCK_REJECTION_TIMEOUT (FPC_TTW_HOLD_TIME - 500) +#endif + #define RESET_LOW_SLEEP_MIN_US 5000 #define RESET_LOW_SLEEP_MAX_US (RESET_LOW_SLEEP_MIN_US + 100) #define RESET_HIGH_SLEEP1_MIN_US 100 @@ -113,6 +125,12 @@ struct fpc1020_data { atomic_t wakeup_enabled; /* Used both in ISR and non-ISR */ int irqf; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) + struct notifier_block fb_notifier; + bool fb_black; + bool wait_finger_down; + struct work_struct work; +#endif }; static int reset_gpio_res(struct fpc1020_data *fpc1020); @@ -742,6 +760,28 @@ static ssize_t irq_ack(struct device *dev, static DEVICE_ATTR(irq, S_IRUSR | S_IWUSR, irq_get, irq_ack); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +static ssize_t fingerdown_wait_set(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct fpc1020_data *fpc1020 = dev_get_drvdata(dev); + + dev_info(fpc1020->dev, "%s -> %s\n", __func__, buf); + if (!strncmp(buf, "enable", strlen("enable")) && fpc1020->prepared) + fpc1020->wait_finger_down = true; + else if (!strncmp(buf, "disable", strlen("disable")) + && fpc1020->prepared) + fpc1020->wait_finger_down = false; + else + return -EINVAL; + + return count; +} + +static DEVICE_ATTR(fingerdown_wait, S_IWUSR, NULL, fingerdown_wait_set); +#endif + static ssize_t vendor_update(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -791,6 +831,9 @@ static struct attribute *attributes[] = { &dev_attr_irq_enable.attr, &dev_attr_power_cfg.attr, &dev_attr_irq.attr, +#if IS_ENABLED(CONFIG_MI_DRM_OPT) + &dev_attr_fingerdown_wait.attr, +#endif &dev_attr_vendor.attr, NULL }; @@ -799,6 +842,14 @@ static const struct attribute_group attribute_group = { .attrs = attributes, }; +#ifdef FPC_DRM_INTERFACE +static void notification_work(struct work_struct *work) +{ + pr_info("%s: unblank\n", __func__); + dsi_bridge_interface_enable(FP_UNLOCK_REJECTION_TIMEOUT); +} +#endif + static irqreturn_t fpc1020_irq_handler(int irq, void *handle) { struct fpc1020_data *fpc1020 = handle; @@ -815,6 +866,13 @@ static irqreturn_t fpc1020_irq_handler(int irq, void *handle) mutex_unlock(&fpc1020->lock); sysfs_notify(&fpc1020->dev->kobj, NULL, dev_attr_irq.attr.name); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) + if (fpc1020->wait_finger_down && fpc1020->fb_black && fpc1020->prepared) { + pr_info("%s enter fingerdown & fb_black then schedule_work\n", __func__); + fpc1020->wait_finger_down = false; + schedule_work(&fpc1020->work); + } +#endif return IRQ_HANDLED; } @@ -847,6 +905,45 @@ static int fpc1020_request_named_gpio(struct fpc1020_data *fpc1020, return 0; } +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +static int fpc_fb_notif_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct fpc1020_data *fpc1020 = container_of(nb, struct fpc1020_data, + fb_notifier); + struct fb_event *evdata = data; + unsigned int blank; + + if (!fpc1020) + return 0; + + if (val != MI_DRM_EVENT_BLANK || fpc1020->prepared == false) + return 0; + + pr_debug("[info] %s value = %d\n", __func__, (int)val); + + if (evdata && evdata->data && val == MI_DRM_EVENT_BLANK) { + blank = *(int *)(evdata->data); + switch (blank) { + case MI_DRM_BLANK_POWERDOWN: + fpc1020->fb_black = true; + break; + case MI_DRM_BLANK_UNBLANK: + fpc1020->fb_black = false; + break; + default: + pr_debug("%s defalut\n", __func__); + break; + } + } + return NOTIFY_OK; +} + +static struct notifier_block fpc_notif_block = { + .notifier_call = fpc_fb_notif_callback, +}; +#endif + static int fpc1020_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -931,6 +1028,14 @@ static int fpc1020_probe(struct platform_device *pdev) (void)device_prepare(fpc1020, true); } +#if IS_ENABLED(CONFIG_MI_DRM_OPT) + fpc1020->fb_black = false; + fpc1020->wait_finger_down = false; + INIT_WORK(&fpc1020->work, notification_work); + fpc1020->fb_notifier = fpc_notif_block; + mi_drm_register_client(&fpc1020->fb_notifier); +#endif + //rc = hw_reset(fpc1020); if (msm_gpio_mpm_wake_set(121, true)) { @@ -949,6 +1054,11 @@ static int fpc1020_probe(struct platform_device *pdev) static int fpc1020_remove(struct platform_device *pdev) { struct fpc1020_data *fpc1020 = platform_get_drvdata(pdev); + +#if IS_ENABLED(CONFIG_MI_DRM_OPT) + mi_drm_unregister_client(&fpc1020->fb_notifier); +#endif + sysfs_remove_group(&pdev->dev.kobj, &attribute_group); mutex_destroy(&fpc1020->lock); wakeup_source_unregister(fpc1020->ttw_wl); diff --git a/drivers/input/fingerprint/goodix/gf_spi.c b/drivers/input/fingerprint/goodix/gf_spi.c index c4fcbfe2d66a..27adfda84312 100644 --- a/drivers/input/fingerprint/goodix/gf_spi.c +++ b/drivers/input/fingerprint/goodix/gf_spi.c @@ -16,7 +16,9 @@ #define DEBUG #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#define GOODIX_DRM_INTERFACE_WA +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +#define GOODIX_DRM_INTERFACE +#endif #include #include @@ -45,8 +47,8 @@ #include #include #include +#if IS_ENABLED(CONFIG_MI_DRM_OPT) #include -#ifndef GOODIX_DRM_INTERFACE_WA #include #endif @@ -550,7 +552,7 @@ static long gf_compat_ioctl(struct file *filp, unsigned int cmd, } #endif /*CONFIG_COMPAT*/ -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE static void notification_work(struct work_struct *work) { pr_debug("%s unblank\n", __func__); @@ -568,11 +570,13 @@ static irqreturn_t gf_irq(int irq, void *handle) __pm_wakeup_event(fp_wakelock, WAKELOCK_HOLD_TIME); sendnlmsg(temp); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if ((gf_dev->wait_finger_down == true) && (gf_dev->device_available == 1) && (gf_dev->fb_black == 1)) { gf_dev->wait_finger_down = false; schedule_work(&gf_dev->work); } +#endif #elif defined (GF_FASYNC) struct gf_dev *gf_dev = &gf; @@ -758,8 +762,7 @@ static const struct file_operations gf_fops = { #endif }; - -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE static int goodix_fb_state_chg_callback(struct notifier_block *nb, unsigned long val, void *data) { @@ -850,9 +853,9 @@ static int gf_probe(struct platform_device *pdev) gf_dev->reset_gpio = -EINVAL; gf_dev->pwr_gpio = -EINVAL; gf_dev->device_available = 0; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) gf_dev->fb_black = 0; gf_dev->wait_finger_down = false; -#ifndef GOODIX_DRM_INTERFACE_WA INIT_WORK(&gf_dev->work, notification_work); #endif @@ -925,7 +928,7 @@ static int gf_probe(struct platform_device *pdev) spi_clock_set(gf_dev, 1000000); #endif -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE gf_dev->notifier = goodix_noti_block; mi_drm_register_client(&gf_dev->notifier); #endif @@ -991,7 +994,7 @@ static int gf_remove(struct platform_device *pdev) gf_cleanup(gf_dev); } -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE mi_drm_unregister_client(&gf_dev->notifier); #endif mutex_unlock(&device_list_lock); diff --git a/drivers/input/fingerprint/goodix_fod/gf_spi.c b/drivers/input/fingerprint/goodix_fod/gf_spi.c index 12b9d8d0421e..83b07647aaab 100644 --- a/drivers/input/fingerprint/goodix_fod/gf_spi.c +++ b/drivers/input/fingerprint/goodix_fod/gf_spi.c @@ -16,7 +16,9 @@ #define DEBUG #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#define GOODIX_DRM_INTERFACE_WA +#if IS_ENABLED(CONFIG_MI_DRM_OPT) +#define GOODIX_DRM_INTERFACE +#endif #include #include @@ -45,8 +47,8 @@ #include #include #include +#if IS_ENABLED(CONFIG_MI_DRM_OPT) #include -#ifndef GOODIX_DRM_INTERFACE_WA #include #endif @@ -614,7 +616,7 @@ static long gf_compat_ioctl(struct file *filp, unsigned int cmd, } #endif /*CONFIG_COMPAT */ -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE static void notification_work(struct work_struct *work) { pr_debug("%s unblank\n", __func__); @@ -624,7 +626,7 @@ static void notification_work(struct work_struct *work) static irqreturn_t gf_irq(int irq, void *handle) { - struct gf_dev *gf_dev = &gf; + __maybe_unused struct gf_dev *gf_dev = &gf; #if defined(GF_NETLINK_ENABLE) char temp[4] = { 0x0 }; uint32_t key_input = 0; @@ -634,6 +636,8 @@ static irqreturn_t gf_irq(int irq, void *handle) __pm_wakeup_event(fp_wakelock, WAKELOCK_HOLD_TIME); sendnlmsg(temp); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) + if ((gf_dev->wait_finger_down == true) && (gf_dev->device_available == 1) && (gf_dev->fb_black == 1)) { key_input = KEY_RIGHT; @@ -644,6 +648,7 @@ static irqreturn_t gf_irq(int irq, void *handle) gf_dev->wait_finger_down = false; schedule_work(&gf_dev->work); } +#endif #elif defined (GF_FASYNC) struct gf_dev *gf_dev = &gf; @@ -834,7 +839,7 @@ static const struct file_operations gf_fops = { #endif }; -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE static int goodix_fb_state_chg_callback(struct notifier_block *nb, unsigned long val, void *data) { @@ -926,9 +931,9 @@ static int gf_probe(struct platform_device *pdev) gf_dev->reset_gpio = -EINVAL; gf_dev->pwr_gpio = -EINVAL; gf_dev->device_available = 0; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) gf_dev->fb_black = 0; gf_dev->wait_finger_down = false; -#ifndef GOODIX_DRM_INTERFACE_WA INIT_WORK(&gf_dev->work, notification_work); #endif @@ -1002,7 +1007,7 @@ static int gf_probe(struct platform_device *pdev) spi_clock_set(gf_dev, 1000000); #endif -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE gf_dev->notifier = goodix_noti_block; drm_register_client(&gf_dev->notifier); #endif @@ -1079,7 +1084,7 @@ static int gf_remove(struct platform_device *pdev) if (gf_dev->users == 0) { gf_cleanup(gf_dev); } -#ifndef GOODIX_DRM_INTERFACE_WA +#ifdef GOODIX_DRM_INTERFACE drm_unregister_client(&gf_dev->notifier); #endif mutex_unlock(&device_list_lock); diff --git a/drivers/input/misc/sparcspkr.c b/drivers/input/misc/sparcspkr.c index 4a5afc7fe96e..f6e1f38267d9 100644 --- a/drivers/input/misc/sparcspkr.c +++ b/drivers/input/misc/sparcspkr.c @@ -204,6 +204,7 @@ static int bbc_beep_probe(struct platform_device *op) info = &state->u.bbc; info->clock_freq = of_getintprop_default(dp, "clock-frequency", 0); + of_node_put(dp); if (!info->clock_freq) goto out_free; diff --git a/drivers/input/mouse/bcm5974.c b/drivers/input/mouse/bcm5974.c index d0122134f320..f68816329a2e 100644 --- a/drivers/input/mouse/bcm5974.c +++ b/drivers/input/mouse/bcm5974.c @@ -956,17 +956,22 @@ static int bcm5974_probe(struct usb_interface *iface, if (!dev->tp_data) goto err_free_bt_buffer; - if (dev->bt_urb) + if (dev->bt_urb) { usb_fill_int_urb(dev->bt_urb, udev, usb_rcvintpipe(udev, cfg->bt_ep), dev->bt_data, dev->cfg.bt_datalen, bcm5974_irq_button, dev, 1); + dev->bt_urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; + } + usb_fill_int_urb(dev->tp_urb, udev, usb_rcvintpipe(udev, cfg->tp_ep), dev->tp_data, dev->cfg.tp_datalen, bcm5974_irq_trackpad, dev, 1); + dev->tp_urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; + /* create bcm5974 device */ usb_make_path(udev, dev->phys, sizeof(dev->phys)); strlcat(dev->phys, "/input0", sizeof(dev->phys)); diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 76ae6968801e..1c61cd0b1d55 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -90,7 +90,7 @@ #define ACPI_DEVFLAG_LINT1 0x80 #define ACPI_DEVFLAG_ATSDIS 0x10000000 -#define LOOP_TIMEOUT 100000 +#define LOOP_TIMEOUT 2000000 /* * ACPI table definitions * diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index fe9eabe9f785..f4bc3e88917d 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -638,16 +638,19 @@ static void insert_iommu_master(struct device *dev, static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *spec) { - struct msm_iommu_dev *iommu; + struct msm_iommu_dev *iommu = NULL, *iter; unsigned long flags; int ret = 0; spin_lock_irqsave(&msm_iommu_lock, flags); - list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) - if (iommu->dev->of_node == spec->np) + list_for_each_entry(iter, &qcom_iommu_devices, dev_node) { + if (iter->dev->of_node == spec->np) { + iommu = iter; break; + } + } - if (!iommu || iommu->dev->of_node != spec->np) { + if (!iommu) { ret = -ENODEV; goto fail; } diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 8e75f34ac886..7304ad88f126 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -702,8 +702,7 @@ static int mtk_iommu_remove(struct platform_device *pdev) iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); - if (iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, NULL); + list_del(&data->list); clk_disable_unprepare(data->bclk); devm_free_irq(&pdev->dev, data->irq, data); diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 5849ac5a2ad3..0fd428db3aa4 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -392,7 +392,16 @@ static void armada_xp_mpic_smp_cpu_init(void) static void armada_xp_mpic_perf_init(void) { - unsigned long cpuid = cpu_logical_map(smp_processor_id()); + unsigned long cpuid; + + /* + * This Performance Counter Overflow interrupt is specific for + * Armada 370 and XP. It is not available on Armada 375, 38x and 39x. + */ + if (!of_machine_is_compatible("marvell,armada-370-xp")) + return; + + cpuid = cpu_logical_map(smp_processor_id()); /* Enable Performance Counter Overflow interrupts */ writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), diff --git a/drivers/irqchip/irq-aspeed-i2c-ic.c b/drivers/irqchip/irq-aspeed-i2c-ic.c index f20200af0992..1274e3bc2213 100644 --- a/drivers/irqchip/irq-aspeed-i2c-ic.c +++ b/drivers/irqchip/irq-aspeed-i2c-ic.c @@ -82,8 +82,8 @@ static int __init aspeed_i2c_ic_of_init(struct device_node *node, } i2c_ic->parent_irq = irq_of_parse_and_map(node, 0); - if (i2c_ic->parent_irq < 0) { - ret = i2c_ic->parent_irq; + if (!i2c_ic->parent_irq) { + ret = -EINVAL; goto err_iounmap; } diff --git a/drivers/irqchip/irq-xtensa-mx.c b/drivers/irqchip/irq-xtensa-mx.c index e539500752d4..345960c60e34 100644 --- a/drivers/irqchip/irq-xtensa-mx.c +++ b/drivers/irqchip/irq-xtensa-mx.c @@ -141,14 +141,25 @@ static struct irq_chip xtensa_mx_irq_chip = { .irq_set_affinity = xtensa_mx_irq_set_affinity, }; +static void __init xtensa_mx_init_common(struct irq_domain *root_domain) +{ + unsigned int i; + + irq_set_default_host(root_domain); + secondary_init_irq(); + + /* Initialize default IRQ routing to CPU 0 */ + for (i = 0; i < XCHAL_NUM_EXTINTERRUPTS; ++i) + set_er(1, MIROUT(i)); +} + int __init xtensa_mx_init_legacy(struct device_node *interrupt_parent) { struct irq_domain *root_domain = irq_domain_add_legacy(NULL, NR_IRQS - 1, 1, 0, &xtensa_mx_irq_domain_ops, &xtensa_mx_irq_chip); - irq_set_default_host(root_domain); - secondary_init_irq(); + xtensa_mx_init_common(root_domain); return 0; } @@ -158,8 +169,7 @@ static int __init xtensa_mx_init(struct device_node *np, struct irq_domain *root_domain = irq_domain_add_linear(np, NR_IRQS, &xtensa_mx_irq_domain_ops, &xtensa_mx_irq_chip); - irq_set_default_host(root_domain); - secondary_init_irq(); + xtensa_mx_init_common(root_domain); return 0; } IRQCHIP_DECLARE(xtensa_mx_irq_chip, "cdns,xtensa-mx", xtensa_mx_init); diff --git a/drivers/macintosh/Kconfig b/drivers/macintosh/Kconfig index 47c350cdfb12..a316624742f6 100644 --- a/drivers/macintosh/Kconfig +++ b/drivers/macintosh/Kconfig @@ -66,6 +66,10 @@ config ADB_PMU this device; you should do so if your machine is one of those mentioned above. +config ADB_PMU_EVENT + def_bool y + depends on ADB_PMU && INPUT=y + config ADB_PMU_LED bool "Support for the Power/iBook front LED" depends on PPC_PMAC && ADB_PMU diff --git a/drivers/macintosh/Makefile b/drivers/macintosh/Makefile index 49819b1b6f20..712edcb3e0b0 100644 --- a/drivers/macintosh/Makefile +++ b/drivers/macintosh/Makefile @@ -12,7 +12,8 @@ obj-$(CONFIG_MAC_EMUMOUSEBTN) += mac_hid.o obj-$(CONFIG_INPUT_ADBHID) += adbhid.o obj-$(CONFIG_ANSLCD) += ans-lcd.o -obj-$(CONFIG_ADB_PMU) += via-pmu.o via-pmu-event.o +obj-$(CONFIG_ADB_PMU) += via-pmu.o +obj-$(CONFIG_ADB_PMU_EVENT) += via-pmu-event.o obj-$(CONFIG_ADB_PMU_LED) += via-pmu-led.o obj-$(CONFIG_PMAC_BACKLIGHT) += via-pmu-backlight.o obj-$(CONFIG_ADB_CUDA) += via-cuda.o diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c index d72c450aebe5..50299d68ddfc 100644 --- a/drivers/macintosh/via-pmu.c +++ b/drivers/macintosh/via-pmu.c @@ -1464,7 +1464,7 @@ pmu_handle_data(unsigned char *data, int len) pmu_pass_intr(data, len); /* len == 6 is probably a bad check. But how do I * know what PMU versions send what events here? */ - if (len == 6) { + if (IS_ENABLED(CONFIG_ADB_PMU_EVENT) && len == 6) { via_pmu_event(PMU_EVT_POWER, !!(data[1]&8)); via_pmu_event(PMU_EVT_LID, data[1]&1); } diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index c1e487d1261c..958c3a1c453a 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -1102,6 +1102,12 @@ static void detached_dev_do_request(struct bcache_device *d, struct bio *bio) * which would call closure_get(&dc->disk.cl) */ ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO); + if (!ddip) { + bio->bi_status = BLK_STS_RESOURCE; + bio->bi_end_io(bio); + return; + } + ddip->d = d; ddip->start_time = jiffies; ddip->bi_end_io = bio->bi_end_io; diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 3f40a0179155..4b6a74db1d85 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -2941,6 +2941,11 @@ static int crypt_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_SUBMITTED; } +static char hex2asc(unsigned char c) +{ + return c + '0' + ((unsigned)(9 - c) >> 4 & 0x27); +} + static void crypt_status(struct dm_target *ti, status_type_t type, unsigned status_flags, char *result, unsigned maxlen) { @@ -2959,9 +2964,12 @@ static void crypt_status(struct dm_target *ti, status_type_t type, if (cc->key_size > 0) { if (cc->key_string) DMEMIT(":%u:%s", cc->key_size, cc->key_string); - else - for (i = 0; i < cc->key_size; i++) - DMEMIT("%02x", cc->key[i]); + else { + for (i = 0; i < cc->key_size; i++) { + DMEMIT("%c%c", hex2asc(cc->key[i] >> 4), + hex2asc(cc->key[i] & 0xf)); + } + } } else DMEMIT("-"); diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 0f333fac664f..d9fa5ace66eb 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -3576,8 +3576,6 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } if (should_write_sb) { - int r; - init_journal(ic, 0, ic->journal_sections, 0); r = dm_integrity_failed(ic); if (unlikely(r)) { diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index 21de30b4e2a1..3d59f3e208c5 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -224,6 +224,7 @@ void dm_stats_cleanup(struct dm_stats *stats) atomic_read(&shared->in_flight[READ]), atomic_read(&shared->in_flight[WRITE])); } + cond_resched(); } dm_stat_free(&s->rcu_head); } @@ -313,6 +314,7 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, for (ni = 0; ni < n_entries; ni++) { atomic_set(&s->stat_shared[ni].in_flight[READ], 0); atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0); + cond_resched(); } if (s->n_histogram_entries) { @@ -325,6 +327,7 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, for (ni = 0; ni < n_entries; ni++) { s->stat_shared[ni].tmp.histogram = hi; hi += s->n_histogram_entries + 1; + cond_resched(); } } @@ -345,6 +348,7 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, for (ni = 0; ni < n_entries; ni++) { p[ni].histogram = hi; hi += s->n_histogram_entries + 1; + cond_resched(); } } } @@ -474,6 +478,7 @@ static int dm_stats_list(struct dm_stats *stats, const char *program, } DMEMIT("\n"); } + cond_resched(); } mutex_unlock(&stats->mutex); @@ -750,6 +755,7 @@ static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end, local_irq_enable(); } } + cond_resched(); } } @@ -865,6 +871,8 @@ static int dm_stats_print(struct dm_stats *stats, int id, if (unlikely(sz + 1 >= maxlen)) goto buffer_overflow; + + cond_resched(); } if (clear) diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index a44d766af305..89ceba21185a 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -1213,6 +1213,7 @@ int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) static struct target_type verity_target = { .name = "verity", + .features = DM_TARGET_IMMUTABLE, .version = {1, 4, 0}, .module = THIS_MODULE, .ctr = verity_ctr, diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index e4b41d6ec95d..29d60325e8b9 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -645,14 +645,6 @@ static int md_bitmap_read_sb(struct bitmap *bitmap) daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ; write_behind = le32_to_cpu(sb->write_behind); sectors_reserved = le32_to_cpu(sb->sectors_reserved); - /* Setup nodes/clustername only if bitmap version is - * cluster-compatible - */ - if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) { - nodes = le32_to_cpu(sb->nodes); - strlcpy(bitmap->mddev->bitmap_info.cluster_name, - sb->cluster_name, 64); - } /* verify that the bitmap-specific fields are valid */ if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) @@ -674,6 +666,16 @@ static int md_bitmap_read_sb(struct bitmap *bitmap) goto out; } + /* + * Setup nodes/clustername only if bitmap version is + * cluster-compatible + */ + if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) { + nodes = le32_to_cpu(sb->nodes); + strlcpy(bitmap->mddev->bitmap_info.cluster_name, + sb->cluster_name, 64); + } + /* keep the array size field of the bitmap superblock up to date */ sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); @@ -706,9 +708,9 @@ static int md_bitmap_read_sb(struct bitmap *bitmap) out: kunmap_atomic(sb); - /* Assigning chunksize is required for "re_read" */ - bitmap->mddev->bitmap_info.chunksize = chunksize; if (err == 0 && nodes && (bitmap->cluster_slot < 0)) { + /* Assigning chunksize is required for "re_read" */ + bitmap->mddev->bitmap_info.chunksize = chunksize; err = md_setup_cluster(bitmap->mddev, nodes); if (err) { pr_warn("%s: Could not setup cluster service (%d)\n", @@ -719,18 +721,18 @@ static int md_bitmap_read_sb(struct bitmap *bitmap) goto re_read; } - out_no_sb: - if (test_bit(BITMAP_STALE, &bitmap->flags)) - bitmap->events_cleared = bitmap->mddev->events; - bitmap->mddev->bitmap_info.chunksize = chunksize; - bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; - bitmap->mddev->bitmap_info.max_write_behind = write_behind; - bitmap->mddev->bitmap_info.nodes = nodes; - if (bitmap->mddev->bitmap_info.space == 0 || - bitmap->mddev->bitmap_info.space > sectors_reserved) - bitmap->mddev->bitmap_info.space = sectors_reserved; - if (err) { + if (err == 0) { + if (test_bit(BITMAP_STALE, &bitmap->flags)) + bitmap->events_cleared = bitmap->mddev->events; + bitmap->mddev->bitmap_info.chunksize = chunksize; + bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; + bitmap->mddev->bitmap_info.max_write_behind = write_behind; + bitmap->mddev->bitmap_info.nodes = nodes; + if (bitmap->mddev->bitmap_info.space == 0 || + bitmap->mddev->bitmap_info.space > sectors_reserved) + bitmap->mddev->bitmap_info.space = sectors_reserved; + } else { md_bitmap_print_sb(bitmap); if (bitmap->cluster_slot < 0) md_cluster_stop(bitmap->mddev); diff --git a/drivers/md/md.c b/drivers/md/md.c index 7e0477e883c7..4594a1ee88b9 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -2443,14 +2443,16 @@ static void sync_sbs(struct mddev *mddev, int nospares) static bool does_sb_need_changing(struct mddev *mddev) { - struct md_rdev *rdev; + struct md_rdev *rdev = NULL, *iter; struct mdp_superblock_1 *sb; int role; /* Find a good rdev */ - rdev_for_each(rdev, mddev) - if ((rdev->raid_disk >= 0) && !test_bit(Faulty, &rdev->flags)) + rdev_for_each(iter, mddev) + if ((iter->raid_disk >= 0) && !test_bit(Faulty, &iter->flags)) { + rdev = iter; break; + } /* No good device found. */ if (!rdev) @@ -7618,17 +7620,22 @@ EXPORT_SYMBOL(md_register_thread); void md_unregister_thread(struct md_thread **threadp) { - struct md_thread *thread = *threadp; - if (!thread) - return; - pr_debug("interrupting MD-thread pid %d\n", task_pid_nr(thread->tsk)); - /* Locking ensures that mddev_unlock does not wake_up a + struct md_thread *thread; + + /* + * Locking ensures that mddev_unlock does not wake_up a * non-existent thread */ spin_lock(&pers_lock); + thread = *threadp; + if (!thread) { + spin_unlock(&pers_lock); + return; + } *threadp = NULL; spin_unlock(&pers_lock); + pr_debug("interrupting MD-thread pid %d\n", task_pid_nr(thread->tsk)); kthread_stop(thread->tsk); kfree(thread); } @@ -9323,16 +9330,18 @@ static int read_rdev(struct mddev *mddev, struct md_rdev *rdev) void md_reload_sb(struct mddev *mddev, int nr) { - struct md_rdev *rdev; + struct md_rdev *rdev = NULL, *iter; int err; /* Find the rdev */ - rdev_for_each_rcu(rdev, mddev) { - if (rdev->desc_nr == nr) + rdev_for_each_rcu(iter, mddev) { + if (iter->desc_nr == nr) { + rdev = iter; break; + } } - if (!rdev || rdev->desc_nr != nr) { + if (!rdev) { pr_warn("%s: %d Could not find rdev with nr %d\n", __func__, __LINE__, nr); return; } diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 0272102b207e..2a782c4f4710 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -150,21 +150,6 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf) pr_debug("md/raid0:%s: FINAL %d zones\n", mdname(mddev), conf->nr_strip_zones); - if (conf->nr_strip_zones == 1) { - conf->layout = RAID0_ORIG_LAYOUT; - } else if (mddev->layout == RAID0_ORIG_LAYOUT || - mddev->layout == RAID0_ALT_MULTIZONE_LAYOUT) { - conf->layout = mddev->layout; - } else if (default_layout == RAID0_ORIG_LAYOUT || - default_layout == RAID0_ALT_MULTIZONE_LAYOUT) { - conf->layout = default_layout; - } else { - pr_err("md/raid0:%s: cannot assemble multi-zone RAID0 with default_layout setting\n", - mdname(mddev)); - pr_err("md/raid0: please set raid0.default_layout to 1 or 2\n"); - err = -ENOTSUPP; - goto abort; - } /* * now since we have the hard sector sizes, we can make sure * chunk size is a multiple of that sector size @@ -295,6 +280,22 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf) (unsigned long long)smallest->sectors); } + if (conf->nr_strip_zones == 1 || conf->strip_zone[1].nb_dev == 1) { + conf->layout = RAID0_ORIG_LAYOUT; + } else if (mddev->layout == RAID0_ORIG_LAYOUT || + mddev->layout == RAID0_ALT_MULTIZONE_LAYOUT) { + conf->layout = mddev->layout; + } else if (default_layout == RAID0_ORIG_LAYOUT || + default_layout == RAID0_ALT_MULTIZONE_LAYOUT) { + conf->layout = default_layout; + } else { + pr_err("md/raid0:%s: cannot assemble multi-zone RAID0 with default_layout setting\n", + mdname(mddev)); + pr_err("md/raid0: please set raid0.default_layout to 1 or 2\n"); + err = -EOPNOTSUPP; + goto abort; + } + pr_debug("md/raid0:%s: done.\n", mdname(mddev)); *private_conf = conf; diff --git a/drivers/media/cec/cec-adap.c b/drivers/media/cec/cec-adap.c index 5eeadab15a5f..a42043379d67 100644 --- a/drivers/media/cec/cec-adap.c +++ b/drivers/media/cec/cec-adap.c @@ -1218,7 +1218,7 @@ static int cec_config_log_addr(struct cec_adapter *adap, * While trying to poll the physical address was reset * and the adapter was unconfigured, so bail out. */ - if (!adap->is_configuring) + if (adap->phys_addr == CEC_PHYS_ADDR_INVALID) return -EINTR; if (err) @@ -1276,7 +1276,6 @@ static void cec_adap_unconfigure(struct cec_adapter *adap) adap->phys_addr != CEC_PHYS_ADDR_INVALID) WARN_ON(adap->ops->adap_log_addr(adap, CEC_LOG_ADDR_INVALID)); adap->log_addrs.log_addr_mask = 0; - adap->is_configuring = false; adap->is_configured = false; memset(adap->phys_addrs, 0xff, sizeof(adap->phys_addrs)); cec_flush(adap); @@ -1469,9 +1468,10 @@ static int cec_config_thread_func(void *arg) for (i = 0; i < las->num_log_addrs; i++) las->log_addr[i] = CEC_LOG_ADDR_INVALID; cec_adap_unconfigure(adap); + adap->is_configuring = false; adap->kthread_config = NULL; - mutex_unlock(&adap->lock); complete(&adap->config_completion); + mutex_unlock(&adap->lock); return 0; } diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c index a1d738969d7b..06e4e1df125c 100644 --- a/drivers/media/pci/cx23885/cx23885-core.c +++ b/drivers/media/pci/cx23885/cx23885-core.c @@ -2164,7 +2164,7 @@ static int cx23885_initdev(struct pci_dev *pci_dev, err = pci_set_dma_mask(pci_dev, 0xffffffff); if (err) { pr_err("%s/0: Oops: no 32bit PCI DMA ???\n", dev->name); - goto fail_ctrl; + goto fail_dma_set_mask; } err = request_irq(pci_dev->irq, cx23885_irq, @@ -2172,7 +2172,7 @@ static int cx23885_initdev(struct pci_dev *pci_dev, if (err < 0) { pr_err("%s: can't get IRQ %d\n", dev->name, pci_dev->irq); - goto fail_irq; + goto fail_dma_set_mask; } switch (dev->board) { @@ -2194,7 +2194,7 @@ static int cx23885_initdev(struct pci_dev *pci_dev, return 0; -fail_irq: +fail_dma_set_mask: cx23885_dev_unregister(dev); fail_ctrl: v4l2_ctrl_handler_free(hdl); diff --git a/drivers/media/pci/cx25821/cx25821-core.c b/drivers/media/pci/cx25821/cx25821-core.c index e04fe9f17b7a..99359eaf8cdc 100644 --- a/drivers/media/pci/cx25821/cx25821-core.c +++ b/drivers/media/pci/cx25821/cx25821-core.c @@ -1350,11 +1350,11 @@ static void cx25821_finidev(struct pci_dev *pci_dev) struct cx25821_dev *dev = get_cx25821(v4l2_dev); cx25821_shutdown(dev); - pci_disable_device(pci_dev); /* unregister stuff */ if (pci_dev->irq) free_irq(pci_dev->irq, dev); + pci_disable_device(pci_dev); cx25821_dev_unregister(dev); v4l2_device_unregister(v4l2_dev); diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c index 5f8da544b98d..d792122b86c2 100644 --- a/drivers/media/platform/coda/coda-common.c +++ b/drivers/media/platform/coda/coda-common.c @@ -1894,8 +1894,8 @@ static void coda_encode_ctrls(struct coda_ctx *ctx) 0x0, V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED); v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_PROFILE, - V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE, 0x0, - V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE); + V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE, 0x0, + V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE); if (ctx->dev->devtype->product == CODA_HX4 || ctx->dev->devtype->product == CODA_7541) { v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops, @@ -1909,12 +1909,15 @@ static void coda_encode_ctrls(struct coda_ctx *ctx) if (ctx->dev->devtype->product == CODA_960) { v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, - V4L2_MPEG_VIDEO_H264_LEVEL_4_0, - ~((1 << V4L2_MPEG_VIDEO_H264_LEVEL_2_0) | + V4L2_MPEG_VIDEO_H264_LEVEL_4_2, + ~((1 << V4L2_MPEG_VIDEO_H264_LEVEL_1_0) | + (1 << V4L2_MPEG_VIDEO_H264_LEVEL_2_0) | (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_0) | (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_1) | (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_2) | - (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_0)), + (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_0) | + (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_1) | + (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_2)), V4L2_MPEG_VIDEO_H264_LEVEL_4_0); } v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops, @@ -1977,7 +1980,7 @@ static void coda_decode_ctrls(struct coda_ctx *ctx) ctx->h264_profile_ctrl = v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_PROFILE, V4L2_MPEG_VIDEO_H264_PROFILE_HIGH, - ~((1 << V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) | + ~((1 << V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) | (1 << V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) | (1 << V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)), V4L2_MPEG_VIDEO_H264_PROFILE_HIGH); diff --git a/drivers/media/platform/exynos4-is/fimc-is.c b/drivers/media/platform/exynos4-is/fimc-is.c index 0fe9be93fabe..0f3f82bd4d20 100644 --- a/drivers/media/platform/exynos4-is/fimc-is.c +++ b/drivers/media/platform/exynos4-is/fimc-is.c @@ -144,7 +144,7 @@ static int fimc_is_enable_clocks(struct fimc_is *is) dev_err(&is->pdev->dev, "clock %s enable failed\n", fimc_is_clocks[i]); for (--i; i >= 0; i--) - clk_disable(is->clocks[i]); + clk_disable_unprepare(is->clocks[i]); return ret; } pr_debug("enabled clock: %s\n", fimc_is_clocks[i]); diff --git a/drivers/media/platform/exynos4-is/fimc-isp-video.h b/drivers/media/platform/exynos4-is/fimc-isp-video.h index f79a1b348aa6..67ef85249912 100644 --- a/drivers/media/platform/exynos4-is/fimc-isp-video.h +++ b/drivers/media/platform/exynos4-is/fimc-isp-video.h @@ -35,7 +35,7 @@ static inline int fimc_isp_video_device_register(struct fimc_isp *isp, return 0; } -void fimc_isp_video_device_unregister(struct fimc_isp *isp, +static inline void fimc_isp_video_device_unregister(struct fimc_isp *isp, enum v4l2_buf_type type) { } diff --git a/drivers/media/platform/qcom/venus/hfi.c b/drivers/media/platform/qcom/venus/hfi.c index 24207829982f..8a99e2d8274a 100644 --- a/drivers/media/platform/qcom/venus/hfi.c +++ b/drivers/media/platform/qcom/venus/hfi.c @@ -113,6 +113,9 @@ int hfi_core_deinit(struct venus_core *core, bool blocking) mutex_lock(&core->lock); } + if (!core->ops) + goto unlock; + ret = core->ops->core_deinit(core); if (!ret) diff --git a/drivers/media/platform/sti/delta/delta-v4l2.c b/drivers/media/platform/sti/delta/delta-v4l2.c index 53dc6da2b09e..eb5e8867967e 100644 --- a/drivers/media/platform/sti/delta/delta-v4l2.c +++ b/drivers/media/platform/sti/delta/delta-v4l2.c @@ -1862,7 +1862,7 @@ static int delta_probe(struct platform_device *pdev) if (ret) { dev_err(delta->dev, "%s failed to initialize firmware ipc channel\n", DELTA_PREFIX); - goto err; + goto err_pm_disable; } /* register all available decoders */ @@ -1876,7 +1876,7 @@ static int delta_probe(struct platform_device *pdev) if (ret) { dev_err(delta->dev, "%s failed to register V4L2 device\n", DELTA_PREFIX); - goto err; + goto err_pm_disable; } delta->work_queue = create_workqueue(DELTA_NAME); @@ -1901,6 +1901,8 @@ static int delta_probe(struct platform_device *pdev) destroy_workqueue(delta->work_queue); err_v4l2: v4l2_device_unregister(&delta->v4l2_dev); +err_pm_disable: + pm_runtime_disable(dev); err: return ret; } diff --git a/drivers/media/platform/vsp1/vsp1_rpf.c b/drivers/media/platform/vsp1/vsp1_rpf.c index f8005b60b9d2..abaf4dde3802 100644 --- a/drivers/media/platform/vsp1/vsp1_rpf.c +++ b/drivers/media/platform/vsp1/vsp1_rpf.c @@ -290,11 +290,11 @@ static void rpf_configure_partition(struct vsp1_entity *entity, + crop.left * fmtinfo->bpp[0] / 8; if (format->num_planes > 1) { + unsigned int bpl = format->plane_fmt[1].bytesperline; unsigned int offset; - offset = crop.top * format->plane_fmt[1].bytesperline - + crop.left / fmtinfo->hsub - * fmtinfo->bpp[1] / 8; + offset = crop.top / fmtinfo->vsub * bpl + + crop.left / fmtinfo->hsub * fmtinfo->bpp[1] / 8; mem.addr[1] += offset; mem.addr[2] += offset; } diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c index 21ccbbd70dce..bbb5ff16abd6 100644 --- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c +++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c @@ -2561,6 +2561,11 @@ struct pvr2_hdw *pvr2_hdw_create(struct usb_interface *intf, } while (0); mutex_unlock(&pvr2_unit_mtx); + INIT_WORK(&hdw->workpoll, pvr2_hdw_worker_poll); + + if (hdw->unit_number == -1) + goto fail; + cnt1 = 0; cnt2 = scnprintf(hdw->name+cnt1,sizeof(hdw->name)-cnt1,"pvrusb2"); cnt1 += cnt2; @@ -2572,8 +2577,6 @@ struct pvr2_hdw *pvr2_hdw_create(struct usb_interface *intf, if (cnt1 >= sizeof(hdw->name)) cnt1 = sizeof(hdw->name)-1; hdw->name[cnt1] = 0; - INIT_WORK(&hdw->workpoll,pvr2_hdw_worker_poll); - pvr2_trace(PVR2_TRACE_INIT,"Driver unit number is %d, name is %s", hdw->unit_number,hdw->name); diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c index e858f4f189ed..0371a4a1cd12 100644 --- a/drivers/media/usb/uvc/uvc_v4l2.c +++ b/drivers/media/usb/uvc/uvc_v4l2.c @@ -865,29 +865,31 @@ static int uvc_ioctl_enum_input(struct file *file, void *fh, struct uvc_video_chain *chain = handle->chain; const struct uvc_entity *selector = chain->selector; struct uvc_entity *iterm = NULL; + struct uvc_entity *it; u32 index = input->index; - int pin = 0; if (selector == NULL || (chain->dev->quirks & UVC_QUIRK_IGNORE_SELECTOR_UNIT)) { if (index != 0) return -EINVAL; - list_for_each_entry(iterm, &chain->entities, chain) { - if (UVC_ENTITY_IS_ITERM(iterm)) + list_for_each_entry(it, &chain->entities, chain) { + if (UVC_ENTITY_IS_ITERM(it)) { + iterm = it; break; + } } - pin = iterm->id; } else if (index < selector->bNrInPins) { - pin = selector->baSourceID[index]; - list_for_each_entry(iterm, &chain->entities, chain) { - if (!UVC_ENTITY_IS_ITERM(iterm)) + list_for_each_entry(it, &chain->entities, chain) { + if (!UVC_ENTITY_IS_ITERM(it)) continue; - if (iterm->id == pin) + if (it->id == selector->baSourceID[index]) { + iterm = it; break; + } } } - if (iterm == NULL || iterm->id != pin) + if (iterm == NULL) return -EINVAL; memset(input, 0, sizeof(*input)); diff --git a/drivers/mfd/ipaq-micro.c b/drivers/mfd/ipaq-micro.c index cd762d08f116..2ba0e2d575c0 100644 --- a/drivers/mfd/ipaq-micro.c +++ b/drivers/mfd/ipaq-micro.c @@ -410,7 +410,7 @@ static int __init micro_probe(struct platform_device *pdev) micro_reset_comm(micro); irq = platform_get_irq(pdev, 0); - if (!irq) + if (irq < 0) return -EINVAL; ret = devm_request_irq(&pdev->dev, irq, micro_serial_isr, IRQF_SHARED, "ipaq-micro", diff --git a/drivers/misc/cardreader/rtsx_usb.c b/drivers/misc/cardreader/rtsx_usb.c index b97903ff1a72..869c176d48cc 100644 --- a/drivers/misc/cardreader/rtsx_usb.c +++ b/drivers/misc/cardreader/rtsx_usb.c @@ -678,6 +678,7 @@ static int rtsx_usb_probe(struct usb_interface *intf, return 0; out_init_fail: + usb_set_intfdata(ucr->pusb_intf, NULL); usb_free_coherent(ucr->pusb_dev, IOBUF_SIZE, ucr->iobuf, ucr->iobuf_dma); return ret; diff --git a/drivers/misc/kxrctrl/README.md b/drivers/misc/kxrctrl/README.md new file mode 100644 index 000000000000..9f012d547776 --- /dev/null +++ b/drivers/misc/kxrctrl/README.md @@ -0,0 +1 @@ +# KXRCtrl diff --git a/drivers/misc/kxrctrl/aphost.c b/drivers/misc/kxrctrl/aphost.c new file mode 100644 index 000000000000..29ac561a5394 --- /dev/null +++ b/drivers/misc/kxrctrl/aphost.c @@ -0,0 +1,1376 @@ +/* + * SPI controller driver for the nordic52832 SoCs + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "aphost.h" + +#define A11B_NRF + + +#define CMD_DATA_TAG 0xA6 +#define CMD_CLR_BOND_TAG 0xA7 +#define CMD_REQUEST_TAG (0xA8) +#define CMD_EXTDATA_RLEDTAG 0xB6 + +//#define INTERFACE_ADD + +#define YC_START_HOST _IO('q', 1) +#define YC_STOP_HOST _IO('q', 2) +#define YC_DATA_NOTIFY _IO('q',3) +#define YC_START_VIB _IO('q', 4) +#define YC_STOP_VIB _IO('q', 5) + + +#define YC_GET_DATA + + + +struct jspinctrl_info { + struct pinctrl *pinctrl; + struct pinctrl_state *active; + struct pinctrl_state *suspend; +}; + + +struct js_spi_client { + struct spi_device *spi_client; + struct task_struct *kthread; + struct mutex js_mutex; /* power mutex*/ + struct mutex js_sm_mutex; /*dma alloc and free mutex*/ + + struct jspinctrl_info pinctrl_info; + + int js_lfen_gpio; /*level shift en gpio*/ + int js_irq_gpio; + int js_rled_en_gpio;/*A11B used as all rled trig ,but v02a used as left rled */ + int js_tst2_gpio; /*just old test gpio ,not used in a11b and v02a*/ + int js_dfu_en_gpio; /*dfu enable gpio ,low enable dfu */ + int js_v02a_rled_right_en_gpio; /* A11B not used , V02A useed for right rled */ + int js_v33en_gpio; + int js_ledl_gpio; /*old test ,not used now*/ + int js_ledr_gpio; /*old test ,not used now*/ + int js_irq; + + atomic_t dataflag; + atomic_t rledchg; + atomic_t userRequest; //request from userspace + atomic_t nordicAcknowledge; //ack from nordic52832 master + unsigned char JoyStickBondState; //1:left JoyStick 2:right JoyStick + bool suspend; + + wait_queue_head_t wait_queue; + void *vaddr; + size_t vsize; + struct dma_buf *js_buf; + spinlock_t smem_lock; + + struct miscdevice miscdev; + uint64_t tss; + uint64_t ts_offset; + + unsigned char txbuffer[255]; + unsigned char rxbuffer[255]; + + uint64_t tsHost; /*linux boottime */ + uint64_t tsoffset; /*time offset between two cpu*/ + uint64_t tsoffsetmono;/*linux monotime ,need by app */ + uint64_t tsSyncPt; + uint64_t tsSyncPtmono; + uint32_t tshmd_tmp;/*get the time from hmd*/ + unsigned char SyncPtFlag; + unsigned char powerstate; + bool irqstate; + unsigned char js_lstate; + unsigned char js_rstate; + + struct hrtimer hr_timer; + ktime_t ktime; + + struct usb_device *udev; + struct usb_host_interface *desc; + struct usb_endpoint_descriptor *endpoint; + struct usb_interface *intf; + struct urb *urb; + unsigned int pipe; + u8 ubuffer[128]; + struct work_struct work; + + int memfd; + atomic_t urbstate; +}; + +struct js_spi_client *gspi_client = NULL; + +cp_buffer_t *u_packet=NULL; + +static char checkoutpoint=0; + + +void d_packet_set_instance(cp_buffer_t *in ) +{ + + if(gspi_client==NULL){ + pr_err("js %s: drv init err", __func__); + } + + spin_lock(&gspi_client->smem_lock); + + if(in==NULL){ + u_packet=NULL; + } + else{ + u_packet=in; + u_packet->c_head=-1; + u_packet->p_head=-1; + } + + spin_unlock(&gspi_client->smem_lock); + + if(in==NULL) + pr_err("js %s: release mem", __func__); + else + pr_err("js %s: alloc mem", __func__); + +} + + + +void js_irq_enable(struct js_spi_client *spi_client,bool enable) +{ + + if(spi_client->irqstate==enable){ + pr_err("js irq already =%d ",enable); + return; + } + + pr_err("js irq en =%d ",enable); + if(enable){ + enable_irq(spi_client->js_irq); + } + else{ + disable_irq(spi_client->js_irq); + } + + spi_client->irqstate=enable; + +} + + + +void js_set_power(int jspower) +{ + if(gspi_client) + { + mutex_lock(&gspi_client->js_mutex); + if(gspi_client->powerstate != jspower) + { + if(jspower==0){/*off */ + gspi_client->powerstate=0; + js_irq_enable(gspi_client,false); + gpio_set_value(gspi_client->js_dfu_en_gpio,1); + gpio_set_value(gspi_client->js_lfen_gpio,0); + gpio_set_value(gspi_client->js_v33en_gpio,0); + } + else if(jspower==1){ /*normal on*/ + gpio_set_value(gspi_client->js_dfu_en_gpio,1); + gpio_set_value(gspi_client->js_v33en_gpio,1); + gpio_set_value(gspi_client->js_lfen_gpio,1); + gspi_client->powerstate=1; + js_irq_enable(gspi_client,true); + } + else if(jspower==2){/*dfu*/ + + gspi_client->powerstate=2; + js_irq_enable(gspi_client,false); + gpio_set_value(gspi_client->js_dfu_en_gpio,0); + gpio_set_value(gspi_client->js_v33en_gpio,1); + gpio_set_value(gspi_client->js_lfen_gpio,1); + msleep(100); + } + } + mutex_unlock(&gspi_client->js_mutex); + } +} + +static ssize_t jspower_show(struct device *dev,struct device_attribute *attr, char *buf) +{ + + return sprintf(buf, "%d\n",(unsigned int)gspi_client->powerstate); +} + + +static ssize_t jspower_store(struct device *dev,struct device_attribute *attr, const char *buf, size_t size) +{ + int ctl=0; + + if(sscanf(buf,"%d",&ctl)==1) + { + printk("[%s]set power:%d\n", __func__, ctl); + if(gspi_client) + { + if(ctl==0){ + js_set_power(0); + } + else if(ctl==1){ + js_set_power(1); + } + else if(ctl==2){ + js_set_power(2); + } + + } + } + + return size; +} + +static ssize_t jsmem_show(struct device *dev,struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", gspi_client->memfd); +} + +static ssize_t jsmem_store(struct device *dev,struct device_attribute *attr, const char *buf, size_t count) +{ + int ret; + cp_buffer_t * inbuf; + + ret = kstrtoint(buf, 10, &gspi_client->memfd); + if (ret < 0) + return ret; + + mutex_lock(&gspi_client->js_sm_mutex); + + if (gspi_client->memfd == -1){ + + if (IS_ERR_OR_NULL(gspi_client->vaddr)) + goto __end; + + d_packet_set_instance(NULL); + dma_buf_kunmap(gspi_client->js_buf, 0, gspi_client->vaddr); + dma_buf_end_cpu_access(gspi_client->js_buf, DMA_BIDIRECTIONAL); + dma_buf_put(gspi_client->js_buf); + gspi_client->vaddr = NULL; + gspi_client->js_buf = NULL; + } + else + { + gspi_client->js_buf = dma_buf_get(gspi_client->memfd); + if (IS_ERR_OR_NULL(gspi_client->js_buf)) { + ret = -ENOMEM; + pr_err("[%s]dma_buf_get failed for fd: %d\n", __func__, gspi_client->memfd); + goto __end; + } + + ret = dma_buf_begin_cpu_access(gspi_client->js_buf, DMA_BIDIRECTIONAL); + if (ret) { + pr_err("[%s]: dma_buf_begin_cpu_access failed\n", __func__); + dma_buf_put(gspi_client->js_buf); + gspi_client->js_buf = NULL; + goto __end; + } + + gspi_client->vsize = gspi_client->js_buf->size; + gspi_client->vaddr = dma_buf_kmap(gspi_client->js_buf, 0); + + if (IS_ERR_OR_NULL(gspi_client->vaddr)) { + + dma_buf_end_cpu_access(gspi_client->js_buf, DMA_BIDIRECTIONAL); + dma_buf_put(gspi_client->js_buf); + gspi_client->js_buf = NULL; + pr_err("[%s]dma_buf_kmap failed for fd: %d\n",__func__, gspi_client->memfd); + goto __end; + } + + inbuf=(cp_buffer_t *)gspi_client->vaddr; + d_packet_set_instance(inbuf); + } + +__end: + mutex_unlock(&gspi_client->js_sm_mutex); + + return count; +} + + +static ssize_t jsoffset_show(struct device *dev,struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%llu,%llu\n",gspi_client->tsoffset,gspi_client->tsoffsetmono); +} + +static ssize_t jsrequest_show(struct device *dev,struct device_attribute *attr, char *buf) +{ + unsigned int input = 0; + acknowledge_t nordicAck; + int size = 0; + + mutex_lock(&gspi_client->js_mutex); + memset(&nordicAck, 0, sizeof(acknowledge_t)); + input = atomic_read(&gspi_client->nordicAcknowledge); + atomic_set(&gspi_client->nordicAcknowledge, 0); + nordicAck.acknowledgeHead.requestType = ((input&0x7f000000) >> 24); + nordicAck.acknowledgeHead.ack = ((input&0x80000000) >> 31); + nordicAck.acknowledgeData[0] = (input&0x000000ff); + nordicAck.acknowledgeData[1] = ((input&0x0000ff00) >> 8); + nordicAck.acknowledgeData[2] = ((input&0x00ff0000) >> 16); + + if (nordicAck.acknowledgeHead.ack == 1) + { + switch(nordicAck.acknowledgeHead.requestType) + { + case getMasterNordicVersionRequest: + size = sprintf(buf, "masterNordic fwVersion:%d.%d\n", nordicAck.acknowledgeData[1], nordicAck.acknowledgeData[0]); + break; + case bondJoyStickRequest: + case disconnectJoyStickRequest: + case setVibStateRequest: + case hostEnterDfuStateRequest: + size = sprintf(buf, "requestType:%d ack:%d\n",nordicAck.acknowledgeHead.requestType, nordicAck.acknowledgeHead.ack); + break; + case getJoyStickBondStateRequest: + gspi_client->JoyStickBondState = (nordicAck.acknowledgeData[0]&0x03); + size = sprintf(buf, "left/right joyStick bond state:%d:%d\n", (gspi_client->JoyStickBondState&0x01), ((gspi_client->JoyStickBondState&0x02)>>1)); + break; + case getLeftJoyStickProductNameRequest: + size = sprintf(buf, "leftJoyStick productNameID:%d\n", nordicAck.acknowledgeData[0]); + break; + case getRightJoyStickProductNameRequest: + size = sprintf(buf, "rightJoyStick productNameID:%d\n", nordicAck.acknowledgeData[0]); + break; + case getLeftJoyStickFwVersionRequest: + size = sprintf(buf, "leftJoyStick fwVersion:%d.%d\n", nordicAck.acknowledgeData[1], nordicAck.acknowledgeData[0]); + break; + case getRightJoyStickFwVersionRequest: + size = sprintf(buf, "rightJoyStick fwVersion:%d.%d\n", nordicAck.acknowledgeData[1], nordicAck.acknowledgeData[0]); + break; + default: + size = sprintf(buf, "invalid requestType\n"); + break; + } + } + else + { + size = sprintf(buf, "no need to ack\n"); + } + mutex_unlock(&gspi_client->js_mutex); + return size; +} + +static ssize_t jsrequest_store(struct device *dev,struct device_attribute *attr, const char *buf, size_t size) +{ + + unsigned int input = 0; + request_t request; + int vibState = 0; + + mutex_lock(&gspi_client->js_mutex); + if(sscanf(buf, "%x", &input) == 1) + { + memset(&request, 0, sizeof(request_t)); + request.requestHead.requestType = ((input&0x7f000000) >> 24); + request.requestData[0] = (input&0x000000ff); + request.requestData[1] = (input&0x0000ff00); + request.requestData[2] = (input&0x00ff0000); + + switch(request.requestHead.requestType) + { + case setVibStateRequest: + vibState = ((request.requestData[1] << 8) | request.requestData[0]); + if (vibState >= 0 && vibState <= 0xffff) + { + if(gspi_client) { + atomic_set(&gspi_client->userRequest, input); + atomic_inc(&gspi_client->dataflag); + wake_up_interruptible(&gspi_client->wait_queue); + } + } + else + { + printk("invalid vibState\n"); + memset(&gspi_client->userRequest, 0, sizeof(gspi_client->userRequest)); + } + break; + case getMasterNordicVersionRequest: + case bondJoyStickRequest: + case disconnectJoyStickRequest: + case getJoyStickBondStateRequest: + case hostEnterDfuStateRequest: + case getLeftJoyStickProductNameRequest: + case getRightJoyStickProductNameRequest: + case getLeftJoyStickFwVersionRequest: + case getRightJoyStickFwVersionRequest: + if(gspi_client) + { + atomic_set(&gspi_client->userRequest, input); + atomic_inc(&gspi_client->dataflag); + wake_up_interruptible(&gspi_client->wait_queue); + } + break; + default: + printk("invalid requestType\n"); + memset(&gspi_client->userRequest, 0, sizeof(gspi_client->userRequest)); + return size; + } + } + mutex_unlock(&gspi_client->js_mutex); + + return size; +} + +//static DEVICE_ATTR(jsbond, S_IRUGO|S_IWUSR|S_IWGRP, jsbond_show, jsbond_store); + +static DEVICE_ATTR(jsmem, S_IRUGO|S_IWUSR|S_IWGRP, jsmem_show, jsmem_store); + +static DEVICE_ATTR(jspower, S_IRUGO|S_IWUSR|S_IWGRP, jspower_show, jspower_store); /*external for power ctl ,avold someone */ + +static DEVICE_ATTR(jsoffset, S_IRUGO, jsoffset_show, NULL); + +static DEVICE_ATTR(jsrequest, S_IRUGO|S_IWUSR|S_IWGRP, jsrequest_show, jsrequest_store); + + +static int js_spi_txfr(struct spi_device *spi, char *txbuf,char *rxbuf, int num_byte,uint64_t *tts) +{ + + int ret=0; + + + struct spi_transfer txfr; + struct spi_message msg; + + memset(&txfr, 0, sizeof(txfr)); + txfr.tx_buf = txbuf; + txfr.rx_buf = rxbuf; + txfr.len = num_byte; + spi_message_init(&msg); + spi_message_add_tail(&txfr, &msg); + + *tts=ktime_to_ns(ktime_get_boottime()); + ret=spi_sync(spi, &msg); + + if(ret<0) + { + pr_err(" js xfr err=%d \n",ret); + } + + return ret; +} + + + +#define XFR_SIZE 188 + +int js_thread(void *data) +{ + int ret; + unsigned char *pbuf; + uint64_t tts; + uint32_t tth[8]; + uint64_t tto[8]; + int num = 0; + int pksz = 0; + int index = 0; + uint32_t hosttime; + bool skiprport = false; + unsigned int input = 0; + request_t currentRequest; + static request_t lastRequest; + acknowledge_t nordicAck; + uint8_t val = 0; + struct js_spi_client *spi_client=(struct js_spi_client *)data; + + struct sched_param param = { + .sched_priority = 88 + }; + + sched_setscheduler(current, SCHED_RR, ¶m); + //set_current_state(TASK_INTERRUPTIBLE); + + pr_err(" js_thread start \n"); + + do { + skiprport = false; + ret = wait_event_interruptible(spi_client->wait_queue, atomic_read(&spi_client->dataflag) || kthread_should_stop()); + + if ((ret < 0) || kthread_should_stop()) { + pr_err("%s: exit\n", __func__); + break; + } + atomic_set(&spi_client->dataflag, 0); + + if(spi_client->powerstate != 1){ + msleep(100); + continue; + } + + input = (unsigned int)atomic_read(&gspi_client->userRequest); + + val = gpio_get_value(spi_client->js_irq_gpio); + if(val == 0 && input == 0) //Filter out the exception trigger + { + continue; + } + + memset(¤tRequest, 0, sizeof(request_t)); + currentRequest.requestHead.needAck = ((input&0x80000000) >> 31); + currentRequest.requestHead.requestType = ((input&0x7f000000) >> 24); + currentRequest.requestData[0] = (input&0x000000ff); + currentRequest.requestData[1] = ((input&0x0000ff00) >> 8); + currentRequest.requestData[2] = ((input&0x00ff0000) >> 16); + + memset(spi_client->txbuffer, 0, sizeof(spi_client->txbuffer)); + memset(spi_client->rxbuffer, 0, sizeof(spi_client->rxbuffer)); + spi_client->txbuffer[0] = CMD_REQUEST_TAG; + spi_client->txbuffer[1] = ((currentRequest.requestHead.needAck << 7)|currentRequest.requestHead.requestType); + + + switch(currentRequest.requestHead.requestType) + { + case setVibStateRequest: + spi_client->txbuffer[2] = currentRequest.requestData[0]; + spi_client->txbuffer[3] = currentRequest.requestData[1]; + break; + case bondJoyStickRequest: + case disconnectJoyStickRequest: + spi_client->txbuffer[2] = (currentRequest.requestData[0]&0x01); + break; + default: + break; + } + if(spi_client->powerstate == 1) + { + ret = js_spi_txfr(spi_client->spi_client, spi_client->txbuffer, spi_client->rxbuffer, XFR_SIZE, &tts); + if (ret != 0) + continue; + } + else + { + continue; + } + + if (spi_client->rxbuffer[4] == 0xff) //Filtering dirty Data + { + continue; + } + + if(lastRequest.requestHead.needAck == 1) + { + memset(&nordicAck, 0, sizeof(acknowledge_t)); + nordicAck.acknowledgeHead.ack = ((spi_client->rxbuffer[0]&0x80)>>7); + nordicAck.acknowledgeHead.requestType = (spi_client->rxbuffer[0]&0x7f); + nordicAck.acknowledgeData[0] = spi_client->rxbuffer[1]; + nordicAck.acknowledgeData[1] = spi_client->rxbuffer[2]; + nordicAck.acknowledgeData[2] = spi_client->rxbuffer[3]; + if (lastRequest.requestHead.requestType == nordicAck.acknowledgeHead.requestType) + { + unsigned int input = 0; + + input = ((spi_client->rxbuffer[0]<<24)|(spi_client->rxbuffer[3]<<16)|(spi_client->rxbuffer[2]<<8)|spi_client->rxbuffer[1]); + atomic_set(&spi_client->nordicAcknowledge, input); + } + memset(&lastRequest, 0, sizeof(lastRequest)); + } + + if ((gspi_client->JoyStickBondState&0x03) != 0 && input == 0) //left or right joyStick are bound + //if ((gspi_client->JoyStickBondState&0x03) != 0) //left or right joyStick are bound + { + pksz = spi_client->rxbuffer[4]; + num = spi_client->rxbuffer[5]; + + if(num == 0 || pksz != 30) + { + //pr_err("wjx no joystick data\n"); + skiprport = true; + } + memcpy(&hosttime, &spi_client->rxbuffer[6], 4); + tts = spi_client->tsHost; + + pbuf = &spi_client->rxbuffer[10]; + if(!skiprport){ + /*add Protection if someone release the memory */ + spin_lock(&gspi_client->smem_lock); + + for(index = 0; index < num; index++) + { + memcpy(&tth[index], pbuf, 4); + tto[index] = tts-(hosttime-tth[index])*100000; + if((u_packet)&&(spi_client->vaddr)) + { + int8_t p_head; + d_packet_t *pdata; + + p_head = (u_packet->p_head + 1) % MAX_PACK_SIZE; + pdata = &u_packet->data[p_head]; + pdata->ts = tto[index]; + pdata->size = pksz - 4; + memcpy((void*)pdata->data, (void*)(pbuf+4), pksz-4); + u_packet->p_head = p_head; + } + pbuf += pksz; + } + spin_unlock(&gspi_client->smem_lock); + } + } + + if (currentRequest.requestHead.requestType != 0) + atomic_set(&gspi_client->userRequest, 0); + + memcpy(&lastRequest, ¤tRequest, sizeof(currentRequest)); + } while (1); + + return 0; +} + + +static int js_pinctrl_init(struct js_spi_client *spi_client) +{ + int rc = 0; + + spi_client->pinctrl_info.pinctrl= devm_pinctrl_get(&spi_client->spi_client->dev); + + if (IS_ERR_OR_NULL(spi_client->pinctrl_info.pinctrl)) { + rc = PTR_ERR(spi_client->pinctrl_info.pinctrl); + pr_err("failed pinctrl, rc=%d\n", rc); + goto error; + } + + spi_client->pinctrl_info.active = pinctrl_lookup_state(spi_client->pinctrl_info.pinctrl, "js_default"); + if (IS_ERR_OR_NULL(spi_client->pinctrl_info.active)) { + rc = PTR_ERR(spi_client->pinctrl_info.active); + pr_err("failed pinctrl active state, rc=%d\n", rc); + goto error; + } + + spi_client->pinctrl_info.suspend =pinctrl_lookup_state(spi_client->pinctrl_info.pinctrl, "js_sleep"); + + if (IS_ERR_OR_NULL(spi_client->pinctrl_info.suspend)) { + rc = PTR_ERR(spi_client->pinctrl_info.suspend); + pr_err("failed pinctrl suspend state, rc=%d\n", rc); + goto error; + } + pr_err("js_pinctrl_init ok \n"); + +error: + return rc; +} + + +static int js_parse_gpios(struct js_spi_client *spi_client) +{ + int rc = 0; + + struct device_node *of_node = spi_client->spi_client->dev.of_node; + + spi_client->js_lfen_gpio= of_get_named_gpio(of_node,"js,lfen-gpio", 0); + if (!gpio_is_valid(spi_client->js_lfen_gpio)) { + pr_err("failed get js_lfen_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto error; + } + +#ifdef A11B_NRF + spi_client->js_v33en_gpio= of_get_named_gpio(of_node,"js,v33en-gpio", 0); + if (!gpio_is_valid(spi_client->js_v33en_gpio)) { + pr_err("failed get js_v33en_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto error; + } +#endif + + spi_client->js_irq_gpio= of_get_named_gpio(of_node,"js,irq-gpio", 0); + if (!gpio_is_valid(spi_client->js_irq_gpio)) { + pr_err("failed get js_irq_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto error; + } + +/*not used now*/ + spi_client->js_ledl_gpio= of_get_named_gpio(of_node,"js,ledl", 0); + if (!gpio_is_valid(spi_client->js_ledl_gpio)) { + pr_err("failed get js_ledl_gpio gpio, rc=%d\n", rc); + } + + spi_client->js_ledr_gpio= of_get_named_gpio(of_node,"js,ledr", 0); + if (!gpio_is_valid(spi_client->js_ledr_gpio)) { + pr_err("failed get js_ledr_gpio gpio, rc=%d\n", rc); + } + + + + spi_client->js_rled_en_gpio= of_get_named_gpio(of_node,"js,tst1", 0); + if (!gpio_is_valid(spi_client->js_rled_en_gpio)) { + pr_err("failed get js_rled_en_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto error; + } + + spi_client->js_tst2_gpio= of_get_named_gpio(of_node,"js,tst2", 0); + if (!gpio_is_valid(spi_client->js_tst2_gpio)) { + pr_err("failed get js_tst2_gpio gpio, rc=%d\n", rc); + } + + spi_client->js_dfu_en_gpio= of_get_named_gpio(of_node,"js,tst3", 0); + if (!gpio_is_valid(spi_client->js_dfu_en_gpio)) { + pr_err("failed get js_dfu_en_gpio gpio, rc=%d\n", rc); + } + + spi_client->js_v02a_rled_right_en_gpio= of_get_named_gpio(of_node,"js,tst4", 0); + if (!gpio_is_valid(spi_client->js_v02a_rled_right_en_gpio)) { + pr_err("failed get js_v02a_rled_right_en_gpio gpio, rc=%d\n", rc); + } + +//tst + + +pr_err("js_parse_gpios ok \n"); + + +error: + return rc; +} + + + +static int js_gpio_request(struct js_spi_client *spi_client) +{ + int rc = 0; + + if (gpio_is_valid(spi_client->js_lfen_gpio)) { + + pr_err("request for js_lfen_gpio =%d ", spi_client->js_lfen_gpio); + rc = gpio_request(spi_client->js_lfen_gpio, "js_lfen_gpio"); + if (rc) { + pr_err("request for js_lfen_gpio failed, rc=%d\n", rc); + goto error; + } + } + + +#ifdef A11B_NRF + if (gpio_is_valid(spi_client->js_v33en_gpio)) { + + pr_err("request for js_v33en_gpio =%d ", spi_client->js_v33en_gpio); + rc = gpio_request(spi_client->js_v33en_gpio, "js_v33en_gpio"); + if (rc) { + pr_err("request for js_v33en_gpio failed, rc=%d\n", rc); + goto error; + } + } +#endif + + if (gpio_is_valid(spi_client->js_irq_gpio)) { + + pr_err("request for js_irq_gpio =%d ", spi_client->js_irq_gpio); + rc = gpio_request(spi_client->js_irq_gpio, "js_irq_gpio"); + if (rc) { + pr_err("request for js_irq_gpio failed, rc=%d\n", rc); + goto error; + } + } + + + if (gpio_is_valid(spi_client->js_ledl_gpio)) { + + pr_err("request for js_ledl_gpio =%d ", spi_client->js_ledl_gpio); + rc = gpio_request(spi_client->js_ledl_gpio, "js_ledl_gpio"); + if (rc) { + pr_err("request for js_ledl_gpio failed, rc=%d\n", rc); + } + else + gpio_direction_output(spi_client->js_ledl_gpio,1); + + } + + if (gpio_is_valid(spi_client->js_ledr_gpio)) { + + pr_err("request for js_ledr_gpio =%d ", spi_client->js_ledr_gpio); + rc = gpio_request(spi_client->js_ledr_gpio, "js_ledr_gpio"); + if (rc) { + pr_err("request for js_ledr_gpio failed, rc=%d\n", rc); + } + else + gpio_direction_output(spi_client->js_ledr_gpio,1); + + } + + + + if (gpio_is_valid(spi_client->js_rled_en_gpio)) { + + pr_err("request for js_rled_en_gpio =%d ", spi_client->js_rled_en_gpio); + rc = gpio_request(spi_client->js_rled_en_gpio, "js_rled_en_gpio"); + if (rc) { + pr_err("request for js_rled_en_gpio failed, rc=%d\n", rc); + goto error; + } + + gpio_direction_output(spi_client->js_rled_en_gpio,0); + + } + + + if (gpio_is_valid(spi_client->js_tst2_gpio)) { + + pr_err("request for js_tst2_gpio =%d ", spi_client->js_tst2_gpio); + rc = gpio_request(spi_client->js_tst2_gpio, "js_tst2_gpio"); + if (rc) { + pr_err("request for js_tst2_gpio failed, rc=%d\n", rc); + } + else + gpio_direction_input(spi_client->js_tst2_gpio); + } + + if (gpio_is_valid(spi_client->js_dfu_en_gpio)) { + + pr_err("request for js_dfu_en_gpio =%d ", spi_client->js_dfu_en_gpio); + rc = gpio_request(spi_client->js_dfu_en_gpio, "js_dfu_en_gpio"); + if (rc) { + pr_err("request for js_dfu_en_gpio failed, rc=%d\n", rc); + } + else + gpio_direction_output(spi_client->js_dfu_en_gpio,0); + + } + + if (gpio_is_valid(spi_client->js_v02a_rled_right_en_gpio)) { + + pr_err("request for js_v02a_rled_right_en_gpio =%d ", spi_client->js_v02a_rled_right_en_gpio); + rc = gpio_request(spi_client->js_v02a_rled_right_en_gpio, "js_v02a_rled_right_en_gpio"); + if (rc) { + pr_err("request for js_v02a_rled_right_en_gpio failed, rc=%d\n", rc); + } + else + gpio_direction_output(spi_client->js_v02a_rled_right_en_gpio,0); + //gpio_direction_input(spi_client->js_v02a_rled_right_en_gpio); + + } + + + + pr_err("js_gpio_request ok \n"); + +error: + return rc; +} + +static irqreturn_t js_irq_handler(int irq, void *dev_id) +{ + int val = 0; + struct js_spi_client *spi_client = (struct js_spi_client *)dev_id; + + if(spi_client->powerstate==1) + { + val = gpio_get_value(spi_client->js_irq_gpio); + if(val == 1) + { + //disable_irq_nosync(spi_client->js_irq); + spi_client->tsHost=ktime_to_ns(ktime_get_boottime()); + atomic_inc(&spi_client->dataflag); + wake_up_interruptible(&spi_client->wait_queue); + } + } + return IRQ_HANDLED; +} + + +static int js_io_init(struct js_spi_client *spi_client) +{ + int ret; + + int rc = 0; + + rc=pinctrl_select_state(spi_client->pinctrl_info.pinctrl ,spi_client->pinctrl_info.active); + if (rc) + pr_err("js failed to set pin state, rc=%d\n",rc); + + + gpio_direction_output(spi_client->js_dfu_en_gpio,1); + gpio_direction_output(spi_client->js_v02a_rled_right_en_gpio,0); + + gpio_direction_input(spi_client->js_irq_gpio); + gpio_direction_output(spi_client->js_lfen_gpio,0); + +#ifdef A11B_NRF + gpio_direction_output(spi_client->js_v33en_gpio,0); +#endif + + gpio_direction_output(spi_client->js_lfen_gpio,0); + + spi_client->powerstate=0; + spi_client->js_lstate=1; + spi_client->js_rstate=1; + + spi_client->js_irq = gpio_to_irq(spi_client->js_irq_gpio); + + if (spi_client->js_irq < 0) { + spi_client->js_irq=-1; + pr_err(" js gpio_to_irq err\n"); + } + else{ + ret = request_irq(spi_client->js_irq, js_irq_handler,IRQF_TRIGGER_RISING, "js", spi_client);//IRQF_TRIGGER_FALLING + disable_irq_nosync(spi_client->js_irq); + if(ret<0) + pr_err("js request_irq err =%d \n",spi_client->js_irq); + else + pr_err("js request_irq =%d\n",spi_client->js_irq); + } + + pr_err(" js_io_init ok\n"); + + return 0; +} + + +/* +note: +this fuction used for : +1 notify to JoyStick the trig rled +2 . sync offset for android user space used to synctime . +*/ +static void glass_private_ep_callback(struct urb *urb) +{ + + struct js_spi_client *client; + + client = urb->context; + + if(urb->status==0) { + + if(gspi_client->js_lstate) + gpio_set_value(client->js_rled_en_gpio,1); + + if(gspi_client->js_rstate) + gpio_set_value(client->js_v02a_rled_right_en_gpio,1); + + if(checkoutpoint==0){ + gspi_client->tsSyncPt=ktime_to_ns(ktime_get_boottime()); + gspi_client->tsSyncPtmono=ktime_to_ns(ktime_get()); + + memcpy(&(gspi_client->tshmd_tmp),(void *)(client->ubuffer),4); + gspi_client->SyncPtFlag=1; + } + + hrtimer_start( &client->hr_timer, client->ktime, HRTIMER_MODE_REL ); + } + else + pr_err("js notify_pri_callback err\n"); + + schedule_work(&client->work); + +} + + + +enum hrtimer_restart timecallback( struct hrtimer *timer ) +{ + + struct js_spi_client *spi_client; + spi_client = container_of(timer, struct js_spi_client ,hr_timer); + + gpio_set_value(spi_client->js_rled_en_gpio,0); + gpio_set_value(spi_client->js_v02a_rled_right_en_gpio,0); + + + return HRTIMER_NORESTART; +} + + +static int glass_private_chanel_probe(struct usb_interface *intf,const struct usb_device_id *id) +{ + int ret =0; + + struct js_spi_client *client; + int maxpacket; + + if(gspi_client==NULL) + return -ENOMEM; + + client =gspi_client; + + client->udev = interface_to_usbdev(intf); + + client->desc = intf->cur_altsetting; + + if (client->desc->desc.bNumEndpoints != 1){ + + pr_err("js bNumEndpoints err \n"); + return -1; + } + + client->endpoint= &client->desc->endpoint[0].desc; + + if (!usb_endpoint_is_int_in( client->endpoint)){ + + pr_err("js not ep \n"); + + return -1; + } + + + + usb_set_intfdata(intf, client); + + + client->intf = intf; + + + client->urb = usb_alloc_urb(0, GFP_KERNEL); + if (!client->urb){ + return -ENOMEM; + } + + client->pipe = usb_rcvintpipe(client->udev, client->endpoint->bEndpointAddress); + + + maxpacket = usb_maxpacket(client->udev, client->pipe, usb_pipeout(client->pipe)); + + usb_fill_int_urb(client->urb, client->udev, client->pipe, client->ubuffer, maxpacket,glass_private_ep_callback, client, client->endpoint->bInterval); + + atomic_set(&client->urbstate, 1); + + ret = usb_submit_urb(client->urb, GFP_KERNEL); + if (ret < 0) { + pr_err("js usb_submit_urb err =%d \n",ret); + usb_free_urb(client->urb); + return ret ; + } + + client->tsoffset=0; + client->tsoffsetmono=0; + checkoutpoint=0; + + return ret; + + +} + + +static void glass_private_chanel_disconnect(struct usb_interface *intf) +{ + struct js_spi_client *client; + + client = usb_get_intfdata(intf); + + client->tsoffset=0; + client->tsoffsetmono=0; + + atomic_set(&client->urbstate, 0); + + usb_poison_urb(client->urb); + + usb_free_urb(client->urb); + + checkoutpoint=0; +} + + + +static int glass_private_chanel_ioctl(struct usb_interface *intf, unsigned int code, void *user_data) +{ + +// todo if need + return -ENOSYS; +} + +static const struct usb_device_id yc_id_table[] = { + { .match_flags = USB_DEVICE_ID_MATCH_DEVICE|USB_DEVICE_ID_MATCH_INT_CLASS, + .idVendor = 0x045e, + .idProduct = 0x0659, + .bInterfaceClass = 0xfe}, + + { } +}; + +MODULE_DEVICE_TABLE(usb, hub_id_table); + + +static struct usb_driver pri_driver = { + .name = "yc", + .probe = glass_private_chanel_probe, + .disconnect = glass_private_chanel_disconnect, + .unlocked_ioctl = glass_private_chanel_ioctl, + .id_table = yc_id_table, +}; + + + +/*note : used to calculate time offset for app use +move from irq to work for irq perfermance . +*/ +static void ts_offset_update_event(struct work_struct *pwork) +{ + struct js_spi_client *client = container_of(pwork, struct js_spi_client, work); + + + int check= atomic_read(&client->urbstate); + + if(check==0) return; + + + usb_submit_urb(client->urb, GFP_KERNEL); + + if(gspi_client->SyncPtFlag==1){ + + gspi_client->SyncPtFlag=0; + + gspi_client->tsoffset = gspi_client->tsSyncPt-300000 - (uint64_t)(gspi_client->tshmd_tmp)*1000000; + gspi_client->tsoffsetmono = gspi_client->tsSyncPtmono-300000 - (uint64_t)(gspi_client->tshmd_tmp)*1000000; + //pr_err("js: offset:=%llu adr =%llu, hmdts=%d\n",gspi_client->tsoffset,gspi_client->tsSyncPt,gspi_client->tshmd_tmp); + } + + checkoutpoint++; + if(checkoutpoint==30) + checkoutpoint=0; + +} + +static int js_spi_setup(struct spi_device *spi) +{ + struct js_spi_client *spi_client; + int rc = 0; + + + pr_err("js js_spi_setup 1 \n"); + + if((spi->dev.of_node)==NULL){ + + pr_err("js failed to check of_node \n"); + return -ENOMEM; + + } + pr_err("js js_spi_setup 2 \n"); + + spi_client = kzalloc(sizeof(*spi_client), GFP_KERNEL); + if (!spi_client) { + + pr_err("js failed to malloc \n"); + return -ENOMEM; + } + + + + pr_err("js js_spi_setup 3 \n"); + + spi_client->spi_client = spi; + + rc=js_parse_gpios(spi_client); + if (rc) { + pr_err("js failed to parse gpio, rc=%d\n", rc); + goto spi_free; + } + + + rc =js_pinctrl_init(spi_client); + if (rc) { + pr_err("js failed to init pinctrl, rc=%d\n", rc); + goto spi_free; + } + + rc = js_gpio_request(spi_client); + if (rc) { + pr_err("js failed to request gpios, rc=%d\n",rc); + goto spi_free; + } + + atomic_set(&spi_client->dataflag, 0); + atomic_set(&spi_client->userRequest, 0); + atomic_set(&spi_client->nordicAcknowledge, 0); + + mutex_init(&(spi_client->js_mutex)); + mutex_init(&(spi_client->js_sm_mutex)); + spin_lock_init(&spi_client->smem_lock); + init_waitqueue_head(&spi_client->wait_queue); + dev_set_drvdata(&spi->dev, spi_client); + + device_create_file(&spi->dev, &dev_attr_jsmem); + device_create_file(&spi->dev, &dev_attr_jspower); + device_create_file(&spi->dev, &dev_attr_jsoffset); + device_create_file(&spi->dev, &dev_attr_jsrequest); + + spi_client->suspend=false; + spi_client->vaddr =NULL; + spi_client->tsoffset=0; + spi_client->tsoffsetmono=0; + + gspi_client = spi_client; + + spi_client->kthread =kthread_run(js_thread, spi_client, "jsthread"); + if (IS_ERR(spi_client->kthread)) + pr_err("js kernel_thread failed\r\n" ); + + js_io_init(spi_client); + + spi_client->ktime = ktime_set(0, 200000); + hrtimer_init( &spi_client->hr_timer, CLOCK_BOOTTIME, HRTIMER_MODE_REL); + spi_client->hr_timer.function = timecallback; + + INIT_WORK(&spi_client->work, ts_offset_update_event); + + atomic_set(&spi_client->urbstate, 0); + usb_register(&pri_driver); + + js_set_power(1); + return rc; +spi_free: + kfree(spi_client); + return rc; +} + + + +static int js_spi_suspend(struct device *dev) +{ + + + struct js_spi_client *spi_client; + + if (!dev) + return -EINVAL; + + spi_client = dev_get_drvdata(dev); + if (!spi_client) + return -EINVAL; + + spi_client->suspend=true ; + js_set_power(0); + pr_err("js_spi_suspend\n"); + + return 0; + + +} + + +/* v02a called by external module to trig the joystick rled */ +void external_ctl_gpio(u8 mask ) +{ + if(gspi_client) + { + if (gpio_is_valid(gspi_client->js_rled_en_gpio)) + { + if(mask&0x01) + gpio_set_value(gspi_client->js_rled_en_gpio,1); + else + gpio_set_value(gspi_client->js_rled_en_gpio,0); + } + + if (gpio_is_valid(gspi_client->js_v02a_rled_right_en_gpio)) + { + if(mask&0x02) + gpio_set_value(gspi_client->js_v02a_rled_right_en_gpio,1); + else + gpio_set_value(gspi_client->js_v02a_rled_right_en_gpio,0); + } + } +} + + + +static int js_spi_resume(struct device *dev) +{ + struct js_spi_client *spi_client; + + if (!dev) + return -EINVAL; + + spi_client = dev_get_drvdata(dev); + if (!spi_client) + return -EINVAL; + + js_set_power(1); + spi_client->suspend=false; + pr_err("[%s] exit\n", __func__); + return 0; +} + + + +static int js_spi_driver_probe(struct spi_device *spi) +{ + int ret; + + pr_err("js_spi_driver_probe start"); + + spi->bits_per_word = 8; + spi->mode = SPI_MODE_0; + + spi->max_speed_hz = 8*1000*1000; + + ret=spi_setup(spi); + + if (ret < 0){ + pr_err("js spi_setup failed ret=%d",ret); + return ret; + } + + pr_err("js_spi_driver_probe ok"); + + return js_spi_setup(spi); +} + + + + +static int js_spi_driver_remove(struct spi_device *sdev) +{ + return 0; +} + + + + +static const struct of_device_id js_dt_match[] = { + { .compatible = "yc,js" }, + { } +}; + + +static const struct dev_pm_ops js_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(js_spi_suspend, js_spi_resume) +}; + + +static struct spi_driver js_spi_driver = { + .driver = { + .name = "yc,js", + .owner = THIS_MODULE, + .of_match_table = js_dt_match, + .pm = &js_pm_ops, + }, + .probe = js_spi_driver_probe, + .remove = js_spi_driver_remove, + //.suspend = js_spi_suspend, + //.resume = js_spi_resume, + +}; + + +static int __init js_driver_init(void) +{ + int rc = 0; + + pr_err("js_driver_init"); + + rc = spi_register_driver(&js_spi_driver); + if (rc < 0) { + pr_err("spi_register_driver failed rc = %d", rc); + return rc; + } + + return rc; +} + +static void __exit js_driver_exit(void) +{ + spi_unregister_driver(&js_spi_driver); +} + + +module_init(js_driver_init); //late_initcall +module_exit(js_driver_exit); +MODULE_DESCRIPTION("joystick nordic52832 driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/misc/kxrctrl/aphost.h b/drivers/misc/kxrctrl/aphost.h new file mode 100644 index 000000000000..d7829a7bc8b1 --- /dev/null +++ b/drivers/misc/kxrctrl/aphost.h @@ -0,0 +1,136 @@ +/* + * SPI controller driver for the nordic52832 SoCs + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#ifndef __APHOST_H__ +#define __APHOST_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include + +#define MAX_PACK_SIZE 100 +#define MAX_DATA_SIZE 32 + +typedef struct { + uint64_t ts; + uint32_t size; + uint8_t data[MAX_DATA_SIZE]; +} d_packet_t; + + +typedef struct { +volatile int8_t c_head; +volatile int8_t p_head; +volatile int8_t packDS; +d_packet_t data[MAX_PACK_SIZE]; +}cp_buffer_t; + +typedef enum _requestType_t +{ + getMasterNordicVersionRequest = 1, + setVibStateRequest, + bondJoyStickRequest, + disconnectJoyStickRequest, + getJoyStickBondStateRequest, + hostEnterDfuStateRequest, + getLeftJoyStickProductNameRequest, + getRightJoyStickProductNameRequest, + getLeftJoyStickFwVersionRequest, + getRightJoyStickFwVersionRequest, + invalidRequest, +}requestType_t; + +typedef struct _request_t +{ + struct _requestHead + { + unsigned char requestType:7; + unsigned char needAck:1; //1:need to ack 0:don't need to ack + } requestHead; + unsigned char requestData[3]; +}request_t; + +typedef struct _acknowledge_t +{ + struct _acknowledgeHead + { + unsigned char requestType:7; + unsigned char ack:1; //1:ack 0:not ack + } acknowledgeHead; + unsigned char acknowledgeData[3]; +}acknowledge_t; +#endif //__APHOST_H__ diff --git a/drivers/misc/kxrctrl/local_rled_chip.c b/drivers/misc/kxrctrl/local_rled_chip.c new file mode 100644 index 000000000000..fdc4e109a40c --- /dev/null +++ b/drivers/misc/kxrctrl/local_rled_chip.c @@ -0,0 +1,618 @@ +/* + * controller's IR driver for the nordic52832 SoCs + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + +struct rled_chip_io { + + int torch_en; + int flash_en; + int charge_en; + int flag_pin; /*rled chip flag pin from chip to cpu,indicate chip state*/ + int irq; +}; + + + +struct local_rled_chip { + struct mutex change_lock;/*rled state chagen lock*/ + struct pinctrl *pinctrl; + struct pinctrl_state *active; + struct pinctrl_state *suspend; + int rled_trig_gpio; + int irq; /*from the camera sensor trig the irq*/ + struct rled_chip_io chip1; + struct rled_chip_io chip2; + struct hrtimer hr_timer; + ktime_t ktime; + int grled_chipenabled; + int grled_swflag;/*Every other frame the trig led will use ,not all frame */ + +}; + + +static enum hrtimer_restart timercallback( struct hrtimer *timer ) +{ + + struct local_rled_chip *rled_data; + rled_data = container_of(timer, struct local_rled_chip,hr_timer); + + gpio_set_value(rled_data->chip1.flash_en, 0); + gpio_set_value(rled_data->chip2.flash_en, 0); + + //enable_irq(rled_data->irq); + //pr_err("local_rled timercallback\n"); + + return HRTIMER_NORESTART; +} + + +static irqreturn_t rled_irq_handler(int irq, void *dev_id) +{ + int val=0; + struct local_rled_chip *rled_data=(struct local_rled_chip *)dev_id; + + if(rled_data->grled_chipenabled==0) + { + gpio_set_value(rled_data->chip1.flash_en,0); + gpio_set_value(rled_data->chip2.flash_en,0); + return IRQ_HANDLED; + } + + val=gpio_get_value(rled_data->rled_trig_gpio); + if(val==1) + { + if(rled_data->grled_swflag) + { + gpio_set_value(rled_data->chip1.flash_en,1); + gpio_set_value(rled_data->chip2.flash_en,1); + } + + //disable_irq_nosync(rled_data->irq); + //hrtimer_start( &rled_data->hr_timer, rled_data->ktime, HRTIMER_MODE_REL ); + } + else + { + gpio_set_value(rled_data->chip1.flash_en,0); + gpio_set_value(rled_data->chip2.flash_en,0); + } + + //pr_err("rled_irq_handler val=%d\n",val); + + return IRQ_HANDLED; +} + + + + +/*just test code for the rled chip */ + +static int grled_enabled=0; +static int grled_mode=0;// 0 flash 1,torch ,0x80 charge enable +static struct local_rled_chip *gprled_data=NULL; + + +static ssize_t rledtest_show(struct device *dev,struct device_attribute *attr, char *buf) +{ + + return sprintf(buf, "mode[%d]enable[%d]\n",grled_mode,grled_enabled ); +} + + +static ssize_t rledtest_store(struct device *dev,struct device_attribute *attr, const char *buf, size_t size) +{ + + if(gprled_data==NULL)return size; + + if(sscanf(buf,"%d:%d",&grled_mode,&grled_enabled)==2) + { + + pr_err("local_rled mode =%d, rled = %d \n",grled_mode,grled_enabled); + + disable_irq_nosync(gprled_data->irq);// + + if((grled_mode&0x80)==0x80){ //TODO + if(grled_enabled){ + gpio_set_value(gprled_data->chip1.charge_en,1); + gpio_set_value(gprled_data->chip2.charge_en,1); + } + else + { + gpio_set_value(gprled_data->chip1.charge_en,0); + gpio_set_value(gprled_data->chip2.charge_en,0); + } + + } + + + if((grled_mode&0x7f)==0){ + if(grled_enabled){ + gpio_set_value(gprled_data->chip1.flash_en,1); + gpio_set_value(gprled_data->chip2.flash_en,1); + } + else + { + gpio_set_value(gprled_data->chip1.flash_en,0); + gpio_set_value(gprled_data->chip2.flash_en,0); + } + + } + + if((grled_mode&0x7f)==1){ + if(grled_enabled){ + gpio_set_value(gprled_data->chip1.torch_en,1); + gpio_set_value(gprled_data->chip2.torch_en,1); + } + else + { + gpio_set_value(gprled_data->chip1.torch_en,0); + gpio_set_value(gprled_data->chip2.torch_en,0); + } + + } + + + + + } + + return size; +} +/*just test code for the rled chip end*/ + + +static ssize_t rledchipenable_show(struct device *dev,struct device_attribute *attr, char *buf) +{ + + return sprintf(buf, "rledchipenable[%d]\n",gprled_data->grled_chipenabled ); +} + + +static ssize_t rledchipenable_store(struct device *dev,struct device_attribute *attr, const char *buf, size_t size) +{ + int enableflag=0; + + if(gprled_data==NULL)return size; + + if(sscanf(buf,"%d",&enableflag)==1) + { + pr_err("local_rled enableflag = %d \n",enableflag); + + mutex_lock(&gprled_data->change_lock); + if(enableflag==1){ + + if(gprled_data->grled_chipenabled==0){ + gprled_data->grled_chipenabled=1; + enable_irq(gprled_data->irq); + } + } + else{ + + if(gprled_data->grled_chipenabled==1){ + gprled_data->grled_chipenabled=0; + disable_irq_nosync(gprled_data->irq); + } + //if irq is not trig when led is on ,disable here . + gpio_set_value(gprled_data->chip1.flash_en,0); + gpio_set_value(gprled_data->chip2.flash_en,0); + } + + mutex_unlock(&gprled_data->change_lock); + } + + return size; +} + + + +extern void external_ctl_gpio(u8 mask); + +static ssize_t grledswflag_show(struct device *dev,struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "rledchipenable[%d]\n",gprled_data->grled_swflag ); +} + + +/*used to ctl V02A joystick */ +static ssize_t grledswflag_store(struct device *dev,struct device_attribute *attr, const char *buf, size_t size) +{ + int swflag=0; + + + if(gprled_data==NULL)return size; + + if(sscanf(buf,"%d",&swflag)==1) + { + + if(swflag){ + external_ctl_gpio((u8)swflag); + gprled_data->grled_swflag=1; + } + else{ + external_ctl_gpio(0); + gprled_data->grled_swflag=0; + + } + + // pr_err("local_rled grledswflag = %d \n",gprled_data->grled_swflag); + + } + + return size; +} + + + +static DEVICE_ATTR(rledtest, S_IRUGO|S_IWUSR|S_IWGRP, rledtest_show, rledtest_store); +static DEVICE_ATTR(rledchipenable, S_IRUGO|S_IWUSR|S_IWGRP, rledchipenable_show, rledchipenable_store); +static DEVICE_ATTR(grledswflag, S_IRUGO|S_IWUSR|S_IWGRP, grledswflag_show, grledswflag_store); + + +static int rled_local_probe(struct platform_device *pdev) +{ + + struct device *dev = &pdev->dev; + struct device_node *of_node = dev->of_node; + + int rc = 0; + + struct local_rled_chip *rled_data; + + if(of_node==NULL){ + pr_err("failed to check of_node \n"); + return -ENOMEM; + } + + rled_data = kzalloc(sizeof(*rled_data), GFP_KERNEL); + if (!rled_data) { + return -ENOMEM; + } + + rled_data->pinctrl= devm_pinctrl_get(dev); + + if (IS_ERR_OR_NULL(rled_data->pinctrl)) { + rc = PTR_ERR(rled_data->pinctrl); + pr_err("failed pinctrl, rc=%d\n", rc); + goto data_free; + } + + rled_data->active = pinctrl_lookup_state(rled_data->pinctrl, "pin_default"); + if (IS_ERR_OR_NULL(rled_data->active)) { + rc = PTR_ERR(rled_data->active); + pr_err("failed pinctrl active state, rc=%d\n", rc); + goto data_free; + } + + rled_data->suspend =pinctrl_lookup_state(rled_data->pinctrl, "pin_sleep"); + + if (IS_ERR_OR_NULL(rled_data->suspend)) { + rc = PTR_ERR(rled_data->suspend); + pr_err("failed pinctrl suspend state, rc=%d\n", rc); + goto data_free; + } + pr_err("rled_pinctrl t ok \n"); + + + rled_data->rled_trig_gpio= of_get_named_gpio(of_node,"yc,irq-gpio", 0); + if (!gpio_is_valid(rled_data->rled_trig_gpio)) { + pr_err("failed get rled_trig_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + + // 1 + + rled_data->chip1.flag_pin= of_get_named_gpio(of_node,"yc,flag1_gpio", 0); + if (!gpio_is_valid(rled_data->chip1.flag_pin)) { + pr_err("failed get flag1_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + rled_data->chip1.charge_en= of_get_named_gpio(of_node,"yc,chgen1_gpio", 0); + if (!gpio_is_valid(rled_data->chip1.charge_en)) { + pr_err("failed get chgen1_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + rled_data->chip1.flash_en= of_get_named_gpio(of_node,"yc,flash1_gpio", 0); + if (!gpio_is_valid(rled_data->chip1.flash_en)) { + pr_err("failed get flash1_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + rled_data->chip1.torch_en= of_get_named_gpio(of_node,"yc,torch1_gpio", 0); + if (!gpio_is_valid(rled_data->chip1.torch_en)) { + pr_err("failed get torch1_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + + // 2 + rled_data->chip2.flag_pin= of_get_named_gpio(of_node,"yc,flag2_gpio", 0); + if (!gpio_is_valid(rled_data->chip2.flag_pin)) { + pr_err("failed get flag2_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + rled_data->chip2.charge_en= of_get_named_gpio(of_node,"yc,chgen2_gpio", 0); + if (!gpio_is_valid(rled_data->chip2.charge_en)) { + pr_err("failed get chgen2_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + rled_data->chip2.flash_en= of_get_named_gpio(of_node,"yc,flash2_gpio", 0); + if (!gpio_is_valid(rled_data->chip2.flash_en)) { + pr_err("failed get flash2_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + rled_data->chip2.torch_en= of_get_named_gpio(of_node,"yc,torch2_gpio", 0); + if (!gpio_is_valid(rled_data->chip2.torch_en)) { + pr_err("failed get torch2_gpio gpio, rc=%d\n", rc); + rc = -EINVAL; + goto data_free; + } + + + + pr_err("rled_get_gpio ok \n"); + +//---------------------------------------------------------- + if (gpio_is_valid(rled_data->rled_trig_gpio)) { + + pr_err("request for rled_trig_gpio =%d ", rled_data->rled_trig_gpio); + rc = devm_gpio_request(dev,rled_data->rled_trig_gpio, "rled_trig_gpio"); + if (rc) { + pr_err("request for rled_trig_gpio failed, rc=%d\n", rc); + goto data_free; + } + + } + + + if (gpio_is_valid(rled_data->chip1.flag_pin)) { + + rc = devm_gpio_request(dev,rled_data->chip1.flag_pin, "rled1_flag_pin"); + if (rc) { + pr_err("request for rled1_flag_pin failed, rc=%d\n", rc); + goto data_free; + } + } + + if (gpio_is_valid(rled_data->chip1.charge_en)) { + + rc = devm_gpio_request(dev,rled_data->chip1.charge_en, "rled1_charge_en"); + if (rc) { + pr_err("request for rled1_charge_en failed, rc=%d\n", rc); + goto data_free; + } + } + + if (gpio_is_valid(rled_data->chip1.flash_en)) { + + rc = devm_gpio_request(dev,rled_data->chip1.flash_en, "rled1_flash_en"); + if (rc) { + pr_err("request for rled1_flash_en failed, rc=%d\n", rc); + goto data_free; + } + } + + if (gpio_is_valid(rled_data->chip1.torch_en)) { + + rc = devm_gpio_request(dev,rled_data->chip1.torch_en, "rled1_torch_en"); + if (rc) { + pr_err("request for rled1_torch_en failed, rc=%d\n", rc); + goto data_free; + } + } + +// + if (gpio_is_valid(rled_data->chip2.flag_pin)) { + + rc = devm_gpio_request(dev,rled_data->chip2.flag_pin, "rled2_flag_pin"); + if (rc) { + pr_err("request for rled2_flag_pin failed, rc=%d\n", rc); + goto data_free; + } + } + + if (gpio_is_valid(rled_data->chip2.charge_en)) { + + rc = devm_gpio_request(dev,rled_data->chip2.charge_en, "rled2_charge_en"); + if (rc) { + pr_err("request for rled2_charge_en failed, rc=%d\n", rc); + goto data_free; + } + } + + if (gpio_is_valid(rled_data->chip2.flash_en)) { + + rc = devm_gpio_request(dev,rled_data->chip2.flash_en, "rled2_flash_en"); + if (rc) { + pr_err("request for rled2_flash_en failed, rc=%d\n", rc); + goto data_free; + } + } + + if (gpio_is_valid(rled_data->chip2.torch_en)) { + + rc = devm_gpio_request(dev,rled_data->chip2.torch_en, "rled2_torch_en"); + if (rc) { + pr_err("request for rled2_torch_en failed, rc=%d\n", rc); + goto data_free; + } + } + + + pr_err("rled_gpio_request ok \n"); + + + +//------------ + rc=pinctrl_select_state(rled_data->pinctrl ,rled_data->active); + if (rc) + pr_err("rled failed to set pin state, rc=%d\n",rc); + + + gpio_direction_output(rled_data->chip1.charge_en, 0); + gpio_direction_output(rled_data->chip1.flash_en, 0); + gpio_direction_output(rled_data->chip1.torch_en, 0); + gpio_direction_input(rled_data->chip1.flag_pin); + + gpio_direction_output(rled_data->chip2.charge_en, 0); + gpio_direction_output(rled_data->chip2.flash_en, 0); + gpio_direction_output(rled_data->chip2.torch_en, 0); + gpio_direction_input(rled_data->chip2.flag_pin); + + gpio_direction_input(rled_data->rled_trig_gpio); + + rled_data->irq = gpio_to_irq(rled_data->rled_trig_gpio); + + if (rled_data->irq < 0) { + rled_data->irq=-1; + pr_err(" rled_irq err\n"); + } + else{ + rled_data->grled_chipenabled=1; + rled_data->grled_swflag=1; + rc = devm_request_irq(dev,rled_data->irq, rled_irq_handler,IRQF_TRIGGER_RISING|IRQF_TRIGGER_FALLING , "rled", rled_data);//IRQF_TRIGGER_FALLING + //disable_irq(rled_data->irq); + if(rc<0) + pr_err("rled request_irq err =%d \n",rled_data->irq); + + pr_err(" yc request_irq =%d\n",rled_data->irq); + + + rled_data->ktime = ktime_set(0, 5000000); + + hrtimer_init( &rled_data->hr_timer, CLOCK_BOOTTIME, HRTIMER_MODE_REL); + + rled_data->hr_timer.function = timercallback; + + gpio_direction_output(rled_data->chip2.charge_en, 1); + gpio_direction_output(rled_data->chip1.charge_en, 1); + } + + + device_create_file(dev, &dev_attr_rledtest); + device_create_file(dev, &dev_attr_rledchipenable); + device_create_file(dev, &dev_attr_grledswflag); + + gprled_data=rled_data; + + mutex_init(&rled_data->change_lock); + + return rc; + +data_free: + kfree(rled_data); + return rc; + +} + + + +static const struct of_device_id rled_test_match_table[] = { + { .compatible = "yc,rled_test", + }, + {} +}; + +static struct platform_driver local_rled_driver = { + .driver = { + .name = "yc,rled_test", + .of_match_table = rled_test_match_table, + }, + .probe = rled_local_probe, +}; + + + +static int __init rled_local_init(void) +{ + return platform_driver_register(&local_rled_driver); + +} + +static void __exit rled_local_exit(void) +{ + return platform_driver_unregister(&local_rled_driver); +} + + +module_init(rled_local_init); +module_exit(rled_local_exit); +MODULE_DESCRIPTION("nordic52832 rled driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c index 4c65add5f457..2e065a4ad318 100644 --- a/drivers/misc/lkdtm/usercopy.c +++ b/drivers/misc/lkdtm/usercopy.c @@ -30,12 +30,12 @@ static const unsigned char test_text[] = "This is a test.\n"; */ static noinline unsigned char *trick_compiler(unsigned char *stack) { - return stack + 0; + return stack + unconst; } static noinline unsigned char *do_usercopy_stack_callee(int value) { - unsigned char buf[32]; + unsigned char buf[128]; int i; /* Exercise stack to avoid everything living in registers. */ @@ -43,7 +43,12 @@ static noinline unsigned char *do_usercopy_stack_callee(int value) buf[i] = value & 0xff; } - return trick_compiler(buf); + /* + * Put the target buffer in the middle of stack allocation + * so that we don't step on future stack users regardless + * of stack growth direction. + */ + return trick_compiler(&buf[(128/2)-32]); } static noinline void do_usercopy_stack(bool to_user, bool bad_frame) @@ -66,6 +71,12 @@ static noinline void do_usercopy_stack(bool to_user, bool bad_frame) bad_stack -= sizeof(unsigned long); } +#ifdef ARCH_HAS_CURRENT_STACK_POINTER + pr_info("stack : %px\n", (void *)current_stack_pointer); +#endif + pr_info("good_stack: %px-%px\n", good_stack, good_stack + sizeof(good_stack)); + pr_info("bad_stack : %px-%px\n", bad_stack, bad_stack + sizeof(good_stack)); + user_addr = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANONYMOUS | MAP_PRIVATE, 0); diff --git a/drivers/mtd/chips/cfi_cmdset_0002.c b/drivers/mtd/chips/cfi_cmdset_0002.c index 3ab75d3e2ce3..fd1814111086 100644 --- a/drivers/mtd/chips/cfi_cmdset_0002.c +++ b/drivers/mtd/chips/cfi_cmdset_0002.c @@ -49,6 +49,10 @@ #define SST49LF008A 0x005a #define AT49BV6416 0x00d6 +enum cfi_quirks { + CFI_QUIRK_DQ_TRUE_DATA = BIT(0), +}; + static int cfi_amdstd_read (struct mtd_info *, loff_t, size_t, size_t *, u_char *); static int cfi_amdstd_write_words(struct mtd_info *, loff_t, size_t, size_t *, const u_char *); static int cfi_amdstd_write_buffers(struct mtd_info *, loff_t, size_t, size_t *, const u_char *); @@ -365,6 +369,15 @@ static void fixup_s29ns512p_sectors(struct mtd_info *mtd) mtd->name); } +static void fixup_quirks(struct mtd_info *mtd) +{ + struct map_info *map = mtd->priv; + struct cfi_private *cfi = map->fldrv_priv; + + if (cfi->mfr == CFI_MFR_AMD && cfi->id == 0x0c01) + cfi->quirks |= CFI_QUIRK_DQ_TRUE_DATA; +} + /* Used to fix CFI-Tables of chips without Extended Query Tables */ static struct cfi_fixup cfi_nopri_fixup_table[] = { { CFI_MFR_SST, 0x234a, fixup_sst39vf }, /* SST39VF1602 */ @@ -403,6 +416,7 @@ static struct cfi_fixup cfi_fixup_table[] = { #if !FORCE_WORD_WRITE { CFI_MFR_ANY, CFI_ID_ANY, fixup_use_write_buffers }, #endif + { CFI_MFR_ANY, CFI_ID_ANY, fixup_quirks }, { 0, 0, NULL } }; static struct cfi_fixup jedec_fixup_table[] = { @@ -731,50 +745,46 @@ static struct mtd_info *cfi_amdstd_setup(struct mtd_info *mtd) } /* - * Return true if the chip is ready. + * Return true if the chip is ready and has the correct value. * * Ready is one of: read mode, query mode, erase-suspend-read mode (in any * non-suspended sector) and is indicated by no toggle bits toggling. * + * Error are indicated by toggling bits or bits held with the wrong value, + * or with bits toggling. + * * Note that anything more complicated than checking if no bits are toggling * (including checking DQ5 for an error status) is tricky to get working * correctly and is therefore not done (particularly with interleaved chips * as each chip must be checked independently of the others). */ -static int __xipram chip_ready(struct map_info *map, unsigned long addr) +static int __xipram chip_ready(struct map_info *map, unsigned long addr, + map_word *expected) { map_word d, t; + int ret; d = map_read(map, addr); t = map_read(map, addr); - return map_word_equal(map, d, t); + ret = map_word_equal(map, d, t); + + if (!ret || !expected) + return ret; + + return map_word_equal(map, t, *expected); } -/* - * Return true if the chip is ready and has the correct value. - * - * Ready is one of: read mode, query mode, erase-suspend-read mode (in any - * non-suspended sector) and it is indicated by no bits toggling. - * - * Error are indicated by toggling bits or bits held with the wrong value, - * or with bits toggling. - * - * Note that anything more complicated than checking if no bits are toggling - * (including checking DQ5 for an error status) is tricky to get working - * correctly and is therefore not done (particularly with interleaved chips - * as each chip must be checked independently of the others). - * - */ -static int __xipram chip_good(struct map_info *map, unsigned long addr, map_word expected) +static int __xipram chip_good(struct map_info *map, unsigned long addr, + map_word *expected) { - map_word oldd, curd; + struct cfi_private *cfi = map->fldrv_priv; + map_word *datum = expected; - oldd = map_read(map, addr); - curd = map_read(map, addr); + if (cfi->quirks & CFI_QUIRK_DQ_TRUE_DATA) + datum = NULL; - return map_word_equal(map, oldd, curd) && - map_word_equal(map, curd, expected); + return chip_ready(map, addr, datum); } static int get_chip(struct map_info *map, struct flchip *chip, unsigned long adr, int mode) @@ -791,7 +801,7 @@ static int get_chip(struct map_info *map, struct flchip *chip, unsigned long adr case FL_STATUS: for (;;) { - if (chip_ready(map, adr)) + if (chip_ready(map, adr, NULL)) break; if (time_after(jiffies, timeo)) { @@ -829,7 +839,7 @@ static int get_chip(struct map_info *map, struct flchip *chip, unsigned long adr chip->state = FL_ERASE_SUSPENDING; chip->erase_suspended = 1; for (;;) { - if (chip_ready(map, adr)) + if (chip_ready(map, adr, NULL)) break; if (time_after(jiffies, timeo)) { @@ -1360,7 +1370,7 @@ static int do_otp_lock(struct map_info *map, struct flchip *chip, loff_t adr, /* wait for chip to become ready */ timeo = jiffies + msecs_to_jiffies(2); for (;;) { - if (chip_ready(map, adr)) + if (chip_ready(map, adr, NULL)) break; if (time_after(jiffies, timeo)) { @@ -1630,7 +1640,8 @@ static int __xipram do_write_oneword(struct map_info *map, struct flchip *chip, * We check "time_after" and "!chip_good" before checking * "chip_good" to avoid the failure due to scheduling. */ - if (time_after(jiffies, timeo) && !chip_good(map, adr, datum)) { + if (time_after(jiffies, timeo) && + !chip_good(map, adr, &datum)) { xip_enable(map, chip, adr); printk(KERN_WARNING "MTD %s(): software timeout\n", __func__); xip_disable(map, chip, adr); @@ -1638,7 +1649,7 @@ static int __xipram do_write_oneword(struct map_info *map, struct flchip *chip, break; } - if (chip_good(map, adr, datum)) + if (chip_good(map, adr, &datum)) break; /* Latency issues. Drop the lock, wait a while and retry */ @@ -1882,13 +1893,13 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, } /* - * We check "time_after" and "!chip_good" before checking "chip_good" to avoid - * the failure due to scheduling. + * We check "time_after" and "!chip_good" before checking + * "chip_good" to avoid the failure due to scheduling. */ - if (time_after(jiffies, timeo) && !chip_good(map, adr, datum)) + if (time_after(jiffies, timeo) && !chip_good(map, adr, &datum)) break; - if (chip_good(map, adr, datum)) { + if (chip_good(map, adr, &datum)) { xip_enable(map, chip, adr); goto op_done; } @@ -2022,7 +2033,7 @@ static int cfi_amdstd_panic_wait(struct map_info *map, struct flchip *chip, * If the driver thinks the chip is idle, and no toggle bits * are changing, then the chip is actually idle for sure. */ - if (chip->state == FL_READY && chip_ready(map, adr)) + if (chip->state == FL_READY && chip_ready(map, adr, NULL)) return 0; /* @@ -2039,7 +2050,7 @@ static int cfi_amdstd_panic_wait(struct map_info *map, struct flchip *chip, /* wait for the chip to become ready */ for (i = 0; i < jiffies_to_usecs(timeo); i++) { - if (chip_ready(map, adr)) + if (chip_ready(map, adr, NULL)) return 0; udelay(1); @@ -2103,13 +2114,13 @@ static int do_panic_write_oneword(struct map_info *map, struct flchip *chip, map_write(map, datum, adr); for (i = 0; i < jiffies_to_usecs(uWriteTimeout); i++) { - if (chip_ready(map, adr)) + if (chip_ready(map, adr, NULL)) break; udelay(1); } - if (!chip_good(map, adr, datum)) { + if (!chip_ready(map, adr, &datum)) { /* reset on all failures. */ map_write(map, CMD(0xF0), chip->start); /* FIXME - should have reset delay before continuing */ @@ -2250,6 +2261,7 @@ static int __xipram do_erase_chip(struct map_info *map, struct flchip *chip) DECLARE_WAITQUEUE(wait, current); int ret = 0; int retry_cnt = 0; + map_word datum = map_word_ff(map); adr = cfi->addr_unlock1; @@ -2304,7 +2316,7 @@ static int __xipram do_erase_chip(struct map_info *map, struct flchip *chip) chip->erase_suspended = 0; } - if (chip_good(map, adr, map_word_ff(map))) + if (chip_ready(map, adr, &datum)) break; if (time_after(jiffies, timeo)) { @@ -2346,6 +2358,7 @@ static int __xipram do_erase_oneblock(struct map_info *map, struct flchip *chip, DECLARE_WAITQUEUE(wait, current); int ret = 0; int retry_cnt = 0; + map_word datum = map_word_ff(map); adr += chip->start; @@ -2400,7 +2413,7 @@ static int __xipram do_erase_oneblock(struct map_info *map, struct flchip *chip, chip->erase_suspended = 0; } - if (chip_good(map, adr, map_word_ff(map))) + if (chip_ready(map, adr, &datum)) break; if (time_after(jiffies, timeo)) { @@ -2593,7 +2606,7 @@ static int __maybe_unused do_ppb_xxlock(struct map_info *map, */ timeo = jiffies + msecs_to_jiffies(2000); /* 2s max (un)locking */ for (;;) { - if (chip_ready(map, adr)) + if (chip_ready(map, adr, NULL)) break; if (time_after(jiffies, timeo)) { diff --git a/drivers/mtd/ubi/vmt.c b/drivers/mtd/ubi/vmt.c index d0c6b66b7450..9f6ffd340a3e 100644 --- a/drivers/mtd/ubi/vmt.c +++ b/drivers/mtd/ubi/vmt.c @@ -322,7 +322,6 @@ int ubi_create_volume(struct ubi_device *ubi, struct ubi_mkvol_req *req) ubi->volumes[vol_id] = NULL; ubi->vol_count -= 1; spin_unlock(&ubi->volumes_lock); - ubi_eba_destroy_table(eba_tbl); out_acc: spin_lock(&ubi->volumes_lock); ubi->rsvd_pebs -= vol->reserved_pebs; diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 1df7aed5ae15..be064bcfd70a 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -2778,6 +2778,7 @@ static int mv88e6xxx_mdios_register(struct mv88e6xxx_chip *chip, */ child = of_get_child_by_name(np, "mdio"); err = mv88e6xxx_mdio_register(chip, child, false); + of_node_put(child); if (err) return err; diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c index 1b4dfd357383..1b9fb78ef824 100644 --- a/drivers/net/ethernet/altera/altera_tse_main.c +++ b/drivers/net/ethernet/altera/altera_tse_main.c @@ -174,7 +174,8 @@ static int altera_tse_mdio_create(struct net_device *dev, unsigned int id) mdio = mdiobus_alloc(); if (mdio == NULL) { netdev_err(dev, "Error allocating MDIO bus\n"); - return -ENOMEM; + ret = -ENOMEM; + goto put_node; } mdio->name = ALTERA_TSE_RESOURCE_NAME; @@ -191,6 +192,7 @@ static int altera_tse_mdio_create(struct net_device *dev, unsigned int id) mdio->id); goto out_free_mdio; } + of_node_put(mdio_node); if (netif_msg_drv(priv)) netdev_info(dev, "MDIO bus %s: created\n", mdio->id); @@ -200,6 +202,8 @@ static int altera_tse_mdio_create(struct net_device *dev, unsigned int id) out_free_mdio: mdiobus_free(mdio); mdio = NULL; +put_node: + of_node_put(mdio_node); return ret; } diff --git a/drivers/net/ethernet/broadcom/Makefile b/drivers/net/ethernet/broadcom/Makefile index 7046ad6d3d0e..ac50da49ca77 100644 --- a/drivers/net/ethernet/broadcom/Makefile +++ b/drivers/net/ethernet/broadcom/Makefile @@ -16,3 +16,8 @@ obj-$(CONFIG_BGMAC_BCMA) += bgmac-bcma.o bgmac-bcma-mdio.o obj-$(CONFIG_BGMAC_PLATFORM) += bgmac-platform.o obj-$(CONFIG_SYSTEMPORT) += bcmsysport.o obj-$(CONFIG_BNXT) += bnxt/ + +# FIXME: temporarily silence -Warray-bounds on non W=1+ builds +ifndef KBUILD_EXTRA_WARN +CFLAGS_tg3.o += -Wno-array-bounds +endif diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index 964407deca35..23c019d1278c 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -1869,6 +1869,11 @@ static int ftgmac100_probe(struct platform_device *pdev) /* AST2400 doesn't have working HW checksum generation */ if (np && (of_device_is_compatible(np, "aspeed,ast2400-mac"))) netdev->hw_features &= ~NETIF_F_HW_CSUM; + + /* AST2600 tx checksum with NCSI is broken */ + if (priv->use_ncsi && of_device_is_compatible(np, "aspeed,ast2600-mac")) + netdev->hw_features &= ~NETIF_F_HW_CSUM; + if (np && of_get_property(np, "no-hw-checksum", NULL)) netdev->hw_features &= ~(NETIF_F_HW_CSUM | NETIF_F_RXCSUM); netdev->features |= netdev->hw_features; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c index 8aaf856771d7..6055a4917ff6 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c @@ -1148,9 +1148,9 @@ static int ixgbe_update_vf_xcast_mode(struct ixgbe_adapter *adapter, switch (xcast_mode) { case IXGBEVF_XCAST_MODE_NONE: - disable = IXGBE_VMOLR_BAM | IXGBE_VMOLR_ROMPE | + disable = IXGBE_VMOLR_ROMPE | IXGBE_VMOLR_MPE | IXGBE_VMOLR_UPE | IXGBE_VMOLR_VPE; - enable = 0; + enable = IXGBE_VMOLR_BAM; break; case IXGBEVF_XCAST_MODE_MULTI: disable = IXGBE_VMOLR_MPE | IXGBE_VMOLR_UPE | IXGBE_VMOLR_VPE; @@ -1172,9 +1172,9 @@ static int ixgbe_update_vf_xcast_mode(struct ixgbe_adapter *adapter, return -EPERM; } - disable = 0; + disable = IXGBE_VMOLR_VPE; enable = IXGBE_VMOLR_BAM | IXGBE_VMOLR_ROMPE | - IXGBE_VMOLR_MPE | IXGBE_VMOLR_UPE | IXGBE_VMOLR_VPE; + IXGBE_VMOLR_MPE | IXGBE_VMOLR_UPE; break; default: return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 59f3dce3ab1d..f2eaf8c13cc2 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -1575,6 +1575,9 @@ static int mtk_hwlro_get_fdir_entry(struct net_device *dev, struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; + if (fsp->location >= ARRAY_SIZE(mac->hwlro_ip)) + return -EINVAL; + /* only tcp dst ipv4 is meaningful, others are meaningless */ fsp->flow_type = TCP_V4_FLOW; fsp->h_u.tcp_ip4_spec.ip4dst = ntohl(mac->hwlro_ip[fsp->location]); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index f652cfd8127b..1d33fae529b6 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -2083,7 +2083,7 @@ static int mlx4_en_get_module_eeprom(struct net_device *dev, en_err(priv, "mlx4_get_module_info i(%d) offset(%d) bytes_to_read(%d) - FAILED (0x%x)\n", i, offset, ee->len - i, ret); - return 0; + return ret; } i += ret; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c index 2266c09b741a..a22e932a00b0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c @@ -637,6 +637,9 @@ static void mlx5_fw_tracer_handle_traces(struct work_struct *work) if (!tracer->owner) return; + if (unlikely(!tracer->str_db.loaded)) + goto arm; + block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE; start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; @@ -694,6 +697,7 @@ static void mlx5_fw_tracer_handle_traces(struct work_struct *work) &tmp_trace_block[TRACES_PER_BLOCK - 1]); } +arm: mlx5_fw_tracer_arm(dev); } @@ -935,8 +939,7 @@ void mlx5_fw_tracer_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe) queue_work(tracer->work_queue, &tracer->ownership_change_work); break; case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE: - if (likely(tracer->str_db.loaded)) - queue_work(tracer->work_queue, &tracer->handle_traces_work); + queue_work(tracer->work_queue, &tracer->handle_traces_work); break; default: mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n", diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 75872aef44d0..6ecb92f55e97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -4327,6 +4327,11 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) unlock: mutex_unlock(&priv->state_lock); + + /* Need to fix some features. */ + if (!err) + netdev_update_features(netdev); + return err; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c index 21296fa7f7fb..bf51ed94952c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c @@ -227,8 +227,6 @@ static int mlxsw_sp_dcbnl_ieee_setets(struct net_device *dev, static int mlxsw_sp_dcbnl_app_validate(struct net_device *dev, struct dcb_app *app) { - int prio; - if (app->priority >= IEEE_8021QAZ_MAX_TCS) { netdev_err(dev, "APP entry with priority value %u is invalid\n", app->priority); @@ -242,17 +240,6 @@ static int mlxsw_sp_dcbnl_app_validate(struct net_device *dev, app->protocol); return -EINVAL; } - - /* Warn about any DSCP APP entries with the same PID. */ - prio = fls(dcb_ieee_getapp_mask(dev, app)); - if (prio--) { - if (prio < app->priority) - netdev_warn(dev, "Choosing priority %d for DSCP %d in favor of previously-active value of %d\n", - app->priority, app->protocol, prio); - else if (prio > app->priority) - netdev_warn(dev, "Ignoring new priority %d for DSCP %d in favor of current value of %d\n", - app->priority, app->protocol, prio); - } break; case IEEE_8021QAZ_APP_SEL_ETHERTYPE: diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index 2e75d0af4a58..a51661cfd27f 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -292,8 +292,6 @@ nfp_net_get_link_ksettings(struct net_device *netdev, /* Init to unknowns */ ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); - ethtool_link_ksettings_add_link_mode(cmd, supported, Pause); - ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause); cmd->base.port = PORT_OTHER; cmd->base.speed = SPEED_UNKNOWN; cmd->base.duplex = DUPLEX_UNKNOWN; @@ -301,6 +299,8 @@ nfp_net_get_link_ksettings(struct net_device *netdev, port = nfp_port_from_netdev(netdev); eth_port = nfp_port_get_eth_port(port); if (eth_port) { + ethtool_link_ksettings_add_link_mode(cmd, supported, Pause); + ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause); cmd->base.autoneg = eth_port->aneg != NFP_ANEG_DISABLED ? AUTONEG_ENABLE : AUTONEG_DISABLE; nfp_net_set_fec_link_mode(eth_port, cmd); diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index eaa890a6a5d2..efdac68da7f4 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -746,7 +746,6 @@ int __init mdio_bus_init(void) return ret; } -EXPORT_SYMBOL_GPL(mdio_bus_init); #if IS_ENABLED(CONFIG_PHYLIB) void mdio_bus_exit(void) diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 013590330059..1d00a563892a 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -285,7 +285,7 @@ static int kszphy_config_reset(struct phy_device *phydev) } } - if (priv->led_mode >= 0) + if (priv->type && priv->led_mode >= 0) kszphy_setup_led(phydev, priv->type->led_mode_reg, priv->led_mode); return 0; @@ -301,10 +301,10 @@ static int kszphy_config_init(struct phy_device *phydev) type = priv->type; - if (type->has_broadcast_disable) + if (type && type->has_broadcast_disable) kszphy_broadcast_disable(phydev); - if (type->has_nand_tree_disable) + if (type && type->has_nand_tree_disable) kszphy_nand_tree_disable(phydev); return kszphy_config_reset(phydev); @@ -775,7 +775,7 @@ static int kszphy_probe(struct phy_device *phydev) priv->type = type; - if (type->led_mode_reg) { + if (type && type->led_mode_reg) { ret = of_property_read_u32(np, "micrel,led-mode", &priv->led_mode); if (ret) @@ -796,7 +796,8 @@ static int kszphy_probe(struct phy_device *phydev) unsigned long rate = clk_get_rate(clk); bool rmii_ref_clk_sel_25_mhz; - priv->rmii_ref_clk_sel = type->has_rmii_ref_clk_sel; + if (type) + priv->rmii_ref_clk_sel = type->has_rmii_ref_clk_sel; rmii_ref_clk_sel_25_mhz = of_property_read_bool(np, "micrel,rmii-reference-clock-select-25-mhz"); diff --git a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c index 4d45d5a8ad2e..db583a6aeb0c 100644 --- a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c +++ b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c @@ -5615,7 +5615,7 @@ unsigned int ar9003_get_paprd_scale_factor(struct ath_hw *ah, static u8 ar9003_get_eepmisc(struct ath_hw *ah) { - return ah->eeprom.map4k.baseEepHeader.eepMisc; + return ah->eeprom.ar9300_eep.baseEepHeader.opCapFlags.eepMisc; } const struct eeprom_ops eep_ar9300_ops = { diff --git a/drivers/net/wireless/ath/ath9k/ar9003_phy.h b/drivers/net/wireless/ath/ath9k/ar9003_phy.h index a171dbb29fbb..ad949eb02f3d 100644 --- a/drivers/net/wireless/ath/ath9k/ar9003_phy.h +++ b/drivers/net/wireless/ath/ath9k/ar9003_phy.h @@ -720,7 +720,7 @@ #define AR_CH0_TOP2 (AR_SREV_9300(ah) ? 0x1628c : \ (AR_SREV_9462(ah) ? 0x16290 : 0x16284)) #define AR_CH0_TOP2_XPABIASLVL (AR_SREV_9561(ah) ? 0x1e00 : 0xf000) -#define AR_CH0_TOP2_XPABIASLVL_S 12 +#define AR_CH0_TOP2_XPABIASLVL_S (AR_SREV_9561(ah) ? 9 : 12) #define AR_CH0_XTAL (AR_SREV_9300(ah) ? 0x16294 : \ ((AR_SREV_9462(ah) || AR_SREV_9565(ah)) ? 0x16298 : \ diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c index d567fbe79cff..3cd3f3ca1000 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c @@ -1005,6 +1005,14 @@ static bool ath9k_rx_prepare(struct ath9k_htc_priv *priv, goto rx_next; } + if (rxstatus->rs_keyix >= ATH_KEYMAX && + rxstatus->rs_keyix != ATH9K_RXKEYIX_INVALID) { + ath_dbg(common, ANY, + "Invalid keyix, dropping (keyix: %d)\n", + rxstatus->rs_keyix); + goto rx_next; + } + /* Get the RX status information */ memset(rx_status, 0, sizeof(struct ieee80211_rx_status)); diff --git a/drivers/net/wireless/ath/carl9170/tx.c b/drivers/net/wireless/ath/carl9170/tx.c index 0cb5b58925dc..40369cb59eb5 100644 --- a/drivers/net/wireless/ath/carl9170/tx.c +++ b/drivers/net/wireless/ath/carl9170/tx.c @@ -1554,6 +1554,9 @@ static struct carl9170_vif_info *carl9170_pick_beaconing_vif(struct ar9170 *ar) goto out; } } while (ar->beacon_enabled && i--); + + /* no entry found in list */ + return NULL; } out: diff --git a/drivers/net/wireless/broadcom/b43/phy_n.c b/drivers/net/wireless/broadcom/b43/phy_n.c index 88446258e775..3508a7822619 100644 --- a/drivers/net/wireless/broadcom/b43/phy_n.c +++ b/drivers/net/wireless/broadcom/b43/phy_n.c @@ -594,7 +594,7 @@ static void b43_nphy_adjust_lna_gain_table(struct b43_wldev *dev) u16 data[4]; s16 gain[2]; u16 minmax[2]; - static const u16 lna_gain[4] = { -2, 10, 19, 25 }; + static const s16 lna_gain[4] = { -2, 10, 19, 25 }; if (nphy->hang_avoid) b43_nphy_stay_in_carrier_search(dev, 1); diff --git a/drivers/net/wireless/broadcom/b43legacy/phy.c b/drivers/net/wireless/broadcom/b43legacy/phy.c index 995c7d0c212a..11ee5ee48976 100644 --- a/drivers/net/wireless/broadcom/b43legacy/phy.c +++ b/drivers/net/wireless/broadcom/b43legacy/phy.c @@ -1148,7 +1148,7 @@ void b43legacy_phy_lo_b_measure(struct b43legacy_wldev *dev) struct b43legacy_phy *phy = &dev->phy; u16 regstack[12] = { 0 }; u16 mls; - u16 fval; + s16 fval; int i; int j; diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_tx.c b/drivers/net/wireless/intel/ipw2x00/libipw_tx.c index 84205aa508df..daa4f9eb08ff 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw_tx.c +++ b/drivers/net/wireless/intel/ipw2x00/libipw_tx.c @@ -397,7 +397,7 @@ netdev_tx_t libipw_xmit(struct sk_buff *skb, struct net_device *dev) /* Each fragment may need to have room for encryption * pre/postfix */ - if (host_encrypt) + if (host_encrypt && crypt && crypt->ops) bytes_per_frag -= crypt->ops->extra_mpdu_prefix_len + crypt->ops->extra_mpdu_postfix_len; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/power.c b/drivers/net/wireless/intel/iwlwifi/mvm/power.c index c11fe2621d51..cd19831ace57 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/power.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/power.c @@ -611,6 +611,9 @@ static void iwl_mvm_power_get_vifs_iterator(void *_data, u8 *mac, struct iwl_power_vifs *power_iterator = _data; bool active = mvmvif->phy_ctxt && mvmvif->phy_ctxt->id < NUM_PHY_CTX; + if (!mvmvif->uploaded) + return; + switch (ieee80211_vif_type_p2p(vif)) { case NL80211_IFTYPE_P2P_DEVICE: break; diff --git a/drivers/net/wireless/marvell/mwifiex/11h.c b/drivers/net/wireless/marvell/mwifiex/11h.c index 238accfe4f41..c4176e357b22 100644 --- a/drivers/net/wireless/marvell/mwifiex/11h.c +++ b/drivers/net/wireless/marvell/mwifiex/11h.c @@ -303,5 +303,7 @@ void mwifiex_dfs_chan_sw_work_queue(struct work_struct *work) mwifiex_dbg(priv->adapter, MSG, "indicating channel switch completion to kernel\n"); + mutex_lock(&priv->wdev.mtx); cfg80211_ch_switch_notify(priv->netdev, &priv->dfs_chandef); + mutex_unlock(&priv->wdev.mtx); } diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c index 225c1c8851cc..77bf8a601ca4 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c @@ -460,8 +460,10 @@ static void rtl8180_tx(struct ieee80211_hw *dev, struct rtl8180_priv *priv = dev->priv; struct rtl8180_tx_ring *ring; struct rtl8180_tx_desc *entry; + unsigned int prio = 0; unsigned long flags; - unsigned int idx, prio, hw_prio; + unsigned int idx, hw_prio; + dma_addr_t mapping; u32 tx_flags; u8 rc_flags; @@ -470,7 +472,9 @@ static void rtl8180_tx(struct ieee80211_hw *dev, /* do arithmetic and then convert to le16 */ u16 frame_duration = 0; - prio = skb_get_queue_mapping(skb); + /* rtl8180/rtl8185 only has one useable tx queue */ + if (dev->queues > IEEE80211_AC_BK) + prio = skb_get_queue_mapping(skb); ring = &priv->tx_ring[prio]; mapping = pci_map_single(priv->pdev, skb->data, diff --git a/drivers/net/wireless/realtek/rtlwifi/usb.c b/drivers/net/wireless/realtek/rtlwifi/usb.c index 3d6c0d8c71d7..395671383ca9 100644 --- a/drivers/net/wireless/realtek/rtlwifi/usb.c +++ b/drivers/net/wireless/realtek/rtlwifi/usb.c @@ -1042,7 +1042,7 @@ int rtl_usb_probe(struct usb_interface *intf, hw = ieee80211_alloc_hw(sizeof(struct rtl_priv) + sizeof(struct rtl_usb_priv), &rtl_ops); if (!hw) { - WARN_ONCE(true, "rtl_usb: ieee80211 alloc failed\n"); + pr_warn("rtl_usb: ieee80211 alloc failed\n"); return -ENOMEM; } rtlpriv = hw->priv; diff --git a/drivers/nfc/st21nfca/se.c b/drivers/nfc/st21nfca/se.c index 5a6d4f041dbb..e111d062d27c 100644 --- a/drivers/nfc/st21nfca/se.c +++ b/drivers/nfc/st21nfca/se.c @@ -252,7 +252,7 @@ int st21nfca_hci_se_io(struct nfc_hci_dev *hdev, u32 se_idx, } EXPORT_SYMBOL(st21nfca_hci_se_io); -static void st21nfca_se_wt_timeout(struct timer_list *t) +static void st21nfca_se_wt_work(struct work_struct *work) { /* * No answer from the secure element @@ -265,8 +265,9 @@ static void st21nfca_se_wt_timeout(struct timer_list *t) */ /* hardware reset managed through VCC_UICC_OUT power supply */ u8 param = 0x01; - struct st21nfca_hci_info *info = from_timer(info, t, - se_info.bwi_timer); + struct st21nfca_hci_info *info = container_of(work, + struct st21nfca_hci_info, + se_info.timeout_work); pr_debug("\n"); @@ -284,6 +285,13 @@ static void st21nfca_se_wt_timeout(struct timer_list *t) info->se_info.cb(info->se_info.cb_context, NULL, 0, -ETIME); } +static void st21nfca_se_wt_timeout(struct timer_list *t) +{ + struct st21nfca_hci_info *info = from_timer(info, t, se_info.bwi_timer); + + schedule_work(&info->se_info.timeout_work); +} + static void st21nfca_se_activation_timeout(struct timer_list *t) { struct st21nfca_hci_info *info = from_timer(info, t, @@ -341,8 +349,10 @@ int st21nfca_connectivity_event_received(struct nfc_hci_dev *hdev, u8 host, transaction->aid_len = skb->data[1]; /* Checking if the length of the AID is valid */ - if (transaction->aid_len > sizeof(transaction->aid)) + if (transaction->aid_len > sizeof(transaction->aid)) { + devm_kfree(dev, transaction); return -EINVAL; + } memcpy(transaction->aid, &skb->data[2], transaction->aid_len); @@ -357,8 +367,11 @@ int st21nfca_connectivity_event_received(struct nfc_hci_dev *hdev, u8 host, } /* Total size is allocated (skb->len - 2) minus fixed array members */ - if (transaction->params_len > ((skb->len - 2) - sizeof(struct nfc_evt_transaction))) + if (transaction->params_len > ((skb->len - 2) - + sizeof(struct nfc_evt_transaction))) { + devm_kfree(dev, transaction); return -EINVAL; + } memcpy(transaction->params, skb->data + transaction->aid_len + 4, transaction->params_len); @@ -385,6 +398,7 @@ int st21nfca_apdu_reader_event_received(struct nfc_hci_dev *hdev, switch (event) { case ST21NFCA_EVT_TRANSMIT_DATA: del_timer_sync(&info->se_info.bwi_timer); + cancel_work_sync(&info->se_info.timeout_work); info->se_info.bwi_active = false; r = nfc_hci_send_event(hdev, ST21NFCA_DEVICE_MGNT_GATE, ST21NFCA_EVT_SE_END_OF_APDU_TRANSFER, NULL, 0); @@ -414,6 +428,7 @@ void st21nfca_se_init(struct nfc_hci_dev *hdev) struct st21nfca_hci_info *info = nfc_hci_get_clientdata(hdev); init_completion(&info->se_info.req_completion); + INIT_WORK(&info->se_info.timeout_work, st21nfca_se_wt_work); /* initialize timers */ timer_setup(&info->se_info.bwi_timer, st21nfca_se_wt_timeout, 0); info->se_info.bwi_active = false; @@ -441,6 +456,7 @@ void st21nfca_se_deinit(struct nfc_hci_dev *hdev) if (info->se_info.se_active) del_timer_sync(&info->se_info.se_active_timer); + cancel_work_sync(&info->se_info.timeout_work); info->se_info.bwi_active = false; info->se_info.se_active = false; } diff --git a/drivers/nfc/st21nfca/st21nfca.h b/drivers/nfc/st21nfca/st21nfca.h index 94ffb0501e87..7e2923ac9263 100644 --- a/drivers/nfc/st21nfca/st21nfca.h +++ b/drivers/nfc/st21nfca/st21nfca.h @@ -152,6 +152,7 @@ struct st21nfca_se_info { se_io_cb_t cb; void *cb_context; + struct work_struct timeout_work; }; struct st21nfca_hci_info { diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d7cf3202cdd3..b06d2b6bd3fe 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1522,6 +1522,7 @@ static int nvme_alloc_admin_tags(struct nvme_dev *dev) dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset); if (IS_ERR(dev->ctrl.admin_q)) { blk_mq_free_tag_set(&dev->admin_tagset); + dev->ctrl.admin_q = NULL; return -ENOMEM; } if (!blk_get_queue(dev->ctrl.admin_q)) { diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index a77bfeac867d..fef5b6c2fae2 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -170,9 +170,7 @@ static int overlay_notify(struct overlay_changeset *ovcs, ret = blocking_notifier_call_chain(&overlay_notify_chain, action, &nd); - if (ret == NOTIFY_OK || ret == NOTIFY_STOP) - return 0; - if (ret) { + if (notifier_to_errno(ret)) { ret = notifier_to_errno(ret); pr_err("overlay changeset %s notifier error %d, target: %pOF\n", of_overlay_action_name[action], ret, nd.target); diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 33e510393976..133fad284c9f 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -1331,22 +1331,21 @@ static int qcom_pcie_probe(struct platform_device *pdev) } ret = phy_init(pcie->phy); - if (ret) { - pm_runtime_disable(&pdev->dev); + if (ret) goto err_pm_runtime_put; - } platform_set_drvdata(pdev, pcie); ret = dw_pcie_host_init(pp); if (ret) { dev_err(dev, "cannot initialize host\n"); - pm_runtime_disable(&pdev->dev); - goto err_pm_runtime_put; + goto err_phy_exit; } return 0; +err_phy_exit: + phy_exit(pcie->phy); err_pm_runtime_put: pm_runtime_put(dev); pm_runtime_disable(dev); diff --git a/drivers/pci/controller/pcie-cadence-ep.c b/drivers/pci/controller/pcie-cadence-ep.c index c3a088910f48..f6da8d562b8a 100644 --- a/drivers/pci/controller/pcie-cadence-ep.c +++ b/drivers/pci/controller/pcie-cadence-ep.c @@ -178,8 +178,7 @@ static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, phys_addr_t addr, struct cdns_pcie *pcie = &ep->pcie; u32 r; - r = find_first_zero_bit(&ep->ob_region_map, - sizeof(ep->ob_region_map) * BITS_PER_LONG); + r = find_first_zero_bit(&ep->ob_region_map, BITS_PER_LONG); if (r >= ep->max_regions - 1) { dev_err(&epc->dev, "no free outbound region\n"); return -EINVAL; diff --git a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c index caf34661d38d..06dd2ab73b6e 100644 --- a/drivers/pci/controller/pcie-rockchip-ep.c +++ b/drivers/pci/controller/pcie-rockchip-ep.c @@ -263,8 +263,7 @@ static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, struct rockchip_pcie *pcie = &ep->rockchip; u32 r; - r = find_first_zero_bit(&ep->ob_region_map, - sizeof(ep->ob_region_map) * BITS_PER_LONG); + r = find_first_zero_bit(&ep->ob_region_map, BITS_PER_LONG); /* * Region 0 is reserved for configuration space and shouldn't * be used elsewhere per TRM, so leave it out. diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index d22637de1607..37f194ab5f4e 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2517,6 +2517,8 @@ static const struct dmi_system_id bridge_d3_blacklist[] = { DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."), DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"), }, + }, + { /* * Downstream device is not accessible after putting a root port * into D3cold and back into D0 on Elo i2. @@ -4668,18 +4670,18 @@ static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe) static void pci_dev_lock(struct pci_dev *dev) { - pci_cfg_access_lock(dev); /* block PM suspend, driver probe, etc. */ device_lock(&dev->dev); + pci_cfg_access_lock(dev); } /* Return 1 on successful lock, 0 on contention */ static int pci_dev_trylock(struct pci_dev *dev) { - if (pci_cfg_access_trylock(dev)) { - if (device_trylock(&dev->dev)) + if (device_trylock(&dev->dev)) { + if (pci_cfg_access_trylock(dev)) return 1; - pci_cfg_access_unlock(dev); + device_unlock(&dev->dev); } return 0; @@ -4687,8 +4689,8 @@ static int pci_dev_trylock(struct pci_dev *dev) static void pci_dev_unlock(struct pci_dev *dev) { - device_unlock(&dev->dev); pci_cfg_access_unlock(dev); + device_unlock(&dev->dev); } static void pci_dev_save_and_disable(struct pci_dev *dev) diff --git a/drivers/pcmcia/Kconfig b/drivers/pcmcia/Kconfig index cbbe4a285b48..a8fdd6df6a12 100644 --- a/drivers/pcmcia/Kconfig +++ b/drivers/pcmcia/Kconfig @@ -146,7 +146,7 @@ config TCIC config PCMCIA_ALCHEMY_DEVBOARD tristate "Alchemy Db/Pb1xxx PCMCIA socket services" - depends on MIPS_ALCHEMY && PCMCIA + depends on MIPS_DB1XXX && PCMCIA help Enable this driver of you want PCMCIA support on your Alchemy Db1000, Db/Pb1100, Db/Pb1500, Db/Pb1550, Db/Pb1200, DB1300 diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index 68107611c70a..c64262f4ff11 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -1426,6 +1426,11 @@ static const struct phy_ops qcom_qmp_phy_gen_ops = { .owner = THIS_MODULE, }; +static void qcom_qmp_reset_control_put(void *data) +{ + reset_control_put(data); +} + static int qcom_qmp_phy_create(struct device *dev, struct device_node *np, int id) { @@ -1468,7 +1473,7 @@ int qcom_qmp_phy_create(struct device *dev, struct device_node *np, int id) * all phys that don't need this. */ snprintf(prop_name, sizeof(prop_name), "pipe%d", id); - qphy->pipe_clk = of_clk_get_by_name(np, prop_name); + qphy->pipe_clk = devm_get_clk_from_child(dev, np, prop_name); if (IS_ERR(qphy->pipe_clk)) { if (qmp->cfg->type == PHY_TYPE_PCIE || qmp->cfg->type == PHY_TYPE_USB3) { @@ -1490,6 +1495,10 @@ int qcom_qmp_phy_create(struct device *dev, struct device_node *np, int id) dev_err(dev, "failed to get lane%d reset\n", id); return PTR_ERR(qphy->lane_rst); } + ret = devm_add_action_or_reset(dev, qcom_qmp_reset_control_put, + qphy->lane_rst); + if (ret) + return ret; } generic_phy = devm_phy_create(dev, np, &qcom_qmp_phy_gen_ops); diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index e69b84d9538a..0c0f834eab82 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -773,7 +773,7 @@ static int armada_37xx_irqchip_register(struct platform_device *pdev, for (i = 0; i < nr_irq_parent; i++) { int irq = irq_of_parse_and_map(np, i); - if (irq < 0) + if (!irq) continue; gpiochip_set_chained_irqchip(gc, irqchip, irq, diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_qmi_service.c b/drivers/platform/msm/ipa/ipa_v3/ipa_qmi_service.c index 40254a4b89fb..b04f1723697c 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_qmi_service.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_qmi_service.c @@ -1601,6 +1601,7 @@ static void ipa3_q6_clnt_svc_arrive(struct work_struct *work) return; } IPAWANERR("Exit from service arrive fun\n"); + return; } if (rc != 0) { diff --git a/drivers/platform/msm/usb_bam.c b/drivers/platform/msm/usb_bam.c index a304574418d4..05a8cf596e17 100644 --- a/drivers/platform/msm/usb_bam.c +++ b/drivers/platform/msm/usb_bam.c @@ -866,6 +866,7 @@ static int connect_pipe(enum usb_ctrl cur_bam, u8 idx, u32 *usb_pipe_idx, sps_disconnect(*pipe); free_sps_endpoint: sps_free_endpoint(*pipe); + *pipe = NULL; return ret; } diff --git a/drivers/pwm/pwm-lp3943.c b/drivers/pwm/pwm-lp3943.c index 5055ba2c6c94..a5f4c39eeb21 100644 --- a/drivers/pwm/pwm-lp3943.c +++ b/drivers/pwm/pwm-lp3943.c @@ -128,6 +128,7 @@ static int lp3943_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (err) return err; + duty_ns = min(duty_ns, period_ns); val = (u8)(duty_ns * LP3943_MAX_DUTY / period_ns); return lp3943_write_byte(lp3943, reg_duty, val); diff --git a/drivers/regulator/pfuze100-regulator.c b/drivers/regulator/pfuze100-regulator.c index 4b8306594c3f..8b1940110561 100644 --- a/drivers/regulator/pfuze100-regulator.c +++ b/drivers/regulator/pfuze100-regulator.c @@ -513,6 +513,7 @@ static int pfuze_parse_regulators_dt(struct pfuze_chip *chip) parent = of_get_child_by_name(np, "regulators"); if (!parent) { dev_err(dev, "regulators node not found\n"); + of_node_put(np); return -EINVAL; } @@ -542,6 +543,7 @@ static int pfuze_parse_regulators_dt(struct pfuze_chip *chip) } of_node_put(parent); + of_node_put(np); if (ret < 0) { dev_err(dev, "Error parsing regulator init data: %d\n", ret); diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c index c99e4c23aad8..1bdb3fa8a2e0 100644 --- a/drivers/rpmsg/qcom_smd.c +++ b/drivers/rpmsg/qcom_smd.c @@ -1582,9 +1582,9 @@ static int qcom_smd_parse_edge(struct device *dev, edge->name = node->name; irq = irq_of_parse_and_map(node, 0); - if (irq < 0) { + if (!irq) { dev_err(dev, "required smd interrupt missing\n"); - ret = irq; + ret = -EINVAL; goto put_node; } diff --git a/drivers/rtc/rtc-mt6397.c b/drivers/rtc/rtc-mt6397.c index 964ed91416e1..671b6d275da3 100644 --- a/drivers/rtc/rtc-mt6397.c +++ b/drivers/rtc/rtc-mt6397.c @@ -339,6 +339,8 @@ static int mtk_rtc_probe(struct platform_device *pdev) return -ENOMEM; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; rtc->addr_base = res->start; rtc->irq = platform_get_irq(pdev, 0); diff --git a/drivers/scsi/dc395x.c b/drivers/scsi/dc395x.c index 16b9dc2fff6b..8b5a07503d5f 100644 --- a/drivers/scsi/dc395x.c +++ b/drivers/scsi/dc395x.c @@ -3771,10 +3771,19 @@ static struct DeviceCtlBlk *device_alloc(struct AdapterCtlBlk *acb, #endif if (dcb->target_lun != 0) { /* Copy settings */ - struct DeviceCtlBlk *p; - list_for_each_entry(p, &acb->dcb_list, list) - if (p->target_id == dcb->target_id) + struct DeviceCtlBlk *p = NULL, *iter; + + list_for_each_entry(iter, &acb->dcb_list, list) + if (iter->target_id == dcb->target_id) { + p = iter; break; + } + + if (!p) { + kfree(dcb); + return NULL; + } + dprintkdbg(DBG_1, "device_alloc: <%02i-%i> copy from <%02i-%i>\n", dcb->target_id, dcb->target_lun, diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c index 658c0726581f..1e087a206f48 100644 --- a/drivers/scsi/fcoe/fcoe_ctlr.c +++ b/drivers/scsi/fcoe/fcoe_ctlr.c @@ -1978,7 +1978,7 @@ EXPORT_SYMBOL(fcoe_ctlr_recv_flogi); * * Returns: u64 fc world wide name */ -u64 fcoe_wwn_from_mac(unsigned char mac[MAX_ADDR_LEN], +u64 fcoe_wwn_from_mac(unsigned char mac[ETH_ALEN], unsigned int scheme, unsigned int port) { u64 wwn; diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c index a84878fbf45d..7352d46ebb09 100644 --- a/drivers/scsi/megaraid.c +++ b/drivers/scsi/megaraid.c @@ -4641,7 +4641,7 @@ static int __init megaraid_init(void) * major number allocation. */ major = register_chrdev(0, "megadev_legacy", &megadev_fops); - if (!major) { + if (major < 0) { printk(KERN_WARNING "megaraid: failed to register char device\n"); } diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c index eab82e8e9356..0f87afae78bb 100644 --- a/drivers/scsi/ufs/ufs-qcom.c +++ b/drivers/scsi/ufs/ufs-qcom.c @@ -1289,8 +1289,11 @@ static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable) writel_relaxed(temp, host->dev_ref_clk_ctrl_mmio); - /* ensure that ref_clk is enabled/disabled before we return */ - wmb(); + /* + * Make sure the write to ref_clk reaches the destination and + * not stored in a Write Buffer (WB). + */ + readl(host->dev_ref_clk_ctrl_mmio); /* * If we call hibern8 exit after this, we need to make sure that diff --git a/drivers/soc/qcom/scm.c b/drivers/soc/qcom/scm.c index ef3436701fa2..bff6cdc9431d 100644 --- a/drivers/soc/qcom/scm.c +++ b/drivers/soc/qcom/scm.c @@ -28,7 +28,6 @@ #define SCM_INTERRUPTED 1 #define SCM_V2_EBUSY -12 -static atomic_t scm_call_count = ATOMIC_INIT(0); static DEFINE_MUTEX(scm_lock); /* @@ -106,7 +105,6 @@ static int __scm_call_armv8_64(u64 x0, u64 x1, u64 x2, u64 x3, u64 x4, u64 x5, register u64 r5 asm("x5") = x5; register u64 r6 asm("x6") = 0; - atomic_inc(&scm_call_count); do { asm volatile( __asmeq("%0", R0_STR) @@ -135,8 +133,6 @@ static int __scm_call_armv8_64(u64 x0, u64 x1, u64 x2, u64 x3, u64 x4, u64 x5, "x14", "x15", "x16", "x17"); } while (r0 == SCM_INTERRUPTED); - atomic_dec(&scm_call_count); - if (ret1) *ret1 = r1; if (ret2) @@ -158,7 +154,6 @@ static int __scm_call_armv8_32(u32 w0, u32 w1, u32 w2, u32 w3, u32 w4, u32 w5, register u32 r5 asm("w5") = w5; register u32 r6 asm("w6") = 0; - atomic_inc(&scm_call_count); do { asm volatile( __asmeq("%0", R0_STR) @@ -188,8 +183,6 @@ static int __scm_call_armv8_32(u32 w0, u32 w1, u32 w2, u32 w3, u32 w4, u32 w5, } while (r0 == SCM_INTERRUPTED); - atomic_dec(&scm_call_count); - if (ret1) *ret1 = r1; if (ret2) @@ -213,7 +206,6 @@ static int __scm_call_armv8_32(u32 w0, u32 w1, u32 w2, u32 w3, u32 w4, u32 w5, register u32 r5 asm("r5") = w5; register u32 r6 asm("r6") = 0; - atomic_inc(&scm_call_count); do { asm volatile( __asmeq("%0", R0_STR) @@ -241,8 +233,6 @@ static int __scm_call_armv8_32(u32 w0, u32 w1, u32 w2, u32 w3, u32 w4, u32 w5, } while (r0 == SCM_INTERRUPTED); - atomic_dec(&scm_call_count); - if (ret1) *ret1 = r1; if (ret2) @@ -710,8 +700,3 @@ early_initcall(scm_mem_protection_init); #endif #endif - -bool under_scm_call(void) -{ - return atomic_read(&scm_call_count); -} diff --git a/drivers/soc/qcom/smp2p.c b/drivers/soc/qcom/smp2p.c index 3d43b478f208..a7eec2eae1b5 100644 --- a/drivers/soc/qcom/smp2p.c +++ b/drivers/soc/qcom/smp2p.c @@ -560,6 +560,7 @@ static int smp2p_parse_ipc(struct qcom_smp2p *smp2p) } smp2p->ipc_regmap = syscon_node_to_regmap(syscon); + of_node_put(syscon); if (IS_ERR(smp2p->ipc_regmap)) return PTR_ERR(smp2p->ipc_regmap); diff --git a/drivers/soc/qcom/smsm.c b/drivers/soc/qcom/smsm.c index 244caef30799..6d74f6ff4006 100644 --- a/drivers/soc/qcom/smsm.c +++ b/drivers/soc/qcom/smsm.c @@ -371,6 +371,7 @@ static int smsm_parse_ipc(struct qcom_smsm *smsm, unsigned host_id) return 0; host->ipc_regmap = syscon_node_to_regmap(syscon); + of_node_put(syscon); if (IS_ERR(host->ipc_regmap)) return PTR_ERR(host->ipc_regmap); diff --git a/drivers/soc/rockchip/grf.c b/drivers/soc/rockchip/grf.c index 3b81e1d75a97..3e7e999ee324 100644 --- a/drivers/soc/rockchip/grf.c +++ b/drivers/soc/rockchip/grf.c @@ -151,12 +151,14 @@ static int __init rockchip_grf_init(void) return -ENODEV; if (!match || !match->data) { pr_err("%s: missing grf data\n", __func__); + of_node_put(np); return -EINVAL; } grf_info = match->data; grf = syscon_node_to_regmap(np); + of_node_put(np); if (IS_ERR(grf)) { pr_err("%s: could not get grf syscon\n", __func__); return PTR_ERR(grf); diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c index 25a545c985d4..c63cceec6312 100644 --- a/drivers/spi/spi-img-spfi.c +++ b/drivers/spi/spi-img-spfi.c @@ -774,7 +774,7 @@ static int img_spfi_resume(struct device *dev) int ret; ret = pm_runtime_get_sync(dev); - if (ret) { + if (ret < 0) { pm_runtime_put_noidle(dev); return ret; } diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c index c70b1790a959..c248b1c38ca6 100644 --- a/drivers/spi/spi-ti-qspi.c +++ b/drivers/spi/spi-ti-qspi.c @@ -408,6 +408,7 @@ static int ti_qspi_dma_xfer(struct ti_qspi *qspi, dma_addr_t dma_dst, enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; struct dma_async_tx_descriptor *tx; int ret; + unsigned long time_left; tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len, flags); if (!tx) { @@ -427,9 +428,9 @@ static int ti_qspi_dma_xfer(struct ti_qspi *qspi, dma_addr_t dma_dst, } dma_async_issue_pending(chan); - ret = wait_for_completion_timeout(&qspi->transfer_complete, + time_left = wait_for_completion_timeout(&qspi->transfer_complete, msecs_to_jiffies(len)); - if (ret <= 0) { + if (time_left == 0) { dmaengine_terminate_sync(chan); dev_err(qspi->dev, "DMA wait_for_completion_timeout\n"); return -ETIMEDOUT; diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig index 2aa7ef0b0318..b95d92858c71 100644 --- a/drivers/staging/Kconfig +++ b/drivers/staging/Kconfig @@ -122,8 +122,6 @@ source "drivers/staging/gasket/Kconfig" source "drivers/staging/axis-fifo/Kconfig" -source "drivers/staging/erofs/Kconfig" - source "drivers/staging/qcacld-3.0/Kconfig" endif # STAGING diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index 4a63d6132b01..6c626c8afb31 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -51,5 +51,4 @@ obj-$(CONFIG_SOC_MT7621) += mt7621-eth/ obj-$(CONFIG_SOC_MT7621) += mt7621-dts/ obj-$(CONFIG_STAGING_GASKET_FRAMEWORK) += gasket/ obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/ -obj-$(CONFIG_EROFS_FS) += erofs/ obj-$(CONFIG_QCA_CLD_WLAN) += qcacld-3.0/ diff --git a/drivers/staging/erofs/Kconfig b/drivers/staging/erofs/Kconfig deleted file mode 100644 index 663b755bf2fb..000000000000 --- a/drivers/staging/erofs/Kconfig +++ /dev/null @@ -1,141 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -config EROFS_FS - tristate "EROFS filesystem support" - depends on BLOCK - help - EROFS(Enhanced Read-Only File System) is a lightweight - read-only file system with modern designs (eg. page-sized - blocks, inline xattrs/data, etc.) for scenarios which need - high-performance read-only requirements, eg. firmwares in - mobile phone or LIVECDs. - - It also provides VLE compression support, focusing on - random read improvements, keeping relatively lower - compression ratios, which is useful for high-performance - devices with limited memory and ROM space. - - If unsure, say N. - -config EROFS_FS_DEBUG - bool "EROFS debugging feature" - depends on EROFS_FS - help - Print EROFS debugging messages and enable more BUG_ONs - which check the filesystem consistency aggressively. - - For daily use, say N. - -config EROFS_FS_XATTR - bool "EROFS extended attributes" - depends on EROFS_FS - default y - help - Extended attributes are name:value pairs associated with inodes by - the kernel or by users (see the attr(5) manual page, or visit - for details). - - If unsure, say N. - -config EROFS_FS_POSIX_ACL - bool "EROFS Access Control Lists" - depends on EROFS_FS_XATTR - select FS_POSIX_ACL - default y - help - Posix Access Control Lists (ACLs) support permissions for users and - groups beyond the owner/group/world scheme. - - To learn more about Access Control Lists, visit the POSIX ACLs for - Linux website . - - If you don't know what Access Control Lists are, say N. - -config EROFS_FS_SECURITY - bool "EROFS Security Labels" - depends on EROFS_FS_XATTR - help - Security labels provide an access control facility to support Linux - Security Models (LSMs) accepted by AppArmor, SELinux, Smack and TOMOYO - Linux. This option enables an extended attribute handler for file - security labels in the erofs filesystem, so that it requires enabling - the extended attribute support in advance. - - If you are not using a security module, say N. - -config EROFS_FS_USE_VM_MAP_RAM - bool "EROFS VM_MAP_RAM Support" - depends on EROFS_FS - help - use vm_map_ram/vm_unmap_ram instead of vmap/vunmap. - - If you don't know what these are, say N. - -config EROFS_FAULT_INJECTION - bool "EROFS fault injection facility" - depends on EROFS_FS - help - Test EROFS to inject faults such as ENOMEM, EIO, and so on. - If unsure, say N. - -config EROFS_FS_ZIP - bool "EROFS Data Compresssion Support" - depends on EROFS_FS - help - Currently we support VLE Compression only. - Play at your own risk. - - If you don't want to use compression feature, say N. - -config EROFS_FS_CLUSTER_PAGE_LIMIT - int "EROFS Cluster Pages Hard Limit" - depends on EROFS_FS_ZIP - range 1 256 - default "1" - help - Indicates VLE compressed pages hard limit of a - compressed cluster. - - For example, if files of a image are compressed - into 8k-unit, the hard limit should not be less - than 2. Otherwise, the image cannot be mounted - correctly on this kernel. - -choice - prompt "EROFS VLE Data Decompression mode" - depends on EROFS_FS_ZIP - default EROFS_FS_ZIP_CACHE_BIPOLAR - help - EROFS supports three options for VLE decompression. - "In-place Decompression Only" consumes the minimum memory - with lowest random read. - - "Bipolar Cached Decompression" consumes the maximum memory - with highest random read. - - If unsure, select "Bipolar Cached Decompression" - -config EROFS_FS_ZIP_NO_CACHE - bool "In-place Decompression Only" - help - Read compressed data into page cache and do in-place - decompression directly. - -config EROFS_FS_ZIP_CACHE_UNIPOLAR - bool "Unipolar Cached Decompression" - help - For each request, it caches the last compressed page - for further reading. - It still decompresses in place for the rest compressed pages. - -config EROFS_FS_ZIP_CACHE_BIPOLAR - bool "Bipolar Cached Decompression" - help - For each request, it caches the both end compressed pages - for further reading. - It still decompresses in place for the rest compressed pages. - - Recommended for performance priority. - -endchoice - diff --git a/drivers/staging/erofs/Makefile b/drivers/staging/erofs/Makefile deleted file mode 100644 index 9a766eb7ed75..000000000000 --- a/drivers/staging/erofs/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -EROFS_VERSION = "1.0pre1" - -ccflags-y += -Wall -DEROFS_VERSION=\"$(EROFS_VERSION)\" - -obj-$(CONFIG_EROFS_FS) += erofs.o -# staging requirement: to be self-contained in its own directory -ccflags-y += -I$(src)/include -erofs-objs := super.o inode.o data.o namei.o dir.o utils.o -erofs-$(CONFIG_EROFS_FS_XATTR) += xattr.o -erofs-$(CONFIG_EROFS_FS_ZIP) += unzip_vle.o unzip_lz4.o unzip_vle_lz4.o - diff --git a/drivers/staging/erofs/erofs_fs.h b/drivers/staging/erofs/erofs_fs.h deleted file mode 100644 index d8838ce66941..000000000000 --- a/drivers/staging/erofs/erofs_fs.h +++ /dev/null @@ -1,276 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 - * - * linux/drivers/staging/erofs/erofs_fs.h - * - * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is dual-licensed; you may select either the GNU General Public - * License version 2 or Apache License, Version 2.0. See the file COPYING - * in the main directory of the Linux distribution for more details. - */ -#ifndef __EROFS_FS_H -#define __EROFS_FS_H - -/* Enhanced(Extended) ROM File System */ -#define EROFS_SUPER_MAGIC_V1 0xE0F5E1E2 -#define EROFS_SUPER_OFFSET 1024 - -/* - * Any bits that aren't in EROFS_ALL_REQUIREMENTS should be - * incompatible with this kernel version. - */ -#define EROFS_ALL_REQUIREMENTS 0 - -struct erofs_super_block { -/* 0 */__le32 magic; /* in the little endian */ -/* 4 */__le32 checksum; /* crc32c(super_block) */ -/* 8 */__le32 features; /* (aka. feature_compat) */ -/* 12 */__u8 blkszbits; /* support block_size == PAGE_SIZE only */ -/* 13 */__u8 reserved; - -/* 14 */__le16 root_nid; -/* 16 */__le64 inos; /* total valid ino # (== f_files - f_favail) */ - -/* 24 */__le64 build_time; /* inode v1 time derivation */ -/* 32 */__le32 build_time_nsec; -/* 36 */__le32 blocks; /* used for statfs */ -/* 40 */__le32 meta_blkaddr; -/* 44 */__le32 xattr_blkaddr; -/* 48 */__u8 uuid[16]; /* 128-bit uuid for volume */ -/* 64 */__u8 volume_name[16]; /* volume name */ -/* 80 */__le32 requirements; /* (aka. feature_incompat) */ - -/* 84 */__u8 reserved2[44]; -} __packed; /* 128 bytes */ - -#define __EROFS_BIT(_prefix, _cur, _pre) enum { \ - _prefix ## _cur ## _BIT = _prefix ## _pre ## _BIT + \ - _prefix ## _pre ## _BITS } - -/* - * erofs inode data mapping: - * 0 - inode plain without inline data A: - * inode, [xattrs], ... | ... | no-holed data - * 1 - inode VLE compression B: - * inode, [xattrs], extents ... | ... - * 2 - inode plain with inline data C: - * inode, [xattrs], last_inline_data, ... | ... | no-holed data - * 3~7 - reserved - */ -enum { - EROFS_INODE_LAYOUT_PLAIN, - EROFS_INODE_LAYOUT_COMPRESSION, - EROFS_INODE_LAYOUT_INLINE, - EROFS_INODE_LAYOUT_MAX -}; -#define EROFS_I_VERSION_BITS 1 -#define EROFS_I_DATA_MAPPING_BITS 3 - -#define EROFS_I_VERSION_BIT 0 -__EROFS_BIT(EROFS_I_, DATA_MAPPING, VERSION); - -#define EROFS_I_ALL \ - ((1 << (EROFS_I_DATA_MAPPING_BIT + EROFS_I_DATA_MAPPING_BITS)) - 1) - -struct erofs_inode_v1 { -/* 0 */__le16 i_advise; - -/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ -/* 2 */__le16 i_xattr_icount; -/* 4 */__le16 i_mode; -/* 6 */__le16 i_nlink; -/* 8 */__le32 i_size; -/* 12 */__le32 i_reserved; -/* 16 */union { - /* file total compressed blocks for data mapping 1 */ - __le32 compressed_blocks; - __le32 raw_blkaddr; - - /* for device files, used to indicate old/new device # */ - __le32 rdev; - } i_u __packed; -/* 20 */__le32 i_ino; /* only used for 32-bit stat compatibility */ -/* 24 */__le16 i_uid; -/* 26 */__le16 i_gid; -/* 28 */__le32 i_checksum; -} __packed; - -/* 32 bytes on-disk inode */ -#define EROFS_INODE_LAYOUT_V1 0 -/* 64 bytes on-disk inode */ -#define EROFS_INODE_LAYOUT_V2 1 - -struct erofs_inode_v2 { - __le16 i_advise; - - /* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ - __le16 i_xattr_icount; - __le16 i_mode; - __le16 i_reserved; /* 8 bytes */ - __le64 i_size; /* 16 bytes */ - union { - /* file total compressed blocks for data mapping 1 */ - __le32 compressed_blocks; - __le32 raw_blkaddr; - - /* for device files, used to indicate old/new device # */ - __le32 rdev; - } i_u __packed; - - /* only used for 32-bit stat compatibility */ - __le32 i_ino; /* 24 bytes */ - - __le32 i_uid; - __le32 i_gid; - __le64 i_ctime; /* 32 bytes */ - __le32 i_ctime_nsec; - __le32 i_nlink; - __u8 i_reserved2[12]; - __le32 i_checksum; /* 64 bytes */ -} __packed; - -#define EROFS_MAX_SHARED_XATTRS (128) -/* h_shared_count between 129 ... 255 are special # */ -#define EROFS_SHARED_XATTR_EXTENT (255) - -/* - * inline xattrs (n == i_xattr_icount): - * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes - * 12 bytes / \ - * / \ - * /-----------------------\ - * | erofs_xattr_entries+ | - * +-----------------------+ - * inline xattrs must starts in erofs_xattr_ibody_header, - * for read-only fs, no need to introduce h_refcount - */ -struct erofs_xattr_ibody_header { - __le32 h_checksum; - __u8 h_shared_count; - __u8 h_reserved[7]; - __le32 h_shared_xattrs[0]; /* shared xattr id array */ -} __packed; - -/* Name indexes */ -#define EROFS_XATTR_INDEX_USER 1 -#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2 -#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 -#define EROFS_XATTR_INDEX_TRUSTED 4 -#define EROFS_XATTR_INDEX_LUSTRE 5 -#define EROFS_XATTR_INDEX_SECURITY 6 - -/* xattr entry (for both inline & shared xattrs) */ -struct erofs_xattr_entry { - __u8 e_name_len; /* length of name */ - __u8 e_name_index; /* attribute name index */ - __le16 e_value_size; /* size of attribute value */ - /* followed by e_name and e_value */ - char e_name[0]; /* attribute name */ -} __packed; - -#define ondisk_xattr_ibody_size(count) ({\ - u32 __count = le16_to_cpu(count); \ - ((__count) == 0) ? 0 : \ - sizeof(struct erofs_xattr_ibody_header) + \ - sizeof(__u32) * ((__count) - 1); }) - -#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry)) -#define EROFS_XATTR_ENTRY_SIZE(entry) EROFS_XATTR_ALIGN( \ - sizeof(struct erofs_xattr_entry) + \ - (entry)->e_name_len + le16_to_cpu((entry)->e_value_size)) - -/* have to be aligned with 8 bytes on disk */ -struct erofs_extent_header { - __le32 eh_checksum; - __le32 eh_reserved[3]; -} __packed; - -/* - * Z_EROFS Variable-sized Logical Extent cluster type: - * 0 - literal (uncompressed) cluster - * 1 - compressed cluster (for the head logical cluster) - * 2 - compressed cluster (for the other logical clusters) - * - * In detail, - * 0 - literal (uncompressed) cluster, - * di_advise = 0 - * di_clusterofs = the literal data offset of the cluster - * di_blkaddr = the blkaddr of the literal cluster - * - * 1 - compressed cluster (for the head logical cluster) - * di_advise = 1 - * di_clusterofs = the decompressed data offset of the cluster - * di_blkaddr = the blkaddr of the compressed cluster - * - * 2 - compressed cluster (for the other logical clusters) - * di_advise = 2 - * di_clusterofs = - * the decompressed data offset in its own head cluster - * di_u.delta[0] = distance to its corresponding head cluster - * di_u.delta[1] = distance to its corresponding tail cluster - * (di_advise could be 0, 1 or 2) - */ -#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS 2 -#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT 0 - -struct z_erofs_vle_decompressed_index { - __le16 di_advise; - /* where to decompress in the head cluster */ - __le16 di_clusterofs; - - union { - /* for the head cluster */ - __le32 blkaddr; - /* - * for the rest clusters - * eg. for 4k page-sized cluster, maximum 4K*64k = 256M) - * [0] - pointing to the head cluster - * [1] - pointing to the tail cluster - */ - __le16 delta[2]; - } di_u __packed; /* 8 bytes */ -} __packed; - -#define Z_EROFS_VLE_EXTENT_ALIGN(size) round_up(size, \ - sizeof(struct z_erofs_vle_decompressed_index)) - -/* dirent sorts in alphabet order, thus we can do binary search */ -struct erofs_dirent { - __le64 nid; /* 0, node number */ - __le16 nameoff; /* 8, start offset of file name */ - __u8 file_type; /* 10, file type */ - __u8 reserved; /* 11, reserved */ -} __packed; - -/* file types used in inode_info->flags */ -enum { - EROFS_FT_UNKNOWN, - EROFS_FT_REG_FILE, - EROFS_FT_DIR, - EROFS_FT_CHRDEV, - EROFS_FT_BLKDEV, - EROFS_FT_FIFO, - EROFS_FT_SOCK, - EROFS_FT_SYMLINK, - EROFS_FT_MAX -}; - -#define EROFS_NAME_LEN 255 - -/* check the EROFS on-disk layout strictly at compile time */ -static inline void erofs_check_ondisk_layout_definitions(void) -{ - BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128); - BUILD_BUG_ON(sizeof(struct erofs_inode_v1) != 32); - BUILD_BUG_ON(sizeof(struct erofs_inode_v2) != 64); - BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12); - BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4); - BUILD_BUG_ON(sizeof(struct erofs_extent_header) != 16); - BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8); - BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12); -} - -#endif - diff --git a/drivers/staging/erofs/inode.c b/drivers/staging/erofs/inode.c deleted file mode 100644 index 02398c7eb4a4..000000000000 --- a/drivers/staging/erofs/inode.c +++ /dev/null @@ -1,341 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/drivers/staging/erofs/inode.c - * - * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ -#include "xattr.h" - -#include - -/* - * if inode is successfully read, return its inode page (or sometimes - * the inode payload page if it's an extended inode) in order to fill - * inline data if possible. - */ -static struct page *read_inode(struct inode *inode, unsigned int *ofs) -{ - struct super_block *sb = inode->i_sb; - struct erofs_sb_info *sbi = EROFS_SB(sb); - struct erofs_vnode *vi = EROFS_V(inode); - const erofs_off_t inode_loc = iloc(sbi, vi->nid); - erofs_blk_t blkaddr; - struct page *page; - struct erofs_inode_v1 *v1; - struct erofs_inode_v2 *v2, *copied = NULL; - unsigned int ifmt; - int err; - - blkaddr = erofs_blknr(inode_loc); - *ofs = erofs_blkoff(inode_loc); - - debugln("%s, reading inode nid %llu at %u of blkaddr %u", - __func__, vi->nid, *ofs, blkaddr); - - page = erofs_get_meta_page(sb, blkaddr, false); - if (IS_ERR(page)) { - errln("failed to get inode (nid: %llu) page, err %ld", - vi->nid, PTR_ERR(page)); - return page; - } - - v1 = page_address(page) + *ofs; - ifmt = le16_to_cpu(v1->i_advise); - - if (ifmt & ~EROFS_I_ALL) { - errln("unsupported i_format %u of nid %llu", ifmt, vi->nid); - err = -EOPNOTSUPP; - goto err_out; - } - - vi->data_mapping_mode = __inode_data_mapping(ifmt); - if (unlikely(vi->data_mapping_mode >= EROFS_INODE_LAYOUT_MAX)) { - errln("unknown data mapping mode %u of nid %llu", - vi->data_mapping_mode, vi->nid); - err = -EOPNOTSUPP; - goto err_out; - } - - switch (__inode_version(ifmt)) { - case EROFS_INODE_LAYOUT_V2: - vi->inode_isize = sizeof(struct erofs_inode_v2); - /* check if the inode acrosses page boundary */ - if (*ofs + vi->inode_isize <= PAGE_SIZE) { - *ofs += vi->inode_isize; - v2 = (struct erofs_inode_v2 *)v1; - } else { - const unsigned int gotten = PAGE_SIZE - *ofs; - - copied = kmalloc(vi->inode_isize, GFP_NOFS); - if (!copied) { - err = -ENOMEM; - goto err_out; - } - memcpy(copied, v1, gotten); - unlock_page(page); - put_page(page); - - page = erofs_get_meta_page(sb, blkaddr + 1, false); - if (IS_ERR(page)) { - errln("failed to get inode payload page (nid: %llu), err %ld", - vi->nid, PTR_ERR(page)); - kfree(copied); - return page; - } - *ofs = vi->inode_isize - gotten; - memcpy((u8 *)copied + gotten, page_address(page), *ofs); - v2 = copied; - } - vi->xattr_isize = ondisk_xattr_ibody_size(v2->i_xattr_icount); - - inode->i_mode = le16_to_cpu(v2->i_mode); - if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || - S_ISLNK(inode->i_mode)) { - vi->raw_blkaddr = le32_to_cpu(v2->i_u.raw_blkaddr); - } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { - inode->i_rdev = - new_decode_dev(le32_to_cpu(v2->i_u.rdev)); - } else if (S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { - inode->i_rdev = 0; - } else { - goto bogusimode; - } - - i_uid_write(inode, le32_to_cpu(v2->i_uid)); - i_gid_write(inode, le32_to_cpu(v2->i_gid)); - set_nlink(inode, le32_to_cpu(v2->i_nlink)); - - /* extended inode has its own timestamp */ - inode->i_ctime.tv_sec = le64_to_cpu(v2->i_ctime); - inode->i_ctime.tv_nsec = le32_to_cpu(v2->i_ctime_nsec); - - inode->i_size = le64_to_cpu(v2->i_size); - kfree(copied); - break; - case EROFS_INODE_LAYOUT_V1: - vi->inode_isize = sizeof(struct erofs_inode_v1); - *ofs += vi->inode_isize; - vi->xattr_isize = ondisk_xattr_ibody_size(v1->i_xattr_icount); - - inode->i_mode = le16_to_cpu(v1->i_mode); - if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || - S_ISLNK(inode->i_mode)) { - vi->raw_blkaddr = le32_to_cpu(v1->i_u.raw_blkaddr); - } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { - inode->i_rdev = - new_decode_dev(le32_to_cpu(v1->i_u.rdev)); - } else if (S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { - inode->i_rdev = 0; - } else { - goto bogusimode; - } - - i_uid_write(inode, le16_to_cpu(v1->i_uid)); - i_gid_write(inode, le16_to_cpu(v1->i_gid)); - set_nlink(inode, le16_to_cpu(v1->i_nlink)); - - /* use build time for compact inodes */ - inode->i_ctime.tv_sec = sbi->build_time; - inode->i_ctime.tv_nsec = sbi->build_time_nsec; - - inode->i_size = le32_to_cpu(v1->i_size); - break; - default: - errln("unsupported on-disk inode version %u of nid %llu", - __inode_version(ifmt), vi->nid); - err = -EOPNOTSUPP; - goto err_out; - } - - inode->i_mtime.tv_sec = inode->i_ctime.tv_sec; - inode->i_atime.tv_sec = inode->i_ctime.tv_sec; - inode->i_mtime.tv_nsec = inode->i_ctime.tv_nsec; - inode->i_atime.tv_nsec = inode->i_ctime.tv_nsec; - - /* measure inode.i_blocks as the generic filesystem */ - inode->i_blocks = ((inode->i_size - 1) >> 9) + 1; - return page; -bogusimode: - errln("bogus i_mode (%o) @ nid %llu", inode->i_mode, vi->nid); - err = -EIO; -err_out: - DBG_BUGON(1); - kfree(copied); - unlock_page(page); - put_page(page); - return ERR_PTR(err); -} - -/* - * try_lock can be required since locking order is: - * file data(fs_inode) - * meta(bd_inode) - * but the majority of the callers is "iget", - * in that case we are pretty sure no deadlock since - * no data operations exist. However I tend to - * try_lock since it takes no much overhead and - * will success immediately. - */ -static int fill_inline_data(struct inode *inode, void *data, unsigned m_pofs) -{ - struct erofs_vnode *vi = EROFS_V(inode); - struct erofs_sb_info *sbi = EROFS_I_SB(inode); - int mode = vi->data_mapping_mode; - - DBG_BUGON(mode >= EROFS_INODE_LAYOUT_MAX); - - /* should be inode inline C */ - if (mode != EROFS_INODE_LAYOUT_INLINE) - return 0; - - /* fast symlink (following ext4) */ - if (S_ISLNK(inode->i_mode) && inode->i_size < PAGE_SIZE) { - char *lnk = erofs_kmalloc(sbi, inode->i_size + 1, GFP_KERNEL); - - if (unlikely(lnk == NULL)) - return -ENOMEM; - - m_pofs += vi->xattr_isize; - - /* inline symlink data shouldn't across page boundary as well */ - if (unlikely(m_pofs + inode->i_size > PAGE_SIZE)) { - DBG_BUGON(1); - kfree(lnk); - return -EIO; - } - - /* get in-page inline data */ - memcpy(lnk, data + m_pofs, inode->i_size); - lnk[inode->i_size] = '\0'; - - inode->i_link = lnk; - set_inode_fast_symlink(inode); - } - return -EAGAIN; -} - -static int fill_inode(struct inode *inode, int isdir) -{ - struct page *page; - unsigned int ofs; - int err = 0; - - trace_erofs_fill_inode(inode, isdir); - - /* read inode base data from disk */ - page = read_inode(inode, &ofs); - if (IS_ERR(page)) { - return PTR_ERR(page); - } else { - /* setup the new inode */ - if (S_ISREG(inode->i_mode)) { -#ifdef CONFIG_EROFS_FS_XATTR - inode->i_op = &erofs_generic_xattr_iops; -#endif - inode->i_fop = &generic_ro_fops; - } else if (S_ISDIR(inode->i_mode)) { - inode->i_op = -#ifdef CONFIG_EROFS_FS_XATTR - &erofs_dir_xattr_iops; -#else - &erofs_dir_iops; -#endif - inode->i_fop = &erofs_dir_fops; - } else if (S_ISLNK(inode->i_mode)) { - /* by default, page_get_link is used for symlink */ - inode->i_op = -#ifdef CONFIG_EROFS_FS_XATTR - &erofs_symlink_xattr_iops, -#else - &page_symlink_inode_operations; -#endif - inode_nohighmem(inode); - } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || - S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { -#ifdef CONFIG_EROFS_FS_XATTR - inode->i_op = &erofs_special_inode_operations; -#endif - init_special_inode(inode, inode->i_mode, inode->i_rdev); - } else { - err = -EIO; - goto out_unlock; - } - - if (is_inode_layout_compression(inode)) { -#ifdef CONFIG_EROFS_FS_ZIP - inode->i_mapping->a_ops = - &z_erofs_vle_normalaccess_aops; -#else - err = -ENOTSUPP; -#endif - goto out_unlock; - } - - inode->i_mapping->a_ops = &erofs_raw_access_aops; - - /* fill last page if inline data is available */ - fill_inline_data(inode, page_address(page), ofs); - } - -out_unlock: - unlock_page(page); - put_page(page); - return err; -} - -struct inode *erofs_iget(struct super_block *sb, - erofs_nid_t nid, bool isdir) -{ - struct inode *inode = iget_locked(sb, nid); - - if (unlikely(inode == NULL)) - return ERR_PTR(-ENOMEM); - - if (inode->i_state & I_NEW) { - int err; - struct erofs_vnode *vi = EROFS_V(inode); - vi->nid = nid; - - err = fill_inode(inode, isdir); - if (likely(!err)) - unlock_new_inode(inode); - else { - iget_failed(inode); - inode = ERR_PTR(err); - } - } - return inode; -} - -#ifdef CONFIG_EROFS_FS_XATTR -const struct inode_operations erofs_generic_xattr_iops = { - .listxattr = erofs_listxattr, -}; -#endif - -#ifdef CONFIG_EROFS_FS_XATTR -const struct inode_operations erofs_symlink_xattr_iops = { - .get_link = page_get_link, - .listxattr = erofs_listxattr, -}; -#endif - -const struct inode_operations erofs_special_inode_operations = { -#ifdef CONFIG_EROFS_FS_XATTR - .listxattr = erofs_listxattr, -#endif -}; - -#ifdef CONFIG_EROFS_FS_XATTR -const struct inode_operations erofs_fast_symlink_xattr_iops = { - .get_link = simple_get_link, - .listxattr = erofs_listxattr, -}; -#endif - diff --git a/drivers/staging/erofs/internal.h b/drivers/staging/erofs/internal.h deleted file mode 100644 index 8ce37091db20..000000000000 --- a/drivers/staging/erofs/internal.h +++ /dev/null @@ -1,585 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * linux/drivers/staging/erofs/internal.h - * - * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ -#ifndef __INTERNAL_H -#define __INTERNAL_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "erofs_fs.h" - -/* redefine pr_fmt "erofs: " */ -#undef pr_fmt -#define pr_fmt(fmt) "erofs: " fmt - -#define errln(x, ...) pr_err(x "\n", ##__VA_ARGS__) -#define infoln(x, ...) pr_info(x "\n", ##__VA_ARGS__) -#ifdef CONFIG_EROFS_FS_DEBUG -#define debugln(x, ...) pr_debug(x "\n", ##__VA_ARGS__) - -#define dbg_might_sleep might_sleep -#define DBG_BUGON BUG_ON -#else -#define debugln(x, ...) ((void)0) - -#define dbg_might_sleep() ((void)0) -#define DBG_BUGON(x) ((void)(x)) -#endif - -#ifdef CONFIG_EROFS_FAULT_INJECTION -enum { - FAULT_KMALLOC, - FAULT_MAX, -}; - -extern char *erofs_fault_name[FAULT_MAX]; -#define IS_FAULT_SET(fi, type) ((fi)->inject_type & (1 << (type))) - -struct erofs_fault_info { - atomic_t inject_ops; - unsigned int inject_rate; - unsigned int inject_type; -}; -#endif - -#ifdef CONFIG_EROFS_FS_ZIP_CACHE_BIPOLAR -#define EROFS_FS_ZIP_CACHE_LVL (2) -#elif defined(EROFS_FS_ZIP_CACHE_UNIPOLAR) -#define EROFS_FS_ZIP_CACHE_LVL (1) -#else -#define EROFS_FS_ZIP_CACHE_LVL (0) -#endif - -#if (!defined(EROFS_FS_HAS_MANAGED_CACHE) && (EROFS_FS_ZIP_CACHE_LVL > 0)) -#define EROFS_FS_HAS_MANAGED_CACHE -#endif - -/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */ -#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1 - -typedef u64 erofs_nid_t; - -struct erofs_sb_info { - /* list for all registered superblocks, mainly for shrinker */ - struct list_head list; - struct mutex umount_mutex; - - u32 blocks; - u32 meta_blkaddr; -#ifdef CONFIG_EROFS_FS_XATTR - u32 xattr_blkaddr; -#endif - - /* inode slot unit size in bit shift */ - unsigned char islotbits; -#ifdef CONFIG_EROFS_FS_ZIP - /* cluster size in bit shift */ - unsigned char clusterbits; - - /* the dedicated workstation for compression */ - struct radix_tree_root workstn_tree; - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - struct inode *managed_cache; -#endif - -#endif - - u32 build_time_nsec; - u64 build_time; - - /* what we really care is nid, rather than ino.. */ - erofs_nid_t root_nid; - /* used for statfs, f_files - f_favail */ - u64 inos; - - u8 uuid[16]; /* 128-bit uuid for volume */ - u8 volume_name[16]; /* volume name */ - u32 requirements; - - char *dev_name; - - unsigned int mount_opt; - unsigned int shrinker_run_no; - -#ifdef CONFIG_EROFS_FAULT_INJECTION - struct erofs_fault_info fault_info; /* For fault injection */ -#endif -}; - -#ifdef CONFIG_EROFS_FAULT_INJECTION -#define erofs_show_injection_info(type) \ - infoln("inject %s in %s of %pS", erofs_fault_name[type], \ - __func__, __builtin_return_address(0)) - -static inline bool time_to_inject(struct erofs_sb_info *sbi, int type) -{ - struct erofs_fault_info *ffi = &sbi->fault_info; - - if (!ffi->inject_rate) - return false; - - if (!IS_FAULT_SET(ffi, type)) - return false; - - atomic_inc(&ffi->inject_ops); - if (atomic_read(&ffi->inject_ops) >= ffi->inject_rate) { - atomic_set(&ffi->inject_ops, 0); - return true; - } - return false; -} -#endif - -static inline void *erofs_kmalloc(struct erofs_sb_info *sbi, - size_t size, gfp_t flags) -{ -#ifdef CONFIG_EROFS_FAULT_INJECTION - if (time_to_inject(sbi, FAULT_KMALLOC)) { - erofs_show_injection_info(FAULT_KMALLOC); - return NULL; - } -#endif - return kmalloc(size, flags); -} - -#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) -#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info) - -/* Mount flags set via mount options or defaults */ -#define EROFS_MOUNT_XATTR_USER 0x00000010 -#define EROFS_MOUNT_POSIX_ACL 0x00000020 -#define EROFS_MOUNT_FAULT_INJECTION 0x00000040 - -#define clear_opt(sbi, option) ((sbi)->mount_opt &= ~EROFS_MOUNT_##option) -#define set_opt(sbi, option) ((sbi)->mount_opt |= EROFS_MOUNT_##option) -#define test_opt(sbi, option) ((sbi)->mount_opt & EROFS_MOUNT_##option) - -#ifdef CONFIG_EROFS_FS_ZIP -#define erofs_workstn_lock(sbi) xa_lock(&(sbi)->workstn_tree) -#define erofs_workstn_unlock(sbi) xa_unlock(&(sbi)->workstn_tree) - -/* basic unit of the workstation of a super_block */ -struct erofs_workgroup { - /* the workgroup index in the workstation */ - pgoff_t index; - - /* overall workgroup reference count */ - atomic_t refcount; -}; - -#define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL) - -#if defined(CONFIG_SMP) -static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp, - int val) -{ - preempt_disable(); - if (val != atomic_cmpxchg(&grp->refcount, val, EROFS_LOCKED_MAGIC)) { - preempt_enable(); - return false; - } - return true; -} - -static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp, - int orig_val) -{ - /* - * other observers should notice all modifications - * in the freezing period. - */ - smp_mb(); - atomic_set(&grp->refcount, orig_val); - preempt_enable(); -} - -static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp) -{ - return atomic_cond_read_relaxed(&grp->refcount, - VAL != EROFS_LOCKED_MAGIC); -} -#else -static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp, - int val) -{ - preempt_disable(); - /* no need to spin on UP platforms, let's just disable preemption. */ - if (val != atomic_read(&grp->refcount)) { - preempt_enable(); - return false; - } - return true; -} - -static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp, - int orig_val) -{ - preempt_enable(); -} - -static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp) -{ - int v = atomic_read(&grp->refcount); - - /* workgroup is never freezed on uniprocessor systems */ - DBG_BUGON(v == EROFS_LOCKED_MAGIC); - return v; -} -#endif - -static inline bool erofs_workgroup_get(struct erofs_workgroup *grp, int *ocnt) -{ - int o; - -repeat: - o = erofs_wait_on_workgroup_freezed(grp); - - if (unlikely(o <= 0)) - return -1; - - if (unlikely(atomic_cmpxchg(&grp->refcount, o, o + 1) != o)) - goto repeat; - - *ocnt = o; - return 0; -} - -#define __erofs_workgroup_get(grp) atomic_inc(&(grp)->refcount) -#define __erofs_workgroup_put(grp) atomic_dec(&(grp)->refcount) - -extern int erofs_workgroup_put(struct erofs_workgroup *grp); - -extern struct erofs_workgroup *erofs_find_workgroup( - struct super_block *sb, pgoff_t index, bool *tag); - -extern int erofs_register_workgroup(struct super_block *sb, - struct erofs_workgroup *grp, bool tag); - -extern unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi, - unsigned long nr_shrink, bool cleanup); - -static inline void erofs_workstation_cleanup_all(struct super_block *sb) -{ - erofs_shrink_workstation(EROFS_SB(sb), ~0UL, true); -} - -#ifdef EROFS_FS_HAS_MANAGED_CACHE -#define EROFS_UNALLOCATED_CACHED_PAGE ((void *)0x5F0EF00D) - -extern int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, - struct erofs_workgroup *egrp); -extern int erofs_try_to_free_cached_page(struct address_space *mapping, - struct page *page); -#endif - -#endif - -/* we strictly follow PAGE_SIZE and no buffer head yet */ -#define LOG_BLOCK_SIZE PAGE_SHIFT - -#undef LOG_SECTORS_PER_BLOCK -#define LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9) - -#undef SECTORS_PER_BLOCK -#define SECTORS_PER_BLOCK (1 << SECTORS_PER_BLOCK) - -#define EROFS_BLKSIZ (1 << LOG_BLOCK_SIZE) - -#if (EROFS_BLKSIZ % 4096 || !EROFS_BLKSIZ) -#error erofs cannot be used in this platform -#endif - -#define ROOT_NID(sb) ((sb)->root_nid) - -#ifdef CONFIG_EROFS_FS_ZIP -/* hard limit of pages per compressed cluster */ -#define Z_EROFS_CLUSTER_MAX_PAGES (CONFIG_EROFS_FS_CLUSTER_PAGE_LIMIT) - -/* page count of a compressed cluster */ -#define erofs_clusterpages(sbi) ((1 << (sbi)->clusterbits) / PAGE_SIZE) -#endif - -typedef u64 erofs_off_t; - -/* data type for filesystem-wide blocks number */ -typedef u32 erofs_blk_t; - -#define erofs_blknr(addr) ((addr) / EROFS_BLKSIZ) -#define erofs_blkoff(addr) ((addr) % EROFS_BLKSIZ) -#define blknr_to_addr(nr) ((erofs_off_t)(nr) * EROFS_BLKSIZ) - -static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid) -{ - return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits); -} - -/* atomic flag definitions */ -#define EROFS_V_EA_INITED_BIT 0 - -/* bitlock definitions (arranged in reverse order) */ -#define EROFS_V_BL_XATTR_BIT (BITS_PER_LONG - 1) - -struct erofs_vnode { - erofs_nid_t nid; - - /* atomic flags (including bitlocks) */ - unsigned long flags; - - unsigned char data_mapping_mode; - /* inline size in bytes */ - unsigned char inode_isize; - unsigned short xattr_isize; - - unsigned xattr_shared_count; - unsigned *xattr_shared_xattrs; - - erofs_blk_t raw_blkaddr; - - /* the corresponding vfs inode */ - struct inode vfs_inode; -}; - -#define EROFS_V(ptr) \ - container_of(ptr, struct erofs_vnode, vfs_inode) - -#define __inode_advise(x, bit, bits) \ - (((x) >> (bit)) & ((1 << (bits)) - 1)) - -#define __inode_version(advise) \ - __inode_advise(advise, EROFS_I_VERSION_BIT, \ - EROFS_I_VERSION_BITS) - -#define __inode_data_mapping(advise) \ - __inode_advise(advise, EROFS_I_DATA_MAPPING_BIT,\ - EROFS_I_DATA_MAPPING_BITS) - -static inline unsigned long inode_datablocks(struct inode *inode) -{ - /* since i_size cannot be changed */ - return DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); -} - -static inline bool is_inode_layout_plain(struct inode *inode) -{ - return EROFS_V(inode)->data_mapping_mode == EROFS_INODE_LAYOUT_PLAIN; -} - -static inline bool is_inode_layout_compression(struct inode *inode) -{ - return EROFS_V(inode)->data_mapping_mode == - EROFS_INODE_LAYOUT_COMPRESSION; -} - -static inline bool is_inode_layout_inline(struct inode *inode) -{ - return EROFS_V(inode)->data_mapping_mode == EROFS_INODE_LAYOUT_INLINE; -} - -extern const struct super_operations erofs_sops; -extern const struct inode_operations erofs_dir_iops; -extern const struct file_operations erofs_dir_fops; - -extern const struct address_space_operations erofs_raw_access_aops; -#ifdef CONFIG_EROFS_FS_ZIP -extern const struct address_space_operations z_erofs_vle_normalaccess_aops; -#endif - -/* - * Logical to physical block mapping, used by erofs_map_blocks() - * - * Different with other file systems, it is used for 2 access modes: - * - * 1) RAW access mode: - * - * Users pass a valid (m_lblk, m_lofs -- usually 0) pair, - * and get the valid m_pblk, m_pofs and the longest m_len(in bytes). - * - * Note that m_lblk in the RAW access mode refers to the number of - * the compressed ondisk block rather than the uncompressed - * in-memory block for the compressed file. - * - * m_pofs equals to m_lofs except for the inline data page. - * - * 2) Normal access mode: - * - * If the inode is not compressed, it has no difference with - * the RAW access mode. However, if the inode is compressed, - * users should pass a valid (m_lblk, m_lofs) pair, and get - * the needed m_pblk, m_pofs, m_len to get the compressed data - * and the updated m_lblk, m_lofs which indicates the start - * of the corresponding uncompressed data in the file. - */ -enum { - BH_Zipped = BH_PrivateStart, -}; - -/* Has a disk mapping */ -#define EROFS_MAP_MAPPED (1 << BH_Mapped) -/* Located in metadata (could be copied from bd_inode) */ -#define EROFS_MAP_META (1 << BH_Meta) -/* The extent has been compressed */ -#define EROFS_MAP_ZIPPED (1 << BH_Zipped) - -struct erofs_map_blocks { - erofs_off_t m_pa, m_la; - u64 m_plen, m_llen; - - unsigned int m_flags; -}; - -/* Flags used by erofs_map_blocks() */ -#define EROFS_GET_BLOCKS_RAW 0x0001 - -/* data.c */ -static inline struct bio *prepare_bio( - struct super_block *sb, - erofs_blk_t blkaddr, unsigned nr_pages, - bio_end_io_t endio) -{ - gfp_t gfp = GFP_NOIO; - struct bio *bio = bio_alloc(gfp, nr_pages); - - if (unlikely(bio == NULL) && - (current->flags & PF_MEMALLOC)) { - do { - nr_pages /= 2; - if (unlikely(!nr_pages)) { - bio = bio_alloc(gfp | __GFP_NOFAIL, 1); - BUG_ON(bio == NULL); - break; - } - bio = bio_alloc(gfp, nr_pages); - } while (bio == NULL); - } - - bio->bi_end_io = endio; - bio_set_dev(bio, sb->s_bdev); - bio->bi_iter.bi_sector = blkaddr << LOG_SECTORS_PER_BLOCK; - return bio; -} - -static inline void __submit_bio(struct bio *bio, unsigned op, unsigned op_flags) -{ - bio_set_op_attrs(bio, op, op_flags); - submit_bio(bio); -} - -extern struct page *erofs_get_meta_page(struct super_block *sb, - erofs_blk_t blkaddr, bool prio); -extern int erofs_map_blocks(struct inode *, struct erofs_map_blocks *, int); -extern int erofs_map_blocks_iter(struct inode *, struct erofs_map_blocks *, - struct page **, int); - -struct erofs_map_blocks_iter { - struct erofs_map_blocks map; - struct page *mpage; -}; - - -static inline struct page * -erofs_get_inline_page(struct inode *inode, - erofs_blk_t blkaddr) -{ - return erofs_get_meta_page(inode->i_sb, - blkaddr, S_ISDIR(inode->i_mode)); -} - -/* inode.c */ -extern struct inode *erofs_iget(struct super_block *sb, - erofs_nid_t nid, bool dir); - -/* dir.c */ -int erofs_namei(struct inode *dir, struct qstr *name, - erofs_nid_t *nid, unsigned *d_type); - -/* xattr.c */ -#ifdef CONFIG_EROFS_FS_XATTR -extern const struct xattr_handler *erofs_xattr_handlers[]; -#endif - -/* symlink */ -#ifdef CONFIG_EROFS_FS_XATTR -extern const struct inode_operations erofs_symlink_xattr_iops; -extern const struct inode_operations erofs_fast_symlink_xattr_iops; -extern const struct inode_operations erofs_special_inode_operations; -#endif - -static inline void set_inode_fast_symlink(struct inode *inode) -{ -#ifdef CONFIG_EROFS_FS_XATTR - inode->i_op = &erofs_fast_symlink_xattr_iops; -#else - inode->i_op = &simple_symlink_inode_operations; -#endif -} - -static inline bool is_inode_fast_symlink(struct inode *inode) -{ -#ifdef CONFIG_EROFS_FS_XATTR - return inode->i_op == &erofs_fast_symlink_xattr_iops; -#else - return inode->i_op == &simple_symlink_inode_operations; -#endif -} - -static inline void *erofs_vmap(struct page **pages, unsigned int count) -{ -#ifdef CONFIG_EROFS_FS_USE_VM_MAP_RAM - int i = 0; - - while (1) { - void *addr = vm_map_ram(pages, count, -1, PAGE_KERNEL); - /* retry two more times (totally 3 times) */ - if (addr != NULL || ++i >= 3) - return addr; - vm_unmap_aliases(); - } - return NULL; -#else - return vmap(pages, count, VM_MAP, PAGE_KERNEL); -#endif -} - -static inline void erofs_vunmap(const void *mem, unsigned int count) -{ -#ifdef CONFIG_EROFS_FS_USE_VM_MAP_RAM - vm_unmap_ram(mem, count); -#else - vunmap(mem); -#endif -} - -/* utils.c */ -extern struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp); - -extern void erofs_register_super(struct super_block *sb); -extern void erofs_unregister_super(struct super_block *sb); - -extern unsigned long erofs_shrink_count(struct shrinker *shrink, - struct shrink_control *sc); -extern unsigned long erofs_shrink_scan(struct shrinker *shrink, - struct shrink_control *sc); - -#ifndef lru_to_page -#define lru_to_page(head) (list_entry((head)->prev, struct page, lru)) -#endif - -#endif - diff --git a/drivers/staging/erofs/lz4defs.h b/drivers/staging/erofs/lz4defs.h deleted file mode 100644 index 00a0b58a0871..000000000000 --- a/drivers/staging/erofs/lz4defs.h +++ /dev/null @@ -1,227 +0,0 @@ -#ifndef __LZ4DEFS_H__ -#define __LZ4DEFS_H__ - -/* - * lz4defs.h -- common and architecture specific defines for the kernel usage - - * LZ4 - Fast LZ compression algorithm - * Copyright (C) 2011-2016, Yann Collet. - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * You can contact the author at : - * - LZ4 homepage : http://www.lz4.org - * - LZ4 source repository : https://github.com/lz4/lz4 - * - * Changed for kernel usage by: - * Sven Schmidt <4sschmid@informatik.uni-hamburg.de> - */ - -#include -#include /* memset, memcpy */ - -#define FORCE_INLINE __always_inline - -/*-************************************ - * Basic Types - **************************************/ -#include - -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -typedef uintptr_t uptrval; - -/*-************************************ - * Architecture specifics - **************************************/ -#if defined(CONFIG_64BIT) -#define LZ4_ARCH64 1 -#else -#define LZ4_ARCH64 0 -#endif - -#if defined(__LITTLE_ENDIAN) -#define LZ4_LITTLE_ENDIAN 1 -#else -#define LZ4_LITTLE_ENDIAN 0 -#endif - -/*-************************************ - * Constants - **************************************/ -#define MINMATCH 4 - -#define WILDCOPYLENGTH 8 -#define LASTLITERALS 5 -#define MFLIMIT (WILDCOPYLENGTH + MINMATCH) - -/* Increase this value ==> compression run slower on incompressible data */ -#define LZ4_SKIPTRIGGER 6 - -#define HASH_UNIT sizeof(size_t) - -#define KB (1 << 10) -#define MB (1 << 20) -#define GB (1U << 30) - -#define MAXD_LOG 16 -#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) -#define STEPSIZE sizeof(size_t) - -#define ML_BITS 4 -#define ML_MASK ((1U << ML_BITS) - 1) -#define RUN_BITS (8 - ML_BITS) -#define RUN_MASK ((1U << RUN_BITS) - 1) - -/*-************************************ - * Reading and writing into memory - **************************************/ -static FORCE_INLINE U16 LZ4_read16(const void *ptr) -{ - return get_unaligned((const U16 *)ptr); -} - -static FORCE_INLINE U32 LZ4_read32(const void *ptr) -{ - return get_unaligned((const U32 *)ptr); -} - -static FORCE_INLINE size_t LZ4_read_ARCH(const void *ptr) -{ - return get_unaligned((const size_t *)ptr); -} - -static FORCE_INLINE void LZ4_write16(void *memPtr, U16 value) -{ - put_unaligned(value, (U16 *)memPtr); -} - -static FORCE_INLINE void LZ4_write32(void *memPtr, U32 value) -{ - put_unaligned(value, (U32 *)memPtr); -} - -static FORCE_INLINE U16 LZ4_readLE16(const void *memPtr) -{ - return get_unaligned_le16(memPtr); -} - -static FORCE_INLINE void LZ4_writeLE16(void *memPtr, U16 value) -{ - return put_unaligned_le16(value, memPtr); -} - -static FORCE_INLINE void LZ4_copy8(void *dst, const void *src) -{ -#if LZ4_ARCH64 - U64 a = get_unaligned((const U64 *)src); - - put_unaligned(a, (U64 *)dst); -#else - U32 a = get_unaligned((const U32 *)src); - U32 b = get_unaligned((const U32 *)src + 1); - - put_unaligned(a, (U32 *)dst); - put_unaligned(b, (U32 *)dst + 1); -#endif -} - -/* - * customized variant of memcpy, - * which can overwrite up to 7 bytes beyond dstEnd - */ -static FORCE_INLINE void LZ4_wildCopy(void *dstPtr, - const void *srcPtr, void *dstEnd) -{ - BYTE *d = (BYTE *)dstPtr; - const BYTE *s = (const BYTE *)srcPtr; - BYTE *const e = (BYTE *)dstEnd; - - do { - LZ4_copy8(d, s); - d += 8; - s += 8; - } while (d < e); -} - -static FORCE_INLINE unsigned int LZ4_NbCommonBytes(register size_t val) -{ -#if LZ4_LITTLE_ENDIAN - return __ffs(val) >> 3; -#else - return (BITS_PER_LONG - 1 - __fls(val)) >> 3; -#endif -} - -static FORCE_INLINE unsigned int LZ4_count( - const BYTE *pIn, - const BYTE *pMatch, - const BYTE *pInLimit) -{ - const BYTE *const pStart = pIn; - - while (likely(pIn < pInLimit - (STEPSIZE - 1))) { - size_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); - - if (!diff) { - pIn += STEPSIZE; - pMatch += STEPSIZE; - continue; - } - - pIn += LZ4_NbCommonBytes(diff); - - return (unsigned int)(pIn - pStart); - } - -#if LZ4_ARCH64 - if ((pIn < (pInLimit - 3)) - && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { - pIn += 4; - pMatch += 4; - } -#endif - - if ((pIn < (pInLimit - 1)) - && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { - pIn += 2; - pMatch += 2; - } - - if ((pIn < pInLimit) && (*pMatch == *pIn)) - pIn++; - - return (unsigned int)(pIn - pStart); -} - -typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; -typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; - -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - -#endif diff --git a/drivers/staging/erofs/super.c b/drivers/staging/erofs/super.c deleted file mode 100644 index b49ebdf6ebda..000000000000 --- a/drivers/staging/erofs/super.c +++ /dev/null @@ -1,661 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/drivers/staging/erofs/super.c - * - * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ -#include -#include -#include -#include -#include -#include "internal.h" - -#define CREATE_TRACE_POINTS -#include - -static struct kmem_cache *erofs_inode_cachep __read_mostly; - -static void init_once(void *ptr) -{ - struct erofs_vnode *vi = ptr; - - inode_init_once(&vi->vfs_inode); -} - -static int erofs_init_inode_cache(void) -{ - erofs_inode_cachep = kmem_cache_create("erofs_inode", - sizeof(struct erofs_vnode), 0, - SLAB_RECLAIM_ACCOUNT, init_once); - - return erofs_inode_cachep != NULL ? 0 : -ENOMEM; -} - -static void erofs_exit_inode_cache(void) -{ - kmem_cache_destroy(erofs_inode_cachep); -} - -static struct inode *alloc_inode(struct super_block *sb) -{ - struct erofs_vnode *vi = - kmem_cache_alloc(erofs_inode_cachep, GFP_KERNEL); - - if (vi == NULL) - return NULL; - - /* zero out everything except vfs_inode */ - memset(vi, 0, offsetof(struct erofs_vnode, vfs_inode)); - return &vi->vfs_inode; -} - -static void i_callback(struct rcu_head *head) -{ - struct inode *inode = container_of(head, struct inode, i_rcu); - struct erofs_vnode *vi = EROFS_V(inode); - - /* be careful RCU symlink path (see ext4_inode_info->i_data)! */ - if (is_inode_fast_symlink(inode)) - kfree(inode->i_link); - - kfree(vi->xattr_shared_xattrs); - - kmem_cache_free(erofs_inode_cachep, vi); -} - -static void destroy_inode(struct inode *inode) -{ - call_rcu(&inode->i_rcu, i_callback); -} - -static bool check_layout_compatibility(struct super_block *sb, - struct erofs_super_block *layout) -{ - const unsigned int requirements = le32_to_cpu(layout->requirements); - - EROFS_SB(sb)->requirements = requirements; - - /* check if current kernel meets all mandatory requirements */ - if (requirements & (~EROFS_ALL_REQUIREMENTS)) { - errln("unidentified requirements %x, please upgrade kernel version", - requirements & ~EROFS_ALL_REQUIREMENTS); - return false; - } - return true; -} - -static int superblock_read(struct super_block *sb) -{ - struct erofs_sb_info *sbi; - struct buffer_head *bh; - struct erofs_super_block *layout; - unsigned blkszbits; - int ret; - - bh = sb_bread(sb, 0); - - if (bh == NULL) { - errln("cannot read erofs superblock"); - return -EIO; - } - - sbi = EROFS_SB(sb); - layout = (struct erofs_super_block *)((u8 *)bh->b_data - + EROFS_SUPER_OFFSET); - - ret = -EINVAL; - if (le32_to_cpu(layout->magic) != EROFS_SUPER_MAGIC_V1) { - errln("cannot find valid erofs superblock"); - goto out; - } - - blkszbits = layout->blkszbits; - /* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */ - if (unlikely(blkszbits != LOG_BLOCK_SIZE)) { - errln("blksize %u isn't supported on this platform", - 1 << blkszbits); - goto out; - } - - if (!check_layout_compatibility(sb, layout)) - goto out; - - sbi->blocks = le32_to_cpu(layout->blocks); - sbi->meta_blkaddr = le32_to_cpu(layout->meta_blkaddr); -#ifdef CONFIG_EROFS_FS_XATTR - sbi->xattr_blkaddr = le32_to_cpu(layout->xattr_blkaddr); -#endif - sbi->islotbits = ffs(sizeof(struct erofs_inode_v1)) - 1; -#ifdef CONFIG_EROFS_FS_ZIP - sbi->clusterbits = 12; - - if (1 << (sbi->clusterbits - 12) > Z_EROFS_CLUSTER_MAX_PAGES) - errln("clusterbits %u is not supported on this kernel", - sbi->clusterbits); -#endif - - sbi->root_nid = le16_to_cpu(layout->root_nid); - sbi->inos = le64_to_cpu(layout->inos); - - sbi->build_time = le64_to_cpu(layout->build_time); - sbi->build_time_nsec = le32_to_cpu(layout->build_time_nsec); - - memcpy(&sb->s_uuid, layout->uuid, sizeof(layout->uuid)); - memcpy(sbi->volume_name, layout->volume_name, - sizeof(layout->volume_name)); - - ret = 0; -out: - brelse(bh); - return ret; -} - -#ifdef CONFIG_EROFS_FAULT_INJECTION -char *erofs_fault_name[FAULT_MAX] = { - [FAULT_KMALLOC] = "kmalloc", -}; - -static void erofs_build_fault_attr(struct erofs_sb_info *sbi, - unsigned int rate) -{ - struct erofs_fault_info *ffi = &sbi->fault_info; - - if (rate) { - atomic_set(&ffi->inject_ops, 0); - ffi->inject_rate = rate; - ffi->inject_type = (1 << FAULT_MAX) - 1; - } else { - memset(ffi, 0, sizeof(struct erofs_fault_info)); - } -} -#endif - -static void default_options(struct erofs_sb_info *sbi) -{ -#ifdef CONFIG_EROFS_FS_XATTR - set_opt(sbi, XATTR_USER); -#endif - -#ifdef CONFIG_EROFS_FS_POSIX_ACL - set_opt(sbi, POSIX_ACL); -#endif -} - -enum { - Opt_user_xattr, - Opt_nouser_xattr, - Opt_acl, - Opt_noacl, - Opt_fault_injection, - Opt_err -}; - -static match_table_t erofs_tokens = { - {Opt_user_xattr, "user_xattr"}, - {Opt_nouser_xattr, "nouser_xattr"}, - {Opt_acl, "acl"}, - {Opt_noacl, "noacl"}, - {Opt_fault_injection, "fault_injection=%u"}, - {Opt_err, NULL} -}; - -static int parse_options(struct super_block *sb, char *options) -{ - substring_t args[MAX_OPT_ARGS]; - char *p; - int arg = 0; - - if (!options) - return 0; - - while ((p = strsep(&options, ",")) != NULL) { - int token; - - if (!*p) - continue; - - args[0].to = args[0].from = NULL; - token = match_token(p, erofs_tokens, args); - - switch (token) { -#ifdef CONFIG_EROFS_FS_XATTR - case Opt_user_xattr: - set_opt(EROFS_SB(sb), XATTR_USER); - break; - case Opt_nouser_xattr: - clear_opt(EROFS_SB(sb), XATTR_USER); - break; -#else - case Opt_user_xattr: - infoln("user_xattr options not supported"); - break; - case Opt_nouser_xattr: - infoln("nouser_xattr options not supported"); - break; -#endif -#ifdef CONFIG_EROFS_FS_POSIX_ACL - case Opt_acl: - set_opt(EROFS_SB(sb), POSIX_ACL); - break; - case Opt_noacl: - clear_opt(EROFS_SB(sb), POSIX_ACL); - break; -#else - case Opt_acl: - infoln("acl options not supported"); - break; - case Opt_noacl: - infoln("noacl options not supported"); - break; -#endif - case Opt_fault_injection: - if (args->from && match_int(args, &arg)) - return -EINVAL; -#ifdef CONFIG_EROFS_FAULT_INJECTION - erofs_build_fault_attr(EROFS_SB(sb), arg); - set_opt(EROFS_SB(sb), FAULT_INJECTION); -#else - infoln("FAULT_INJECTION was not selected"); -#endif - break; - default: - errln("Unrecognized mount option \"%s\" " - "or missing value", p); - return -EINVAL; - } - } - return 0; -} - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - -static const struct address_space_operations managed_cache_aops; - -static int managed_cache_releasepage(struct page *page, gfp_t gfp_mask) -{ - int ret = 1; /* 0 - busy */ - struct address_space *const mapping = page->mapping; - - DBG_BUGON(!PageLocked(page)); - DBG_BUGON(mapping->a_ops != &managed_cache_aops); - - if (PagePrivate(page)) - ret = erofs_try_to_free_cached_page(mapping, page); - - return ret; -} - -static void managed_cache_invalidatepage(struct page *page, - unsigned int offset, unsigned int length) -{ - const unsigned int stop = length + offset; - - DBG_BUGON(!PageLocked(page)); - - /* Check for potential overflow in debug mode */ - DBG_BUGON(stop > PAGE_SIZE || stop < length); - - if (offset == 0 && stop == PAGE_SIZE) - while (!managed_cache_releasepage(page, GFP_NOFS)) - cond_resched(); -} - -static const struct address_space_operations managed_cache_aops = { - .releasepage = managed_cache_releasepage, - .invalidatepage = managed_cache_invalidatepage, -}; - -static struct inode *erofs_init_managed_cache(struct super_block *sb) -{ - struct inode *inode = new_inode(sb); - - if (unlikely(inode == NULL)) - return ERR_PTR(-ENOMEM); - - set_nlink(inode, 1); - inode->i_size = OFFSET_MAX; - - inode->i_mapping->a_ops = &managed_cache_aops; - mapping_set_gfp_mask(inode->i_mapping, - GFP_NOFS | __GFP_HIGHMEM | - __GFP_MOVABLE | __GFP_NOFAIL); - return inode; -} - -#endif - -static int erofs_read_super(struct super_block *sb, - const char *dev_name, void *data, int silent) -{ - struct inode *inode; - struct erofs_sb_info *sbi; - int err = -EINVAL; - - infoln("read_super, device -> %s", dev_name); - infoln("options -> %s", (char *)data); - - if (unlikely(!sb_set_blocksize(sb, EROFS_BLKSIZ))) { - errln("failed to set erofs blksize"); - goto err; - } - - sbi = kzalloc(sizeof(struct erofs_sb_info), GFP_KERNEL); - if (unlikely(sbi == NULL)) { - err = -ENOMEM; - goto err; - } - sb->s_fs_info = sbi; - - err = superblock_read(sb); - if (err) - goto err_sbread; - - sb->s_magic = EROFS_SUPER_MAGIC; - sb->s_flags |= SB_RDONLY | SB_NOATIME; - sb->s_maxbytes = MAX_LFS_FILESIZE; - sb->s_time_gran = 1; - - sb->s_op = &erofs_sops; - -#ifdef CONFIG_EROFS_FS_XATTR - sb->s_xattr = erofs_xattr_handlers; -#endif - - /* set erofs default mount options */ - default_options(sbi); - - err = parse_options(sb, data); - if (err) - goto err_parseopt; - - if (!silent) - infoln("root inode @ nid %llu", ROOT_NID(sbi)); - -#ifdef CONFIG_EROFS_FS_ZIP - INIT_RADIX_TREE(&sbi->workstn_tree, GFP_ATOMIC); -#endif - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - sbi->managed_cache = erofs_init_managed_cache(sb); - if (IS_ERR(sbi->managed_cache)) { - err = PTR_ERR(sbi->managed_cache); - goto err_init_managed_cache; - } -#endif - - /* get the root inode */ - inode = erofs_iget(sb, ROOT_NID(sbi), true); - if (IS_ERR(inode)) { - err = PTR_ERR(inode); - goto err_iget; - } - - if (!S_ISDIR(inode->i_mode)) { - errln("rootino(nid %llu) is not a directory(i_mode %o)", - ROOT_NID(sbi), inode->i_mode); - err = -EINVAL; - goto err_isdir; - } - - sb->s_root = d_make_root(inode); - if (sb->s_root == NULL) { - err = -ENOMEM; - goto err_makeroot; - } - - /* save the device name to sbi */ - sbi->dev_name = __getname(); - if (sbi->dev_name == NULL) { - err = -ENOMEM; - goto err_devname; - } - - snprintf(sbi->dev_name, PATH_MAX, "%s", dev_name); - sbi->dev_name[PATH_MAX - 1] = '\0'; - - erofs_register_super(sb); - - if (!silent) - infoln("mounted on %s with opts: %s.", dev_name, - (char *)data); - return 0; - /* - * please add a label for each exit point and use - * the following name convention, thus new features - * can be integrated easily without renaming labels. - */ -err_devname: - dput(sb->s_root); -err_makeroot: -err_isdir: - if (sb->s_root == NULL) - iput(inode); -err_iget: -#ifdef EROFS_FS_HAS_MANAGED_CACHE - iput(sbi->managed_cache); -err_init_managed_cache: -#endif -err_parseopt: -err_sbread: - sb->s_fs_info = NULL; - kfree(sbi); -err: - return err; -} - -/* - * could be triggered after deactivate_locked_super() - * is called, thus including umount and failed to initialize. - */ -static void erofs_put_super(struct super_block *sb) -{ - struct erofs_sb_info *sbi = EROFS_SB(sb); - - /* for cases which are failed in "read_super" */ - if (sbi == NULL) - return; - - WARN_ON(sb->s_magic != EROFS_SUPER_MAGIC); - - infoln("unmounted for %s", sbi->dev_name); - __putname(sbi->dev_name); - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - iput(sbi->managed_cache); -#endif - - mutex_lock(&sbi->umount_mutex); - -#ifdef CONFIG_EROFS_FS_ZIP - erofs_workstation_cleanup_all(sb); -#endif - - erofs_unregister_super(sb); - mutex_unlock(&sbi->umount_mutex); - - kfree(sbi); - sb->s_fs_info = NULL; -} - - -struct erofs_mount_private { - const char *dev_name; - char *options; -}; - -/* support mount_bdev() with options */ -static int erofs_fill_super(struct super_block *sb, - void *_priv, int silent) -{ - struct erofs_mount_private *priv = _priv; - - return erofs_read_super(sb, priv->dev_name, - priv->options, silent); -} - -static struct dentry *erofs_mount( - struct file_system_type *fs_type, int flags, - const char *dev_name, void *data) -{ - struct erofs_mount_private priv = { - .dev_name = dev_name, - .options = data - }; - - return mount_bdev(fs_type, flags, dev_name, - &priv, erofs_fill_super); -} - -static void erofs_kill_sb(struct super_block *sb) -{ - kill_block_super(sb); -} - -static struct shrinker erofs_shrinker_info = { - .scan_objects = erofs_shrink_scan, - .count_objects = erofs_shrink_count, - .seeks = DEFAULT_SEEKS, -}; - -static struct file_system_type erofs_fs_type = { - .owner = THIS_MODULE, - .name = "erofs", - .mount = erofs_mount, - .kill_sb = erofs_kill_sb, - .fs_flags = FS_REQUIRES_DEV, -}; -MODULE_ALIAS_FS("erofs"); - -#ifdef CONFIG_EROFS_FS_ZIP -extern int z_erofs_init_zip_subsystem(void); -extern void z_erofs_exit_zip_subsystem(void); -#endif - -static int __init erofs_module_init(void) -{ - int err; - - erofs_check_ondisk_layout_definitions(); - infoln("initializing erofs " EROFS_VERSION); - - err = erofs_init_inode_cache(); - if (err) - goto icache_err; - - err = register_shrinker(&erofs_shrinker_info); - if (err) - goto shrinker_err; - -#ifdef CONFIG_EROFS_FS_ZIP - err = z_erofs_init_zip_subsystem(); - if (err) - goto zip_err; -#endif - - err = register_filesystem(&erofs_fs_type); - if (err) - goto fs_err; - - infoln("successfully to initialize erofs"); - return 0; - -fs_err: -#ifdef CONFIG_EROFS_FS_ZIP - z_erofs_exit_zip_subsystem(); -zip_err: -#endif - unregister_shrinker(&erofs_shrinker_info); -shrinker_err: - erofs_exit_inode_cache(); -icache_err: - return err; -} - -static void __exit erofs_module_exit(void) -{ - unregister_filesystem(&erofs_fs_type); -#ifdef CONFIG_EROFS_FS_ZIP - z_erofs_exit_zip_subsystem(); -#endif - unregister_shrinker(&erofs_shrinker_info); - erofs_exit_inode_cache(); - infoln("successfully finalize erofs"); -} - -/* get filesystem statistics */ -static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf) -{ - struct super_block *sb = dentry->d_sb; - struct erofs_sb_info *sbi = EROFS_SB(sb); - u64 id = huge_encode_dev(sb->s_bdev->bd_dev); - - buf->f_type = sb->s_magic; - buf->f_bsize = EROFS_BLKSIZ; - buf->f_blocks = sbi->blocks; - buf->f_bfree = buf->f_bavail = 0; - - buf->f_files = ULLONG_MAX; - buf->f_ffree = ULLONG_MAX - sbi->inos; - - buf->f_namelen = EROFS_NAME_LEN; - - buf->f_fsid.val[0] = (u32)id; - buf->f_fsid.val[1] = (u32)(id >> 32); - return 0; -} - -static int erofs_show_options(struct seq_file *seq, struct dentry *root) -{ - struct erofs_sb_info *sbi __maybe_unused = EROFS_SB(root->d_sb); - -#ifdef CONFIG_EROFS_FS_XATTR - if (test_opt(sbi, XATTR_USER)) - seq_puts(seq, ",user_xattr"); - else - seq_puts(seq, ",nouser_xattr"); -#endif -#ifdef CONFIG_EROFS_FS_POSIX_ACL - if (test_opt(sbi, POSIX_ACL)) - seq_puts(seq, ",acl"); - else - seq_puts(seq, ",noacl"); -#endif -#ifdef CONFIG_EROFS_FAULT_INJECTION - if (test_opt(sbi, FAULT_INJECTION)) - seq_printf(seq, ",fault_injection=%u", - sbi->fault_info.inject_rate); -#endif - return 0; -} - -static int erofs_remount(struct super_block *sb, int *flags, char *data) -{ - DBG_BUGON(!sb_rdonly(sb)); - - *flags |= SB_RDONLY; - return 0; -} - -const struct super_operations erofs_sops = { - .put_super = erofs_put_super, - .alloc_inode = alloc_inode, - .destroy_inode = destroy_inode, - .statfs = erofs_statfs, - .show_options = erofs_show_options, - .remount_fs = erofs_remount, -}; - -module_init(erofs_module_init); -module_exit(erofs_module_exit); - -MODULE_DESCRIPTION("Enhanced ROM File System"); -MODULE_AUTHOR("Gao Xiang, Yu Chao, Miao Xie, CONSUMER BG, HUAWEI Inc."); -MODULE_LICENSE("GPL"); - diff --git a/drivers/staging/erofs/unzip_lz4.c b/drivers/staging/erofs/unzip_lz4.c deleted file mode 100644 index b1ea23f66c4e..000000000000 --- a/drivers/staging/erofs/unzip_lz4.c +++ /dev/null @@ -1,251 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -/* - * linux/drivers/staging/erofs/unzip_lz4.c - * - * Copyright (C) 2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * Original code taken from 'linux/lib/lz4/lz4_decompress.c' - */ - -/* - * LZ4 - Fast LZ compression algorithm - * Copyright (C) 2011 - 2016, Yann Collet. - * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php) - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * You can contact the author at : - * - LZ4 homepage : http://www.lz4.org - * - LZ4 source repository : https://github.com/lz4/lz4 - * - * Changed for kernel usage by: - * Sven Schmidt <4sschmid@informatik.uni-hamburg.de> - */ -#include "internal.h" -#include -#include "lz4defs.h" - -/* - * no public solution to solve our requirement yet. - * see: - * https://groups.google.com/forum/#!topic/lz4c/_3kkz5N6n00 - */ -static FORCE_INLINE int customized_lz4_decompress_safe_partial( - const void * const source, - void * const dest, - int inputSize, - int outputSize) -{ - /* Local Variables */ - const BYTE *ip = (const BYTE *) source; - const BYTE * const iend = ip + inputSize; - - BYTE *op = (BYTE *) dest; - BYTE * const oend = op + outputSize; - BYTE *cpy; - - static const unsigned int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; - static const int dec64table[] = { 0, 0, 0, -1, 0, 1, 2, 3 }; - - /* Empty output buffer */ - if (unlikely(outputSize == 0)) - return ((inputSize == 1) && (*ip == 0)) ? 0 : -1; - - /* Main Loop : decode sequences */ - while (1) { - size_t length; - const BYTE *match; - size_t offset; - - /* get literal length */ - unsigned int const token = *ip++; - - length = token>>ML_BITS; - - if (length == RUN_MASK) { - unsigned int s; - - do { - s = *ip++; - length += s; - } while ((ip < iend - RUN_MASK) & (s == 255)); - - if (unlikely((size_t)(op + length) < (size_t)(op))) { - /* overflow detection */ - goto _output_error; - } - if (unlikely((size_t)(ip + length) < (size_t)(ip))) { - /* overflow detection */ - goto _output_error; - } - } - - /* copy literals */ - cpy = op + length; - if ((cpy > oend - WILDCOPYLENGTH) || - (ip + length > iend - (2 + 1 + LASTLITERALS))) { - if (cpy > oend) { - memcpy(op, ip, length = oend - op); - op += length; - break; - } - - if (unlikely(ip + length > iend)) { - /* - * Error : - * read attempt beyond - * end of input buffer - */ - goto _output_error; - } - - memcpy(op, ip, length); - ip += length; - op += length; - - if (ip > iend - 2) - break; - /* Necessarily EOF, due to parsing restrictions */ - /* break; */ - } else { - LZ4_wildCopy(op, ip, cpy); - ip += length; - op = cpy; - } - - /* get offset */ - offset = LZ4_readLE16(ip); - ip += 2; - match = op - offset; - - if (unlikely(match < (const BYTE *)dest)) { - /* Error : offset outside buffers */ - goto _output_error; - } - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned int s; - - do { - s = *ip++; - - if (ip > iend - LASTLITERALS) - goto _output_error; - - length += s; - } while (s == 255); - - if (unlikely((size_t)(op + length) < (size_t)op)) { - /* overflow detection */ - goto _output_error; - } - } - - length += MINMATCH; - - /* copy match within block */ - cpy = op + length; - - if (unlikely(cpy >= oend - WILDCOPYLENGTH)) { - if (cpy >= oend) { - while (op < oend) - *op++ = *match++; - break; - } - goto __match; - } - - /* costs ~1%; silence an msan warning when offset == 0 */ - LZ4_write32(op, (U32)offset); - - if (unlikely(offset < 8)) { - const int dec64 = dec64table[offset]; - - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op + 4, match, 4); - match -= dec64; - } else { - LZ4_copy8(op, match); - match += 8; - } - - op += 8; - - if (unlikely(cpy > oend - 12)) { - BYTE * const oCopyLimit = oend - (WILDCOPYLENGTH - 1); - - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } -__match: - while (op < cpy) - *op++ = *match++; - } else { - LZ4_copy8(op, match); - - if (length > 16) - LZ4_wildCopy(op + 8, match + 8, cpy); - } - - op = cpy; /* correction */ - } - DBG_BUGON((void *)ip - source > inputSize); - DBG_BUGON((void *)op - dest > outputSize); - - /* Nb of output bytes decoded */ - return (int) ((void *)op - dest); - - /* Overflow error detected */ -_output_error: - return -ERANGE; -} - -int z_erofs_unzip_lz4(void *in, void *out, size_t inlen, size_t outlen) -{ - int ret = customized_lz4_decompress_safe_partial(in, - out, inlen, outlen); - - if (ret >= 0) - return ret; - - /* - * LZ4_decompress_safe will return an error code - * (< 0) if decompression failed - */ - errln("%s, failed to decompress, in[%p, %zu] outlen[%p, %zu]", - __func__, in, inlen, out, outlen); - WARN_ON(1); - print_hex_dump(KERN_DEBUG, "raw data [in]: ", DUMP_PREFIX_OFFSET, - 16, 1, in, inlen, true); - print_hex_dump(KERN_DEBUG, "raw data [out]: ", DUMP_PREFIX_OFFSET, - 16, 1, out, outlen, true); - return -EIO; -} - diff --git a/drivers/staging/erofs/unzip_vle.c b/drivers/staging/erofs/unzip_vle.c deleted file mode 100644 index 83e4d9384bd2..000000000000 --- a/drivers/staging/erofs/unzip_vle.c +++ /dev/null @@ -1,1694 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/drivers/staging/erofs/unzip_vle.c - * - * Copyright (C) 2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ -#include "unzip_vle.h" -#include - -static struct workqueue_struct *z_erofs_workqueue __read_mostly; -static struct kmem_cache *z_erofs_workgroup_cachep __read_mostly; - -void z_erofs_exit_zip_subsystem(void) -{ - destroy_workqueue(z_erofs_workqueue); - kmem_cache_destroy(z_erofs_workgroup_cachep); -} - -static inline int init_unzip_workqueue(void) -{ - const unsigned onlinecpus = num_possible_cpus(); - - /* - * we don't need too many threads, limiting threads - * could improve scheduling performance. - */ - z_erofs_workqueue = alloc_workqueue("erofs_unzipd", - WQ_UNBOUND | WQ_HIGHPRI | WQ_CPU_INTENSIVE, - onlinecpus + onlinecpus / 4); - - return z_erofs_workqueue != NULL ? 0 : -ENOMEM; -} - -int z_erofs_init_zip_subsystem(void) -{ - z_erofs_workgroup_cachep = - kmem_cache_create("erofs_compress", - Z_EROFS_WORKGROUP_SIZE, 0, - SLAB_RECLAIM_ACCOUNT, NULL); - - if (z_erofs_workgroup_cachep != NULL) { - if (!init_unzip_workqueue()) - return 0; - - kmem_cache_destroy(z_erofs_workgroup_cachep); - } - return -ENOMEM; -} - -enum z_erofs_vle_work_role { - Z_EROFS_VLE_WORK_SECONDARY, - Z_EROFS_VLE_WORK_PRIMARY, - /* - * The current work was the tail of an exist chain, and the previous - * processed chained works are all decided to be hooked up to it. - * A new chain should be created for the remaining unprocessed works, - * therefore different from Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED, - * the next work cannot reuse the whole page in the following scenario: - * ________________________________________________________________ - * | tail (partial) page | head (partial) page | - * | (belongs to the next work) | (belongs to the current work) | - * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________| - */ - Z_EROFS_VLE_WORK_PRIMARY_HOOKED, - /* - * The current work has been linked with the processed chained works, - * and could be also linked with the potential remaining works, which - * means if the processing page is the tail partial page of the work, - * the current work can safely use the whole page (since the next work - * is under control) for in-place decompression, as illustrated below: - * ________________________________________________________________ - * | tail (partial) page | head (partial) page | - * | (of the current work) | (of the previous work) | - * | PRIMARY_FOLLOWED or | | - * |_____PRIMARY_HOOKED____|____________PRIMARY_FOLLOWED____________| - * - * [ (*) the above page can be used for the current work itself. ] - */ - Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED, - Z_EROFS_VLE_WORK_MAX -}; - -struct z_erofs_vle_work_builder { - enum z_erofs_vle_work_role role; - /* - * 'hosted = false' means that the current workgroup doesn't belong to - * the owned chained workgroups. In the other words, it is none of our - * business to submit this workgroup. - */ - bool hosted; - - struct z_erofs_vle_workgroup *grp; - struct z_erofs_vle_work *work; - struct z_erofs_pagevec_ctor vector; - - /* pages used for reading the compressed data */ - struct page **compressed_pages; - unsigned compressed_deficit; -}; - -#define VLE_WORK_BUILDER_INIT() \ - { .work = NULL, .role = Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED } - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - -static bool grab_managed_cache_pages(struct address_space *mapping, - erofs_blk_t start, - struct page **compressed_pages, - int clusterblks, - bool reserve_allocation) -{ - bool noio = true; - unsigned int i; - - /* TODO: optimize by introducing find_get_pages_range */ - for (i = 0; i < clusterblks; ++i) { - struct page *page, *found; - - if (READ_ONCE(compressed_pages[i]) != NULL) - continue; - - page = found = find_get_page(mapping, start + i); - if (found == NULL) { - noio = false; - if (!reserve_allocation) - continue; - page = EROFS_UNALLOCATED_CACHED_PAGE; - } - - if (NULL == cmpxchg(compressed_pages + i, NULL, page)) - continue; - - if (found != NULL) - put_page(found); - } - return noio; -} - -/* called by erofs_shrinker to get rid of all compressed_pages */ -int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, - struct erofs_workgroup *egrp) -{ - struct z_erofs_vle_workgroup *const grp = - container_of(egrp, struct z_erofs_vle_workgroup, obj); - struct address_space *const mapping = sbi->managed_cache->i_mapping; - const int clusterpages = erofs_clusterpages(sbi); - int i; - - /* - * refcount of workgroup is now freezed as 1, - * therefore no need to worry about available decompression users. - */ - for (i = 0; i < clusterpages; ++i) { - struct page *page = grp->compressed_pages[i]; - - if (page == NULL || page->mapping != mapping) - continue; - - /* block other users from reclaiming or migrating the page */ - if (!trylock_page(page)) - return -EBUSY; - - /* barrier is implied in the following 'unlock_page' */ - WRITE_ONCE(grp->compressed_pages[i], NULL); - - set_page_private(page, 0); - ClearPagePrivate(page); - - unlock_page(page); - put_page(page); - } - return 0; -} - -int erofs_try_to_free_cached_page(struct address_space *mapping, - struct page *page) -{ - struct erofs_sb_info *const sbi = EROFS_SB(mapping->host->i_sb); - const unsigned int clusterpages = erofs_clusterpages(sbi); - - struct z_erofs_vle_workgroup *grp; - int ret = 0; /* 0 - busy */ - - /* prevent the workgroup from being freed */ - rcu_read_lock(); - grp = (void *)page_private(page); - - if (erofs_workgroup_try_to_freeze(&grp->obj, 1)) { - unsigned int i; - - for (i = 0; i < clusterpages; ++i) { - if (grp->compressed_pages[i] == page) { - WRITE_ONCE(grp->compressed_pages[i], NULL); - ret = 1; - break; - } - } - erofs_workgroup_unfreeze(&grp->obj, 1); - } - rcu_read_unlock(); - - if (ret) { - ClearPagePrivate(page); - put_page(page); - } - return ret; -} -#endif - -/* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */ -static inline bool try_to_reuse_as_compressed_page( - struct z_erofs_vle_work_builder *b, - struct page *page) -{ - while (b->compressed_deficit) { - --b->compressed_deficit; - if (NULL == cmpxchg(b->compressed_pages++, NULL, page)) - return true; - } - - return false; -} - -/* callers must be with work->lock held */ -static int z_erofs_vle_work_add_page(struct z_erofs_vle_work_builder *builder, - struct page *page, - enum z_erofs_page_type type, - bool pvec_safereuse) -{ - int ret; - - /* give priority for the compressed data storage */ - if (builder->role >= Z_EROFS_VLE_WORK_PRIMARY && - type == Z_EROFS_PAGE_TYPE_EXCLUSIVE && - try_to_reuse_as_compressed_page(builder, page)) - return 0; - - ret = z_erofs_pagevec_ctor_enqueue(&builder->vector, page, type, - pvec_safereuse); - builder->work->vcnt += (unsigned)ret; - return ret ? 0 : -EAGAIN; -} - -static enum z_erofs_vle_work_role -try_to_claim_workgroup(struct z_erofs_vle_workgroup *grp, - z_erofs_vle_owned_workgrp_t *owned_head, - bool *hosted) -{ - DBG_BUGON(*hosted == true); - - /* let's claim these following types of workgroup */ -retry: - if (grp->next == Z_EROFS_VLE_WORKGRP_NIL) { - /* type 1, nil workgroup */ - if (Z_EROFS_VLE_WORKGRP_NIL != cmpxchg(&grp->next, - Z_EROFS_VLE_WORKGRP_NIL, *owned_head)) - goto retry; - - *owned_head = grp; - *hosted = true; - /* lucky, I am the followee :) */ - return Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED; - - } else if (grp->next == Z_EROFS_VLE_WORKGRP_TAIL) { - /* - * type 2, link to the end of a existing open chain, - * be careful that its submission itself is governed - * by the original owned chain. - */ - if (Z_EROFS_VLE_WORKGRP_TAIL != cmpxchg(&grp->next, - Z_EROFS_VLE_WORKGRP_TAIL, *owned_head)) - goto retry; - *owned_head = Z_EROFS_VLE_WORKGRP_TAIL; - return Z_EROFS_VLE_WORK_PRIMARY_HOOKED; - } - - return Z_EROFS_VLE_WORK_PRIMARY; /* :( better luck next time */ -} - -static struct z_erofs_vle_work * -z_erofs_vle_work_lookup(struct super_block *sb, - pgoff_t idx, unsigned pageofs, - struct z_erofs_vle_workgroup **grp_ret, - enum z_erofs_vle_work_role *role, - z_erofs_vle_owned_workgrp_t *owned_head, - bool *hosted) -{ - bool tag, primary; - struct erofs_workgroup *egrp; - struct z_erofs_vle_workgroup *grp; - struct z_erofs_vle_work *work; - - egrp = erofs_find_workgroup(sb, idx, &tag); - if (egrp == NULL) { - *grp_ret = NULL; - return NULL; - } - - *grp_ret = grp = container_of(egrp, - struct z_erofs_vle_workgroup, obj); - - work = z_erofs_vle_grab_work(grp, pageofs); - /* if multiref is disabled, `primary' is always true */ - primary = true; - - if (work->pageofs != pageofs) { - DBG_BUGON(1); - erofs_workgroup_put(egrp); - return ERR_PTR(-EIO); - } - - /* - * lock must be taken first to avoid grp->next == NIL between - * claiming workgroup and adding pages: - * grp->next != NIL - * grp->next = NIL - * mutex_unlock_all - * mutex_lock(&work->lock) - * add all pages to pagevec - * - * [correct locking case 1]: - * mutex_lock(grp->work[a]) - * ... - * mutex_lock(grp->work[b]) mutex_lock(grp->work[c]) - * ... *role = SECONDARY - * add all pages to pagevec - * ... - * mutex_unlock(grp->work[c]) - * mutex_lock(grp->work[c]) - * ... - * grp->next = NIL - * mutex_unlock_all - * - * [correct locking case 2]: - * mutex_lock(grp->work[b]) - * ... - * mutex_lock(grp->work[a]) - * ... - * mutex_lock(grp->work[c]) - * ... - * grp->next = NIL - * mutex_unlock_all - * mutex_lock(grp->work[a]) - * *role = PRIMARY_OWNER - * add all pages to pagevec - * ... - */ - mutex_lock(&work->lock); - - *hosted = false; - if (!primary) - *role = Z_EROFS_VLE_WORK_SECONDARY; - else /* claim the workgroup if possible */ - *role = try_to_claim_workgroup(grp, owned_head, hosted); - return work; -} - -static struct z_erofs_vle_work * -z_erofs_vle_work_register(struct super_block *sb, - struct z_erofs_vle_workgroup **grp_ret, - struct erofs_map_blocks *map, - pgoff_t index, unsigned pageofs, - enum z_erofs_vle_work_role *role, - z_erofs_vle_owned_workgrp_t *owned_head, - bool *hosted) -{ - bool newgrp = false; - struct z_erofs_vle_workgroup *grp = *grp_ret; - struct z_erofs_vle_work *work; - - /* if multiref is disabled, grp should never be nullptr */ - if (unlikely(grp)) { - DBG_BUGON(1); - return ERR_PTR(-EINVAL); - } - - /* no available workgroup, let's allocate one */ - grp = kmem_cache_zalloc(z_erofs_workgroup_cachep, GFP_NOFS); - if (unlikely(grp == NULL)) - return ERR_PTR(-ENOMEM); - - grp->obj.index = index; - grp->llen = map->m_llen; - - z_erofs_vle_set_workgrp_fmt(grp, - (map->m_flags & EROFS_MAP_ZIPPED) ? - Z_EROFS_VLE_WORKGRP_FMT_LZ4 : - Z_EROFS_VLE_WORKGRP_FMT_PLAIN); - atomic_set(&grp->obj.refcount, 1); - - /* new workgrps have been claimed as type 1 */ - WRITE_ONCE(grp->next, *owned_head); - /* primary and followed work for all new workgrps */ - *role = Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED; - /* it should be submitted by ourselves */ - *hosted = true; - - newgrp = true; - work = z_erofs_vle_grab_primary_work(grp); - work->pageofs = pageofs; - - mutex_init(&work->lock); - - if (newgrp) { - int err = erofs_register_workgroup(sb, &grp->obj, 0); - - if (err) { - kmem_cache_free(z_erofs_workgroup_cachep, grp); - return ERR_PTR(-EAGAIN); - } - } - - *owned_head = *grp_ret = grp; - - mutex_lock(&work->lock); - return work; -} - -static inline void __update_workgrp_llen(struct z_erofs_vle_workgroup *grp, - unsigned int llen) -{ - while (1) { - unsigned int orig_llen = grp->llen; - - if (orig_llen >= llen || orig_llen == - cmpxchg(&grp->llen, orig_llen, llen)) - break; - } -} - -#define builder_is_hooked(builder) \ - ((builder)->role >= Z_EROFS_VLE_WORK_PRIMARY_HOOKED) - -#define builder_is_followed(builder) \ - ((builder)->role >= Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED) - -static int z_erofs_vle_work_iter_begin(struct z_erofs_vle_work_builder *builder, - struct super_block *sb, - struct erofs_map_blocks *map, - z_erofs_vle_owned_workgrp_t *owned_head) -{ - const unsigned clusterpages = erofs_clusterpages(EROFS_SB(sb)); - const erofs_blk_t index = erofs_blknr(map->m_pa); - const unsigned pageofs = map->m_la & ~PAGE_MASK; - struct z_erofs_vle_workgroup *grp; - struct z_erofs_vle_work *work; - - DBG_BUGON(builder->work != NULL); - - /* must be Z_EROFS_WORK_TAIL or the next chained work */ - DBG_BUGON(*owned_head == Z_EROFS_VLE_WORKGRP_NIL); - DBG_BUGON(*owned_head == Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); - - DBG_BUGON(erofs_blkoff(map->m_pa)); - -repeat: - work = z_erofs_vle_work_lookup(sb, index, - pageofs, &grp, &builder->role, owned_head, &builder->hosted); - if (work != NULL) { - __update_workgrp_llen(grp, map->m_llen); - goto got_it; - } - - work = z_erofs_vle_work_register(sb, &grp, map, index, pageofs, - &builder->role, owned_head, &builder->hosted); - - if (unlikely(work == ERR_PTR(-EAGAIN))) - goto repeat; - - if (unlikely(IS_ERR(work))) - return PTR_ERR(work); -got_it: - z_erofs_pagevec_ctor_init(&builder->vector, - Z_EROFS_VLE_INLINE_PAGEVECS, work->pagevec, work->vcnt); - - if (builder->role >= Z_EROFS_VLE_WORK_PRIMARY) { - /* enable possibly in-place decompression */ - builder->compressed_pages = grp->compressed_pages; - builder->compressed_deficit = clusterpages; - } else { - builder->compressed_pages = NULL; - builder->compressed_deficit = 0; - } - - builder->grp = grp; - builder->work = work; - return 0; -} - -/* - * keep in mind that no referenced workgroups will be freed - * only after a RCU grace period, so rcu_read_lock() could - * prevent a workgroup from being freed. - */ -static void z_erofs_rcu_callback(struct rcu_head *head) -{ - struct z_erofs_vle_work *work = container_of(head, - struct z_erofs_vle_work, rcu); - struct z_erofs_vle_workgroup *grp = - z_erofs_vle_work_workgroup(work, true); - - kmem_cache_free(z_erofs_workgroup_cachep, grp); -} - -void erofs_workgroup_free_rcu(struct erofs_workgroup *grp) -{ - struct z_erofs_vle_workgroup *const vgrp = container_of(grp, - struct z_erofs_vle_workgroup, obj); - struct z_erofs_vle_work *const work = &vgrp->work; - - call_rcu(&work->rcu, z_erofs_rcu_callback); -} - -static void __z_erofs_vle_work_release(struct z_erofs_vle_workgroup *grp, - struct z_erofs_vle_work *work __maybe_unused) -{ - erofs_workgroup_put(&grp->obj); -} - -void z_erofs_vle_work_release(struct z_erofs_vle_work *work) -{ - struct z_erofs_vle_workgroup *grp = - z_erofs_vle_work_workgroup(work, true); - - __z_erofs_vle_work_release(grp, work); -} - -static inline bool -z_erofs_vle_work_iter_end(struct z_erofs_vle_work_builder *builder) -{ - struct z_erofs_vle_work *work = builder->work; - - if (work == NULL) - return false; - - z_erofs_pagevec_ctor_exit(&builder->vector, false); - mutex_unlock(&work->lock); - - /* - * if all pending pages are added, don't hold work reference - * any longer if the current work isn't hosted by ourselves. - */ - if (!builder->hosted) - __z_erofs_vle_work_release(builder->grp, work); - - builder->work = NULL; - builder->grp = NULL; - return true; -} - -static inline struct page *__stagingpage_alloc(struct list_head *pagepool, - gfp_t gfp) -{ - struct page *page = erofs_allocpage(pagepool, gfp); - - if (unlikely(page == NULL)) - return NULL; - - page->mapping = Z_EROFS_MAPPING_STAGING; - return page; -} - -struct z_erofs_vle_frontend { - struct inode *const inode; - - struct z_erofs_vle_work_builder builder; - struct erofs_map_blocks_iter m_iter; - - z_erofs_vle_owned_workgrp_t owned_head; - - bool initial; -#if (EROFS_FS_ZIP_CACHE_LVL >= 2) - erofs_off_t cachedzone_la; -#endif -}; - -#define VLE_FRONTEND_INIT(__i) { \ - .inode = __i, \ - .m_iter = { \ - { .m_llen = 0, .m_plen = 0 }, \ - .mpage = NULL \ - }, \ - .builder = VLE_WORK_BUILDER_INIT(), \ - .owned_head = Z_EROFS_VLE_WORKGRP_TAIL, \ - .initial = true, } - -static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, - struct page *page, - struct list_head *page_pool) -{ - struct super_block *const sb = fe->inode->i_sb; - struct erofs_sb_info *const sbi __maybe_unused = EROFS_SB(sb); - struct erofs_map_blocks_iter *const m = &fe->m_iter; - struct erofs_map_blocks *const map = &m->map; - struct z_erofs_vle_work_builder *const builder = &fe->builder; - const loff_t offset = page_offset(page); - - bool tight = builder_is_hooked(builder); - struct z_erofs_vle_work *work = builder->work; - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - struct address_space *const mngda = sbi->managed_cache->i_mapping; - struct z_erofs_vle_workgroup *grp; - bool noio_outoforder; -#endif - - enum z_erofs_page_type page_type; - unsigned cur, end, spiltted, index; - int err = 0; - - /* register locked file pages as online pages in pack */ - z_erofs_onlinepage_init(page); - - spiltted = 0; - end = PAGE_SIZE; -repeat: - cur = end - 1; - - /* lucky, within the range of the current map_blocks */ - if (offset + cur >= map->m_la && - offset + cur < map->m_la + map->m_llen) { - /* didn't get a valid unzip work previously (very rare) */ - if (!builder->work) - goto restart_now; - goto hitted; - } - - /* go ahead the next map_blocks */ - debugln("%s: [out-of-range] pos %llu", __func__, offset + cur); - - if (z_erofs_vle_work_iter_end(builder)) - fe->initial = false; - - map->m_la = offset + cur; - map->m_llen = 0; - err = erofs_map_blocks_iter(fe->inode, map, &m->mpage, 0); - if (unlikely(err)) - goto err_out; - -restart_now: - if (unlikely(!(map->m_flags & EROFS_MAP_MAPPED))) - goto hitted; - - DBG_BUGON(map->m_plen != 1 << sbi->clusterbits); - DBG_BUGON(erofs_blkoff(map->m_pa)); - - err = z_erofs_vle_work_iter_begin(builder, sb, map, &fe->owned_head); - if (unlikely(err)) - goto err_out; - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - grp = fe->builder.grp; - - /* let's do out-of-order decompression for noio */ - noio_outoforder = grab_managed_cache_pages(mngda, - erofs_blknr(map->m_pa), - grp->compressed_pages, erofs_blknr(map->m_plen), - /* compressed page caching selection strategy */ - fe->initial | (EROFS_FS_ZIP_CACHE_LVL >= 2 ? - map->m_la < fe->cachedzone_la : 0)); - - if (noio_outoforder && builder_is_followed(builder)) - builder->role = Z_EROFS_VLE_WORK_PRIMARY; -#endif - - tight &= builder_is_hooked(builder); - work = builder->work; -hitted: - cur = end - min_t(unsigned, offset + end - map->m_la, end); - if (unlikely(!(map->m_flags & EROFS_MAP_MAPPED))) { - zero_user_segment(page, cur, end); - goto next_part; - } - - /* let's derive page type */ - page_type = cur ? Z_EROFS_VLE_PAGE_TYPE_HEAD : - (!spiltted ? Z_EROFS_PAGE_TYPE_EXCLUSIVE : - (tight ? Z_EROFS_PAGE_TYPE_EXCLUSIVE : - Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED)); - - if (cur) - tight &= builder_is_followed(builder); - -retry: - err = z_erofs_vle_work_add_page(builder, page, page_type, - builder_is_followed(builder)); - /* should allocate an additional staging page for pagevec */ - if (err == -EAGAIN) { - struct page *const newpage = - __stagingpage_alloc(page_pool, GFP_NOFS); - - err = z_erofs_vle_work_add_page(builder, - newpage, Z_EROFS_PAGE_TYPE_EXCLUSIVE, true); - if (likely(!err)) - goto retry; - } - - if (unlikely(err)) - goto err_out; - - index = page->index - map->m_la / PAGE_SIZE; - - /* FIXME! avoid the last relundant fixup & endio */ - z_erofs_onlinepage_fixup(page, index, true); - - /* bump up the number of spiltted parts of a page */ - ++spiltted; - /* also update nr_pages */ - work->nr_pages = max_t(pgoff_t, work->nr_pages, index + 1); -next_part: - /* can be used for verification */ - map->m_llen = offset + cur - map->m_la; - - end = cur; - if (end > 0) - goto repeat; - -out: - /* FIXME! avoid the last relundant fixup & endio */ - z_erofs_onlinepage_endio(page); - - debugln("%s, finish page: %pK spiltted: %u map->m_llen %llu", - __func__, page, spiltted, map->m_llen); - return err; - - /* if some error occurred while processing this page */ -err_out: - SetPageError(page); - goto out; -} - -static void z_erofs_vle_unzip_kickoff(void *ptr, int bios) -{ - tagptr1_t t = tagptr_init(tagptr1_t, ptr); - struct z_erofs_vle_unzip_io *io = tagptr_unfold_ptr(t); - bool background = tagptr_unfold_tags(t); - - if (!background) { - unsigned long flags; - - spin_lock_irqsave(&io->u.wait.lock, flags); - if (!atomic_add_return(bios, &io->pending_bios)) - wake_up_locked(&io->u.wait); - spin_unlock_irqrestore(&io->u.wait.lock, flags); - return; - } - - if (!atomic_add_return(bios, &io->pending_bios)) - queue_work(z_erofs_workqueue, &io->u.work); -} - -static inline void z_erofs_vle_read_endio(struct bio *bio) -{ - const blk_status_t err = bio->bi_status; - unsigned i; - struct bio_vec *bvec; -#ifdef EROFS_FS_HAS_MANAGED_CACHE - struct address_space *mngda = NULL; -#endif - - bio_for_each_segment_all(bvec, bio, i) { - struct page *page = bvec->bv_page; - bool cachemngd = false; - - DBG_BUGON(PageUptodate(page)); - DBG_BUGON(!page->mapping); - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - if (unlikely(mngda == NULL && !z_erofs_is_stagingpage(page))) { - struct inode *const inode = page->mapping->host; - struct super_block *const sb = inode->i_sb; - - mngda = EROFS_SB(sb)->managed_cache->i_mapping; - } - - /* - * If mngda has not gotten, it equals NULL, - * however, page->mapping never be NULL if working properly. - */ - cachemngd = (page->mapping == mngda); -#endif - - if (unlikely(err)) - SetPageError(page); - else if (cachemngd) - SetPageUptodate(page); - - if (cachemngd) - unlock_page(page); - } - - z_erofs_vle_unzip_kickoff(bio->bi_private, -1); - bio_put(bio); -} - -static struct page *z_pagemap_global[Z_EROFS_VLE_VMAP_GLOBAL_PAGES]; -static DEFINE_MUTEX(z_pagemap_global_lock); - -static int z_erofs_vle_unzip(struct super_block *sb, - struct z_erofs_vle_workgroup *grp, - struct list_head *page_pool) -{ - struct erofs_sb_info *const sbi = EROFS_SB(sb); -#ifdef EROFS_FS_HAS_MANAGED_CACHE - struct address_space *const mngda = sbi->managed_cache->i_mapping; -#endif - const unsigned clusterpages = erofs_clusterpages(sbi); - - struct z_erofs_pagevec_ctor ctor; - unsigned int nr_pages; - unsigned int sparsemem_pages = 0; - struct page *pages_onstack[Z_EROFS_VLE_VMAP_ONSTACK_PAGES]; - struct page **pages, **compressed_pages, *page; - unsigned i, llen; - - enum z_erofs_page_type page_type; - bool overlapped; - struct z_erofs_vle_work *work; - void *vout; - int err; - - might_sleep(); - work = z_erofs_vle_grab_primary_work(grp); - DBG_BUGON(!READ_ONCE(work->nr_pages)); - - mutex_lock(&work->lock); - nr_pages = work->nr_pages; - - if (likely(nr_pages <= Z_EROFS_VLE_VMAP_ONSTACK_PAGES)) - pages = pages_onstack; - else if (nr_pages <= Z_EROFS_VLE_VMAP_GLOBAL_PAGES && - mutex_trylock(&z_pagemap_global_lock)) - pages = z_pagemap_global; - else { -repeat: - pages = kvmalloc_array(nr_pages, - sizeof(struct page *), GFP_KERNEL); - - /* fallback to global pagemap for the lowmem scenario */ - if (unlikely(pages == NULL)) { - if (nr_pages > Z_EROFS_VLE_VMAP_GLOBAL_PAGES) - goto repeat; - else { - mutex_lock(&z_pagemap_global_lock); - pages = z_pagemap_global; - } - } - } - - for (i = 0; i < nr_pages; ++i) - pages[i] = NULL; - - err = 0; - z_erofs_pagevec_ctor_init(&ctor, - Z_EROFS_VLE_INLINE_PAGEVECS, work->pagevec, 0); - - for (i = 0; i < work->vcnt; ++i) { - unsigned pagenr; - - page = z_erofs_pagevec_ctor_dequeue(&ctor, &page_type); - - /* all pages in pagevec ought to be valid */ - DBG_BUGON(page == NULL); - DBG_BUGON(page->mapping == NULL); - - if (z_erofs_gather_if_stagingpage(page_pool, page)) - continue; - - if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD) - pagenr = 0; - else - pagenr = z_erofs_onlinepage_index(page); - - DBG_BUGON(pagenr >= nr_pages); - - /* - * currently EROFS doesn't support multiref(dedup), - * so here erroring out one multiref page. - */ - if (pages[pagenr]) { - DBG_BUGON(1); - SetPageError(pages[pagenr]); - z_erofs_onlinepage_endio(pages[pagenr]); - err = -EIO; - } - pages[pagenr] = page; - } - sparsemem_pages = i; - - z_erofs_pagevec_ctor_exit(&ctor, true); - - overlapped = false; - compressed_pages = grp->compressed_pages; - - for (i = 0; i < clusterpages; ++i) { - unsigned pagenr; - - page = compressed_pages[i]; - - /* all compressed pages ought to be valid */ - DBG_BUGON(page == NULL); - DBG_BUGON(page->mapping == NULL); - - if (!z_erofs_is_stagingpage(page)) { -#ifdef EROFS_FS_HAS_MANAGED_CACHE - if (page->mapping == mngda) { - if (unlikely(!PageUptodate(page))) - err = -EIO; - continue; - } -#endif - - /* - * only if non-head page can be selected - * for inplace decompression - */ - pagenr = z_erofs_onlinepage_index(page); - - DBG_BUGON(pagenr >= nr_pages); - if (pages[pagenr]) { - DBG_BUGON(1); - SetPageError(pages[pagenr]); - z_erofs_onlinepage_endio(pages[pagenr]); - err = -EIO; - } - ++sparsemem_pages; - pages[pagenr] = page; - - overlapped = true; - } - - /* PG_error needs checking for inplaced and staging pages */ - if (unlikely(PageError(page))) { - DBG_BUGON(PageUptodate(page)); - err = -EIO; - } - } - - if (unlikely(err)) - goto out; - - llen = (nr_pages << PAGE_SHIFT) - work->pageofs; - - if (z_erofs_vle_workgrp_fmt(grp) == Z_EROFS_VLE_WORKGRP_FMT_PLAIN) { - err = z_erofs_vle_plain_copy(compressed_pages, clusterpages, - pages, nr_pages, work->pageofs); - goto out; - } - - if (llen > grp->llen) - llen = grp->llen; - - err = z_erofs_vle_unzip_fast_percpu(compressed_pages, clusterpages, - pages, llen, work->pageofs); - if (err != -ENOTSUPP) - goto out; - - if (sparsemem_pages >= nr_pages) - goto skip_allocpage; - - for (i = 0; i < nr_pages; ++i) { - if (pages[i] != NULL) - continue; - - pages[i] = __stagingpage_alloc(page_pool, GFP_NOFS); - } - -skip_allocpage: - vout = erofs_vmap(pages, nr_pages); - if (!vout) { - err = -ENOMEM; - goto out; - } - - err = z_erofs_vle_unzip_vmap(compressed_pages, - clusterpages, vout, llen, work->pageofs, overlapped); - - erofs_vunmap(vout, nr_pages); - -out: - /* must handle all compressed pages before endding pages */ - for (i = 0; i < clusterpages; ++i) { - page = compressed_pages[i]; - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - if (page->mapping == mngda) - continue; -#endif - /* recycle all individual staging pages */ - (void)z_erofs_gather_if_stagingpage(page_pool, page); - - WRITE_ONCE(compressed_pages[i], NULL); - } - - for (i = 0; i < nr_pages; ++i) { - page = pages[i]; - if (!page) - continue; - - DBG_BUGON(page->mapping == NULL); - - /* recycle all individual staging pages */ - if (z_erofs_gather_if_stagingpage(page_pool, page)) - continue; - - if (unlikely(err < 0)) - SetPageError(page); - - z_erofs_onlinepage_endio(page); - } - - if (pages == z_pagemap_global) - mutex_unlock(&z_pagemap_global_lock); - else if (unlikely(pages != pages_onstack)) - kvfree(pages); - - work->nr_pages = 0; - work->vcnt = 0; - - /* all work locks MUST be taken before the following line */ - - WRITE_ONCE(grp->next, Z_EROFS_VLE_WORKGRP_NIL); - - /* all work locks SHOULD be released right now */ - mutex_unlock(&work->lock); - - z_erofs_vle_work_release(work); - return err; -} - -static void z_erofs_vle_unzip_all(struct super_block *sb, - struct z_erofs_vle_unzip_io *io, - struct list_head *page_pool) -{ - z_erofs_vle_owned_workgrp_t owned = io->head; - - while (owned != Z_EROFS_VLE_WORKGRP_TAIL_CLOSED) { - struct z_erofs_vle_workgroup *grp; - - /* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */ - DBG_BUGON(owned == Z_EROFS_VLE_WORKGRP_TAIL); - - /* no possible that 'owned' equals NULL */ - DBG_BUGON(owned == Z_EROFS_VLE_WORKGRP_NIL); - - grp = owned; - owned = READ_ONCE(grp->next); - - z_erofs_vle_unzip(sb, grp, page_pool); - } -} - -static void z_erofs_vle_unzip_wq(struct work_struct *work) -{ - struct z_erofs_vle_unzip_io_sb *iosb = container_of(work, - struct z_erofs_vle_unzip_io_sb, io.u.work); - LIST_HEAD(page_pool); - - DBG_BUGON(iosb->io.head == Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); - z_erofs_vle_unzip_all(iosb->sb, &iosb->io, &page_pool); - - put_pages_list(&page_pool); - kvfree(iosb); -} - -static inline struct z_erofs_vle_unzip_io * -prepare_io_handler(struct super_block *sb, - struct z_erofs_vle_unzip_io *io, - bool background) -{ - struct z_erofs_vle_unzip_io_sb *iosb; - - if (!background) { - /* waitqueue available for foreground io */ - BUG_ON(io == NULL); - - init_waitqueue_head(&io->u.wait); - atomic_set(&io->pending_bios, 0); - goto out; - } - - if (io != NULL) - BUG(); - else { - /* allocate extra io descriptor for background io */ - iosb = kvzalloc(sizeof(struct z_erofs_vle_unzip_io_sb), - GFP_KERNEL | __GFP_NOFAIL); - BUG_ON(iosb == NULL); - - io = &iosb->io; - } - - iosb->sb = sb; - INIT_WORK(&io->u.work, z_erofs_vle_unzip_wq); -out: - io->head = Z_EROFS_VLE_WORKGRP_TAIL_CLOSED; - return io; -} - -#ifdef EROFS_FS_HAS_MANAGED_CACHE -/* true - unlocked (noio), false - locked (need submit io) */ -static inline bool recover_managed_page(struct z_erofs_vle_workgroup *grp, - struct page *page) -{ - wait_on_page_locked(page); - if (PagePrivate(page) && PageUptodate(page)) - return true; - - lock_page(page); - ClearPageError(page); - - if (unlikely(!PagePrivate(page))) { - set_page_private(page, (unsigned long)grp); - SetPagePrivate(page); - } - if (unlikely(PageUptodate(page))) { - unlock_page(page); - return true; - } - return false; -} - -#define __FSIO_1 1 -#else -#define __FSIO_1 0 -#endif - -static bool z_erofs_vle_submit_all(struct super_block *sb, - z_erofs_vle_owned_workgrp_t owned_head, - struct list_head *pagepool, - struct z_erofs_vle_unzip_io *fg_io, - bool force_fg) -{ - struct erofs_sb_info *const sbi = EROFS_SB(sb); - const unsigned clusterpages = erofs_clusterpages(sbi); - const gfp_t gfp = GFP_NOFS; -#ifdef EROFS_FS_HAS_MANAGED_CACHE - struct address_space *const mngda = sbi->managed_cache->i_mapping; - struct z_erofs_vle_workgroup *lstgrp_noio = NULL, *lstgrp_io = NULL; -#endif - struct z_erofs_vle_unzip_io *ios[1 + __FSIO_1]; - struct bio *bio; - tagptr1_t bi_private; - /* since bio will be NULL, no need to initialize last_index */ - pgoff_t uninitialized_var(last_index); - bool force_submit = false; - unsigned nr_bios; - - if (unlikely(owned_head == Z_EROFS_VLE_WORKGRP_TAIL)) - return false; - - /* - * force_fg == 1, (io, fg_io[0]) no io, (io, fg_io[1]) need submit io - * force_fg == 0, (io, fg_io[0]) no io; (io[1], bg_io) need submit io - */ -#ifdef EROFS_FS_HAS_MANAGED_CACHE - ios[0] = prepare_io_handler(sb, fg_io + 0, false); -#endif - - if (force_fg) { - ios[__FSIO_1] = prepare_io_handler(sb, fg_io + __FSIO_1, false); - bi_private = tagptr_fold(tagptr1_t, ios[__FSIO_1], 0); - } else { - ios[__FSIO_1] = prepare_io_handler(sb, NULL, true); - bi_private = tagptr_fold(tagptr1_t, ios[__FSIO_1], 1); - } - - nr_bios = 0; - force_submit = false; - bio = NULL; - - /* by default, all need io submission */ - ios[__FSIO_1]->head = owned_head; - - do { - struct z_erofs_vle_workgroup *grp; - struct page **compressed_pages, *oldpage, *page; - pgoff_t first_index; - unsigned i = 0; -#ifdef EROFS_FS_HAS_MANAGED_CACHE - unsigned int noio = 0; - bool cachemngd; -#endif - int err; - - /* no possible 'owned_head' equals the following */ - DBG_BUGON(owned_head == Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); - DBG_BUGON(owned_head == Z_EROFS_VLE_WORKGRP_NIL); - - grp = owned_head; - - /* close the main owned chain at first */ - owned_head = cmpxchg(&grp->next, Z_EROFS_VLE_WORKGRP_TAIL, - Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); - - first_index = grp->obj.index; - compressed_pages = grp->compressed_pages; - - force_submit |= (first_index != last_index + 1); -repeat: - /* fulfill all compressed pages */ - oldpage = page = READ_ONCE(compressed_pages[i]); - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - cachemngd = false; - - if (page == EROFS_UNALLOCATED_CACHED_PAGE) { - cachemngd = true; - goto do_allocpage; - } else if (page != NULL) { - if (page->mapping != mngda) - BUG_ON(PageUptodate(page)); - else if (recover_managed_page(grp, page)) { - /* page is uptodate, skip io submission */ - force_submit = true; - ++noio; - goto skippage; - } - } else { -do_allocpage: -#else - if (page != NULL) - BUG_ON(PageUptodate(page)); - else { -#endif - page = __stagingpage_alloc(pagepool, gfp); - - if (oldpage != cmpxchg(compressed_pages + i, - oldpage, page)) { - list_add(&page->lru, pagepool); - goto repeat; -#ifdef EROFS_FS_HAS_MANAGED_CACHE - } else if (cachemngd && !add_to_page_cache_lru(page, - mngda, first_index + i, gfp)) { - set_page_private(page, (unsigned long)grp); - SetPagePrivate(page); -#endif - } - } - - if (bio != NULL && force_submit) { -submit_bio_retry: - __submit_bio(bio, REQ_OP_READ, 0); - bio = NULL; - } - - if (bio == NULL) { - bio = prepare_bio(sb, first_index + i, - BIO_MAX_PAGES, z_erofs_vle_read_endio); - bio->bi_private = tagptr_cast_ptr(bi_private); - - ++nr_bios; - } - - err = bio_add_page(bio, page, PAGE_SIZE, 0); - if (err < PAGE_SIZE) - goto submit_bio_retry; - - force_submit = false; - last_index = first_index + i; -#ifdef EROFS_FS_HAS_MANAGED_CACHE -skippage: -#endif - if (++i < clusterpages) - goto repeat; - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - if (noio < clusterpages) { - lstgrp_io = grp; - } else { - z_erofs_vle_owned_workgrp_t iogrp_next = - owned_head == Z_EROFS_VLE_WORKGRP_TAIL ? - Z_EROFS_VLE_WORKGRP_TAIL_CLOSED : - owned_head; - - if (lstgrp_io == NULL) - ios[1]->head = iogrp_next; - else - WRITE_ONCE(lstgrp_io->next, iogrp_next); - - if (lstgrp_noio == NULL) - ios[0]->head = grp; - else - WRITE_ONCE(lstgrp_noio->next, grp); - - lstgrp_noio = grp; - } -#endif - } while (owned_head != Z_EROFS_VLE_WORKGRP_TAIL); - - if (bio != NULL) - __submit_bio(bio, REQ_OP_READ, 0); - -#ifndef EROFS_FS_HAS_MANAGED_CACHE - BUG_ON(!nr_bios); -#else - if (lstgrp_noio != NULL) - WRITE_ONCE(lstgrp_noio->next, Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); - - if (!force_fg && !nr_bios) { - kvfree(container_of(ios[1], - struct z_erofs_vle_unzip_io_sb, io)); - return true; - } -#endif - - z_erofs_vle_unzip_kickoff(tagptr_cast_ptr(bi_private), nr_bios); - return true; -} - -static void z_erofs_submit_and_unzip(struct z_erofs_vle_frontend *f, - struct list_head *pagepool, - bool force_fg) -{ - struct super_block *sb = f->inode->i_sb; - struct z_erofs_vle_unzip_io io[1 + __FSIO_1]; - - if (!z_erofs_vle_submit_all(sb, f->owned_head, pagepool, io, force_fg)) - return; - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - z_erofs_vle_unzip_all(sb, &io[0], pagepool); -#endif - if (!force_fg) - return; - - /* wait until all bios are completed */ - wait_event(io[__FSIO_1].u.wait, - !atomic_read(&io[__FSIO_1].pending_bios)); - - /* let's synchronous decompression */ - z_erofs_vle_unzip_all(sb, &io[__FSIO_1], pagepool); -} - -static int z_erofs_vle_normalaccess_readpage(struct file *file, - struct page *page) -{ - struct inode *const inode = page->mapping->host; - struct z_erofs_vle_frontend f = VLE_FRONTEND_INIT(inode); - int err; - LIST_HEAD(pagepool); - -#if (EROFS_FS_ZIP_CACHE_LVL >= 2) - f.cachedzone_la = page->index << PAGE_SHIFT; -#endif - err = z_erofs_do_read_page(&f, page, &pagepool); - (void)z_erofs_vle_work_iter_end(&f.builder); - - /* if some compressed cluster ready, need submit them anyway */ - z_erofs_submit_and_unzip(&f, &pagepool, true); - - if (err) - errln("%s, failed to read, err [%d]", __func__, err); - - if (f.m_iter.mpage != NULL) - put_page(f.m_iter.mpage); - - /* clean up the remaining free pages */ - put_pages_list(&pagepool); - return err; -} - -static inline int __z_erofs_vle_normalaccess_readpages( - struct file *filp, - struct address_space *mapping, - struct list_head *pages, unsigned nr_pages, bool sync) -{ - struct inode *const inode = mapping->host; - - struct z_erofs_vle_frontend f = VLE_FRONTEND_INIT(inode); - gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL); - struct page *head = NULL; - LIST_HEAD(pagepool); - -#if (EROFS_FS_ZIP_CACHE_LVL >= 2) - f.cachedzone_la = lru_to_page(pages)->index << PAGE_SHIFT; -#endif - for (; nr_pages; --nr_pages) { - struct page *page = lru_to_page(pages); - - prefetchw(&page->flags); - list_del(&page->lru); - - if (add_to_page_cache_lru(page, mapping, page->index, gfp)) { - list_add(&page->lru, &pagepool); - continue; - } - - set_page_private(page, (unsigned long)head); - head = page; - } - - while (head != NULL) { - struct page *page = head; - int err; - - /* traversal in reverse order */ - head = (void *)page_private(page); - - err = z_erofs_do_read_page(&f, page, &pagepool); - if (err) { - struct erofs_vnode *vi = EROFS_V(inode); - - errln("%s, readahead error at page %lu of nid %llu", - __func__, page->index, vi->nid); - } - - put_page(page); - } - - (void)z_erofs_vle_work_iter_end(&f.builder); - - z_erofs_submit_and_unzip(&f, &pagepool, sync); - - if (f.m_iter.mpage != NULL) - put_page(f.m_iter.mpage); - - /* clean up the remaining free pages */ - put_pages_list(&pagepool); - return 0; -} - -static int z_erofs_vle_normalaccess_readpages( - struct file *filp, - struct address_space *mapping, - struct list_head *pages, unsigned nr_pages) -{ - return __z_erofs_vle_normalaccess_readpages(filp, - mapping, pages, nr_pages, - nr_pages < 4 /* sync */); -} - -const struct address_space_operations z_erofs_vle_normalaccess_aops = { - .readpage = z_erofs_vle_normalaccess_readpage, - .readpages = z_erofs_vle_normalaccess_readpages, -}; - -#define __vle_cluster_advise(x, bit, bits) \ - ((le16_to_cpu(x) >> (bit)) & ((1 << (bits)) - 1)) - -#define __vle_cluster_type(advise) __vle_cluster_advise(advise, \ - Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT, Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - -enum { - Z_EROFS_VLE_CLUSTER_TYPE_PLAIN, - Z_EROFS_VLE_CLUSTER_TYPE_HEAD, - Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD, - Z_EROFS_VLE_CLUSTER_TYPE_RESERVED, - Z_EROFS_VLE_CLUSTER_TYPE_MAX -}; - -#define vle_cluster_type(di) \ - __vle_cluster_type((di)->di_advise) - -static inline unsigned -vle_compressed_index_clusterofs(unsigned clustersize, - struct z_erofs_vle_decompressed_index *di) -{ - debugln("%s, vle=%pK, advise=%x (type %u), clusterofs=%x blkaddr=%x", - __func__, di, di->di_advise, vle_cluster_type(di), - di->di_clusterofs, di->di_u.blkaddr); - - switch (vle_cluster_type(di)) { - case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: - break; - case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: - case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: - return di->di_clusterofs; - default: - BUG_ON(1); - } - return clustersize; -} - -static inline erofs_blk_t -vle_extent_blkaddr(struct inode *inode, pgoff_t index) -{ - struct erofs_sb_info *sbi = EROFS_I_SB(inode); - struct erofs_vnode *vi = EROFS_V(inode); - - unsigned ofs = Z_EROFS_VLE_EXTENT_ALIGN(vi->inode_isize + - vi->xattr_isize) + sizeof(struct erofs_extent_header) + - index * sizeof(struct z_erofs_vle_decompressed_index); - - return erofs_blknr(iloc(sbi, vi->nid) + ofs); -} - -static inline unsigned int -vle_extent_blkoff(struct inode *inode, pgoff_t index) -{ - struct erofs_sb_info *sbi = EROFS_I_SB(inode); - struct erofs_vnode *vi = EROFS_V(inode); - - unsigned ofs = Z_EROFS_VLE_EXTENT_ALIGN(vi->inode_isize + - vi->xattr_isize) + sizeof(struct erofs_extent_header) + - index * sizeof(struct z_erofs_vle_decompressed_index); - - return erofs_blkoff(iloc(sbi, vi->nid) + ofs); -} - -/* - * Variable-sized Logical Extent (Fixed Physical Cluster) Compression Mode - * --- - * VLE compression mode attempts to compress a number of logical data into - * a physical cluster with a fixed size. - * VLE compression mode uses "struct z_erofs_vle_decompressed_index". - */ -static erofs_off_t vle_get_logical_extent_head( - struct inode *inode, - struct page **page_iter, - void **kaddr_iter, - unsigned lcn, /* logical cluster number */ - erofs_blk_t *pcn, - unsigned *flags) -{ - /* for extent meta */ - struct page *page = *page_iter; - erofs_blk_t blkaddr = vle_extent_blkaddr(inode, lcn); - struct z_erofs_vle_decompressed_index *di; - unsigned long long ofs; - const unsigned int clusterbits = EROFS_SB(inode->i_sb)->clusterbits; - const unsigned int clustersize = 1 << clusterbits; - unsigned int delta0; - - if (page->index != blkaddr) { - kunmap_atomic(*kaddr_iter); - unlock_page(page); - put_page(page); - - *page_iter = page = erofs_get_meta_page(inode->i_sb, - blkaddr, false); - *kaddr_iter = kmap_atomic(page); - } - - di = *kaddr_iter + vle_extent_blkoff(inode, lcn); - switch (vle_cluster_type(di)) { - case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: - delta0 = le16_to_cpu(di->di_u.delta[0]); - DBG_BUGON(!delta0); - DBG_BUGON(lcn < delta0); - - ofs = vle_get_logical_extent_head(inode, - page_iter, kaddr_iter, - lcn - delta0, pcn, flags); - break; - case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: - *flags ^= EROFS_MAP_ZIPPED; - case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: - /* clustersize should be a power of two */ - ofs = ((unsigned long long)lcn << clusterbits) + - (le16_to_cpu(di->di_clusterofs) & (clustersize - 1)); - *pcn = le32_to_cpu(di->di_u.blkaddr); - break; - default: - BUG_ON(1); - } - return ofs; -} - -int z_erofs_map_blocks_iter(struct inode *inode, - struct erofs_map_blocks *map, - struct page **mpage_ret, int flags) -{ - /* logicial extent (start, end) offset */ - unsigned long long ofs, end; - struct z_erofs_vle_decompressed_index *di; - erofs_blk_t e_blkaddr, pcn; - unsigned lcn, logical_cluster_ofs, cluster_type; - u32 ofs_rem; - struct page *mpage = *mpage_ret; - void *kaddr; - bool initial; - const unsigned int clusterbits = EROFS_SB(inode->i_sb)->clusterbits; - const unsigned int clustersize = 1 << clusterbits; - int err = 0; - - /* if both m_(l,p)len are 0, regularize l_lblk, l_lofs, etc... */ - initial = !map->m_llen; - - /* when trying to read beyond EOF, leave it unmapped */ - if (unlikely(map->m_la >= inode->i_size)) { - BUG_ON(!initial); - map->m_llen = map->m_la + 1 - inode->i_size; - map->m_la = inode->i_size - 1; - map->m_flags = 0; - goto out; - } - - debugln("%s, m_la %llu m_llen %llu --- start", __func__, - map->m_la, map->m_llen); - - ofs = map->m_la + map->m_llen; - - /* clustersize should be power of two */ - lcn = ofs >> clusterbits; - ofs_rem = ofs & (clustersize - 1); - - e_blkaddr = vle_extent_blkaddr(inode, lcn); - - if (mpage == NULL || mpage->index != e_blkaddr) { - if (mpage != NULL) - put_page(mpage); - - mpage = erofs_get_meta_page(inode->i_sb, e_blkaddr, false); - *mpage_ret = mpage; - } else { - lock_page(mpage); - DBG_BUGON(!PageUptodate(mpage)); - } - - kaddr = kmap_atomic(mpage); - di = kaddr + vle_extent_blkoff(inode, lcn); - - debugln("%s, lcn %u e_blkaddr %u e_blkoff %u", __func__, lcn, - e_blkaddr, vle_extent_blkoff(inode, lcn)); - - logical_cluster_ofs = vle_compressed_index_clusterofs(clustersize, di); - if (!initial) { - /* [walking mode] 'map' has been already initialized */ - map->m_llen += logical_cluster_ofs; - goto unmap_out; - } - - /* by default, compressed */ - map->m_flags |= EROFS_MAP_ZIPPED; - - end = (u64)(lcn + 1) * clustersize; - - cluster_type = vle_cluster_type(di); - - switch (cluster_type) { - case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: - if (ofs_rem >= logical_cluster_ofs) - map->m_flags ^= EROFS_MAP_ZIPPED; - /* fallthrough */ - case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: - if (ofs_rem == logical_cluster_ofs) { - pcn = le32_to_cpu(di->di_u.blkaddr); - goto exact_hitted; - } - - if (ofs_rem > logical_cluster_ofs) { - ofs = lcn * clustersize | logical_cluster_ofs; - pcn = le32_to_cpu(di->di_u.blkaddr); - break; - } - - /* logical cluster number should be >= 1 */ - if (unlikely(!lcn)) { - errln("invalid logical cluster 0 at nid %llu", - EROFS_V(inode)->nid); - err = -EIO; - goto unmap_out; - } - end = (lcn-- * clustersize) | logical_cluster_ofs; - /* fallthrough */ - case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: - /* get the correspoinding first chunk */ - ofs = vle_get_logical_extent_head(inode, mpage_ret, - &kaddr, lcn, &pcn, &map->m_flags); - mpage = *mpage_ret; - break; - default: - errln("unknown cluster type %u at offset %llu of nid %llu", - cluster_type, ofs, EROFS_V(inode)->nid); - err = -EIO; - goto unmap_out; - } - - map->m_la = ofs; -exact_hitted: - map->m_llen = end - ofs; - map->m_plen = clustersize; - map->m_pa = blknr_to_addr(pcn); - map->m_flags |= EROFS_MAP_MAPPED; -unmap_out: - kunmap_atomic(kaddr); - unlock_page(mpage); -out: - debugln("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", - __func__, map->m_la, map->m_pa, - map->m_llen, map->m_plen, map->m_flags); - - /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ - DBG_BUGON(err < 0); - return err; -} - diff --git a/drivers/staging/erofs/unzip_vle_lz4.c b/drivers/staging/erofs/unzip_vle_lz4.c deleted file mode 100644 index 3a7428317f0a..000000000000 --- a/drivers/staging/erofs/unzip_vle_lz4.c +++ /dev/null @@ -1,210 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/drivers/staging/erofs/unzip_vle_lz4.c - * - * Copyright (C) 2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ -#include "unzip_vle.h" - -#if Z_EROFS_CLUSTER_MAX_PAGES > Z_EROFS_VLE_INLINE_PAGEVECS -#define EROFS_PERCPU_NR_PAGES Z_EROFS_CLUSTER_MAX_PAGES -#else -#define EROFS_PERCPU_NR_PAGES Z_EROFS_VLE_INLINE_PAGEVECS -#endif - -static struct { - char data[PAGE_SIZE * EROFS_PERCPU_NR_PAGES]; -} erofs_pcpubuf[NR_CPUS]; - -int z_erofs_vle_plain_copy(struct page **compressed_pages, - unsigned clusterpages, - struct page **pages, - unsigned nr_pages, - unsigned short pageofs) -{ - unsigned i, j; - void *src = NULL; - const unsigned righthalf = PAGE_SIZE - pageofs; - char *percpu_data; - bool mirrored[Z_EROFS_CLUSTER_MAX_PAGES] = { 0 }; - - preempt_disable(); - percpu_data = erofs_pcpubuf[smp_processor_id()].data; - - j = 0; - for (i = 0; i < nr_pages; j = i++) { - struct page *page = pages[i]; - void *dst; - - if (page == NULL) { - if (src != NULL) { - if (!mirrored[j]) - kunmap_atomic(src); - src = NULL; - } - continue; - } - - dst = kmap_atomic(page); - - for (; j < clusterpages; ++j) { - if (compressed_pages[j] != page) - continue; - - DBG_BUGON(mirrored[j]); - memcpy(percpu_data + j * PAGE_SIZE, dst, PAGE_SIZE); - mirrored[j] = true; - break; - } - - if (i) { - if (src == NULL) - src = mirrored[i-1] ? - percpu_data + (i-1) * PAGE_SIZE : - kmap_atomic(compressed_pages[i-1]); - - memcpy(dst, src + righthalf, pageofs); - - if (!mirrored[i-1]) - kunmap_atomic(src); - - if (unlikely(i >= clusterpages)) { - kunmap_atomic(dst); - break; - } - } - - if (!righthalf) - src = NULL; - else { - src = mirrored[i] ? percpu_data + i * PAGE_SIZE : - kmap_atomic(compressed_pages[i]); - - memcpy(dst + pageofs, src, righthalf); - } - - kunmap_atomic(dst); - } - - if (src != NULL && !mirrored[j]) - kunmap_atomic(src); - - preempt_enable(); - return 0; -} - -extern int z_erofs_unzip_lz4(void *in, void *out, size_t inlen, size_t outlen); - -int z_erofs_vle_unzip_fast_percpu(struct page **compressed_pages, - unsigned clusterpages, - struct page **pages, - unsigned outlen, - unsigned short pageofs) -{ - void *vin, *vout; - unsigned nr_pages, i, j; - int ret; - - if (outlen + pageofs > EROFS_PERCPU_NR_PAGES * PAGE_SIZE) - return -ENOTSUPP; - - nr_pages = DIV_ROUND_UP(outlen + pageofs, PAGE_SIZE); - - if (clusterpages == 1) { - vin = kmap_atomic(compressed_pages[0]); - } else { - vin = erofs_vmap(compressed_pages, clusterpages); - if (!vin) - return -ENOMEM; - } - - preempt_disable(); - vout = erofs_pcpubuf[smp_processor_id()].data; - - ret = z_erofs_unzip_lz4(vin, vout + pageofs, - clusterpages * PAGE_SIZE, outlen); - - if (ret < 0) - goto out; - ret = 0; - - for (i = 0; i < nr_pages; ++i) { - j = min((unsigned)PAGE_SIZE - pageofs, outlen); - - if (pages[i] != NULL) { - if (clusterpages == 1 && - pages[i] == compressed_pages[0]) { - memcpy(vin + pageofs, vout + pageofs, j); - } else { - void *dst = kmap_atomic(pages[i]); - - memcpy(dst + pageofs, vout + pageofs, j); - kunmap_atomic(dst); - } - } - vout += PAGE_SIZE; - outlen -= j; - pageofs = 0; - } - -out: - preempt_enable(); - - if (clusterpages == 1) - kunmap_atomic(vin); - else - erofs_vunmap(vin, clusterpages); - - return ret; -} - -int z_erofs_vle_unzip_vmap(struct page **compressed_pages, - unsigned clusterpages, - void *vout, - unsigned llen, - unsigned short pageofs, - bool overlapped) -{ - void *vin; - unsigned i; - int ret; - - if (overlapped) { - preempt_disable(); - vin = erofs_pcpubuf[smp_processor_id()].data; - - for (i = 0; i < clusterpages; ++i) { - void *t = kmap_atomic(compressed_pages[i]); - - memcpy(vin + PAGE_SIZE *i, t, PAGE_SIZE); - kunmap_atomic(t); - } - } else if (clusterpages == 1) - vin = kmap_atomic(compressed_pages[0]); - else { - vin = erofs_vmap(compressed_pages, clusterpages); - } - - ret = z_erofs_unzip_lz4(vin, vout + pageofs, - clusterpages * PAGE_SIZE, llen); - if (ret > 0) - ret = 0; - - if (!overlapped) { - if (clusterpages == 1) - kunmap_atomic(vin); - else { - erofs_vunmap(vin, clusterpages); - } - } else - preempt_enable(); - - return ret; -} - diff --git a/drivers/staging/erofs/xattr.h b/drivers/staging/erofs/xattr.h deleted file mode 100644 index 0c7379282fc5..000000000000 --- a/drivers/staging/erofs/xattr.h +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * linux/drivers/staging/erofs/xattr.h - * - * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ -#ifndef __EROFS_XATTR_H -#define __EROFS_XATTR_H - -#include "internal.h" -#include -#include - -/* Attribute not found */ -#define ENOATTR ENODATA - -static inline unsigned inlinexattr_header_size(struct inode *inode) -{ - return sizeof(struct erofs_xattr_ibody_header) - + sizeof(u32) * EROFS_V(inode)->xattr_shared_count; -} - -static inline erofs_blk_t -xattrblock_addr(struct erofs_sb_info *sbi, unsigned xattr_id) -{ -#ifdef CONFIG_EROFS_FS_XATTR - return sbi->xattr_blkaddr + - xattr_id * sizeof(__u32) / EROFS_BLKSIZ; -#else - return 0; -#endif -} - -static inline unsigned -xattrblock_offset(struct erofs_sb_info *sbi, unsigned xattr_id) -{ - return (xattr_id * sizeof(__u32)) % EROFS_BLKSIZ; -} - -extern const struct xattr_handler erofs_xattr_user_handler; -extern const struct xattr_handler erofs_xattr_trusted_handler; -#ifdef CONFIG_EROFS_FS_SECURITY -extern const struct xattr_handler erofs_xattr_security_handler; -#endif - -static inline const struct xattr_handler *erofs_xattr_handler(unsigned index) -{ -static const struct xattr_handler *xattr_handler_map[] = { - [EROFS_XATTR_INDEX_USER] = &erofs_xattr_user_handler, -#ifdef CONFIG_EROFS_FS_POSIX_ACL - [EROFS_XATTR_INDEX_POSIX_ACL_ACCESS] = &posix_acl_access_xattr_handler, - [EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT] = - &posix_acl_default_xattr_handler, -#endif - [EROFS_XATTR_INDEX_TRUSTED] = &erofs_xattr_trusted_handler, -#ifdef CONFIG_EROFS_FS_SECURITY - [EROFS_XATTR_INDEX_SECURITY] = &erofs_xattr_security_handler, -#endif -}; - return index && index < ARRAY_SIZE(xattr_handler_map) ? - xattr_handler_map[index] : NULL; -} - -#ifdef CONFIG_EROFS_FS_XATTR - -extern const struct inode_operations erofs_generic_xattr_iops; -extern const struct inode_operations erofs_dir_xattr_iops; - -int erofs_getxattr(struct inode *, int, const char *, void *, size_t); -ssize_t erofs_listxattr(struct dentry *, char *, size_t); -#else -static int __maybe_unused erofs_getxattr(struct inode *inode, int index, - const char *name, - void *buffer, size_t buffer_size) -{ - return -ENOTSUPP; -} - -static ssize_t __maybe_unused erofs_listxattr(struct dentry *dentry, - char *buffer, size_t buffer_size) -{ - return -ENOTSUPP; -} -#endif - -#endif - diff --git a/drivers/staging/fw-api/fw/htt.h b/drivers/staging/fw-api/fw/htt.h index 665ce2acfc02..4b103f1e7a2f 100644 --- a/drivers/staging/fw-api/fw/htt.h +++ b/drivers/staging/fw-api/fw/htt.h @@ -230,9 +230,12 @@ * 3.103 Add HTT_T2H_SAWF_MSDUQ_INFO_IND defs. * 3.104 Add mgmt/ctrl/data specs in rx ring cfg. * 3.105 Add HTT_H2T STREAMING_STATS_REQ + HTT_T2H STREAMING_STATS_IND defs. + * 3.106 Add HTT_T2H_PPDU_ID_FMT_IND def. + * 3.107 Add traffic_end_indication bitfield in htt_tx_msdu_desc_ext2_t. + * 3.108 Add HTT_H2T_MSG_TYPE_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP def. */ #define HTT_CURRENT_VERSION_MAJOR 3 -#define HTT_CURRENT_VERSION_MINOR 105 +#define HTT_CURRENT_VERSION_MINOR 108 #define HTT_NUM_TX_FRAG_DESC 1024 @@ -741,6 +744,8 @@ typedef enum { HTT_STATS_TXBF_OFDMA_BE_BRP_STATS_TAG = 153, /* htt_txbf_ofdma_be_brp_stats_tlv */ HTT_STATS_TXBF_OFDMA_BE_STEER_STATS_TAG = 154, /* htt_txbf_ofdma_be_steer_stats_tlv */ HTT_STATS_DMAC_RESET_STATS_TAG = 155, /* htt_dmac_reset_stats_tlv */ + HTT_STATS_RX_PDEV_BE_UL_OFDMA_USER_STATS_TAG = 156, /* htt_rx_pdev_be_ul_ofdma_user_stats_tlv */ + HTT_STATS_PHY_TPC_STATS_TAG = 157, /* htt_phy_tpc_stats_tlv */ HTT_STATS_MAX_TAG, @@ -809,6 +814,7 @@ enum htt_h2t_msg_type { HTT_H2T_SAWF_DEF_QUEUES_MAP_REPORT_REQ = 0x1e, HTT_H2T_MSG_TYPE_MSI_SETUP = 0x1f, HTT_H2T_MSG_TYPE_STREAMING_STATS_REQ = 0x20, + HTT_H2T_MSG_TYPE_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP = 0x21, /* keep this last */ HTT_H2T_NUM_MSGS @@ -1971,7 +1977,8 @@ PREPACK struct htt_tx_msdu_desc_ext2_t { * with valid information. */ is_host_opaque_valid : 1, - rsvd0 : 29; + traffic_end_indication: 1, + rsvd0 : 28; /* DWORD 6 : Host opaque cookie for special frames */ A_UINT32 host_opaque_cookie : 16, /* see is_host_opaque_valid */ @@ -9186,7 +9193,10 @@ PREPACK struct htt_h2t_host_paddr_size_entry_t { A_UINT32 physical_address_hi; } POSTPACK; -#define HTT_H2T_HOST_PADDR_SIZE_ENTRY_SIZE (sizeof(struct htt_h2t_host_paddr_size_entry_t)) +#define HTT_H2T_HOST_PADDR_SIZE_ENTRY_SIZE \ + (sizeof(struct htt_h2t_host_paddr_size_entry_t)) +#define HTT_H2T_HOST_PADDR_SIZE_ENTRY_DWORDS \ + (HTT_H2T_HOST_PADDR_SIZE_ENTRY_SIZE >> 2) #define HTT_H2T_HOST_PADDR_SIZE_NUM_ENTRIES_M 0x0000FF00 #define HTT_H2T_HOST_PADDR_SIZE_NUM_ENTRIES_S 8 @@ -9600,6 +9610,237 @@ PREPACK struct htt_h2t_sawf_def_queues_map_report_req { ((word1) |= ((_val) << HTT_H2T_SAWF_DEF_QUEUES_MAP_REPORT_REQ_EXISTING_TIDS_ONLY_S)); \ } while (0) +/** + * @brief Format of shared memory between Host and Target + * for UMAC hang recovery feature messaging. + * @details + * This is shared memory between Host and Target allocated + * and used in chips where UMAC hang recovery feature is supported. + * If target sets a bit in t2h_msg (provided it's valid bit offset) + * then host interprets it as a new message from target. + * Host clears that particular read bit in t2h_msg after each read + * operation. It is vice versa for h2t_msg. At any given point + * of time there is expected to be only one bit set + * either in t2h_msg or h2t_msg (referring to valid bit offset). + * + * The message is interpreted as follows: + * dword0 - b'0:31 - magic_num: Magic number for the shared memory region + * added for debuggability purpose. + * dword1 - b'0 - do_pre_reset + * b'1 - do_post_reset_start + * b'2 - do_post_reset_complete + * b'3:31 - rsvd_t2h + * dword2 - b'0 - pre_reset_done + * b'1 - post_reset_start_done + * b'2 - post_reset_complete_done + * b'3:31 - rsvd_h2t + */ +PREPACK typedef struct { + /** Magic number added for debuggability. */ + A_UINT32 magic_num; + union { + /* + * BIT [0] :- T2H msg to do pre-reset + * BIT [1] :- T2H msg to do post-reset start + * BIT [2] :- T2H msg to do post-reset complete + * BIT [31 : 3] :- reserved + */ + A_UINT32 t2h_msg; + struct { + A_UINT32 do_pre_reset : 1, /* BIT [0] */ + do_post_reset_start : 1, /* BIT [1] */ + do_post_reset_complete : 1, /* BIT [2] */ + rsvd_t2h : 29; /* BIT [31 : 3] */ + }; + }; + + union { + /* + * BIT [0] :- H2T msg to send pre-reset done + * BIT [1] :- H2T msg to send post-reset start done + * BIT [2] :- H2T msg to send post-reset complete done + * BIT [31 : 3] :- reserved + */ + A_UINT32 h2t_msg; + struct { + A_UINT32 pre_reset_done : 1, /* BIT [0] */ + post_reset_start_done : 1, /* BIT [1] */ + post_reset_complete_done : 1, /* BIT [2] */ + rsvd_h2t : 29; /* BIT [31 : 3] */ + }; + }; +} POSTPACK htt_umac_hang_recovery_msg_shmem_t; + +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_BYTES \ + (sizeof(htt_umac_hang_recovery_msg_shmem_t)) +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DWORDS \ + (HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_BYTES >> 2) + +/* dword1 - b'0 - do_pre_reset */ +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET_M 0x00000001 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET_S 0 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET_GET(word1) \ + (((word1) & HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET_M) >> \ + HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET_S) +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET_SET(word1, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET, _val); \ + ((word1) |= ((_val) << HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_PRE_RESET_S));\ + } while (0) + +/* dword1 - b'1 - do_post_reset_start */ +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START_M 0x00000002 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START_S 1 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START_GET(word1) \ + (((word1) & HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START_M) >> \ + HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START_S) +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START_SET(word1, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START, _val); \ + ((word1) |= ((_val) << HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_START_S));\ + } while (0) + +/* dword1 - b'2 - do_post_reset_complete */ +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE_M 0x00000004 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE_S 2 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE_GET(word1) \ + (((word1) & HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE_M) >> \ + HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE_S) +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE_SET(word1, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE, _val); \ + ((word1) |= ((_val) << HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_DO_POST_RESET_COMPLETE_S));\ + } while (0) + +/* dword2 - b'0 - pre_reset_done */ +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE_M 0x00000001 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE_S 0 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE_GET(word2) \ + (((word2) & HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE_M) >> \ + HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE_S) +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE_SET(word2, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE, _val); \ + ((word2) |= ((_val) << HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_PRE_RESET_DONE_S));\ + } while (0) + +/* dword2 - b'1 - post_reset_start_done */ +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE_M 0x00000002 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE_S 1 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE_GET(word2) \ + (((word2) & HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE_M) >> \ + HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE_S) +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE_SET(word2, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE, _val); \ + ((word2) |= ((_val) << HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_START_DONE_S));\ + } while (0) + +/* dword2 - b'2 - post_reset_complete_done */ +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE_M 0x00000004 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE_S 2 +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE_GET(word2) \ + (((word2) & HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE_M) >> \ + HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE_S) +#define HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE_SET(word2, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE, _val); \ + ((word2) |= ((_val) << HTT_UMAC_HANG_RECOVERY_MSG_SHMEM_POST_RESET_COMPLETE_DONE_S));\ + } while (0) + +/** + * @brief HTT_H2T_MSG_TYPE_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP message + * + * @details + * The HTT_H2T_MSG_TYPE_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP message is sent + * by the host to provide prerequisite info to target for the UMAC hang + * recovery feature. + * The info sent in this H2T message are T2H message method, H2T message + * method, T2H MSI interrupt number and physical start address, size of + * the shared memory (refers to the shared memory dedicated for messaging + * between host and target when the DUT is in UMAC hang recovery mode). + * This H2T message is expected to be only sent if the WMI service bit + * WMI_SERVICE_UMAC_HANG_RECOVERY_SUPPORT was firstly indicated by the target. + * + * |31 16|15 12|11 8|7 0| + * |-------------------------------+--------------+--------------+------------| + * | reserved |h2t msg method|t2h msg method| msg_type | + * |--------------------------------------------------------------------------| + * | t2h msi interrupt number | + * |--------------------------------------------------------------------------| + * | shared memory area size | + * |--------------------------------------------------------------------------| + * | shared memory area physical address low | + * |--------------------------------------------------------------------------| + * | shared memory area physical address high | + * |--------------------------------------------------------------------------| + * + * The message is interpreted as follows: + * dword0 - b'0:7 - msg_type (= HTT_H2T_MSG_TYPE_UMAC_HANG_RECOVERY_SETUP) + * b'8:11 - t2h_msg_method: indicates method to be used for + * T2H communication in UMAC hang recovery mode. + * Value zero indicates MSI interrupt (default method). + * Refer to htt_umac_hang_recovery_msg_method enum. + * b'12:15 - h2t_msg_method: indicates method to be used for + * H2T communication in UMAC hang recovery mode. + * Value zero indicates polling by target for this h2t msg + * during UMAC hang recovery mode. + * Refer to htt_umac_hang_recovery_msg_method enum. + * b'16:31 - reserved. + * dword1 - b'0:31 - t2h_msi_data: MSI data to be used for + * T2H communication in UMAC hang recovery mode. + * dword2 - b'0:31 - size: size of shared memory dedicated for messaging + * only when in UMAC hang recovery mode. + * This refers to size in bytes. + * dword3 - b'0:31 - physical_address_lo: lower 32 bit physical address + * of the shared memory dedicated for messaging only when + * in UMAC hang recovery mode. + * dword4 - b'0:31 - physical_address_hi: higher 32 bit physical address + * of the shared memory dedicated for messaging only when + * in UMAC hang recovery mode. + */ + +/* t2h_msg_method and h2t_msg_method */ +enum htt_umac_hang_recovery_msg_method { + htt_umac_hang_recovery_msg_t2h_msi_and_h2t_polling = 0, +}; + +PREPACK typedef struct { + A_UINT32 msg_type : 8, + t2h_msg_method : 4, + h2t_msg_method : 4, + reserved : 16; + A_UINT32 t2h_msi_data; + /* size bytes and physical address of shared memory. */ + struct htt_h2t_host_paddr_size_entry_t msg_shared_mem; +} POSTPACK htt_h2t_umac_hang_recovery_prerequisite_setup_t; + +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_BYTES \ + (sizeof(htt_h2t_umac_hang_recovery_prerequisite_setup_t)) +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_DWORDS \ + (HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_BYTES >> 2) + +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD_M 0x00000F00 +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD_S 8 +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD_GET(word0) \ + (((word0) & HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD_M) >> \ + HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD_S) +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD_SET(word0, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD, _val); \ + ((word0) |= ((_val) << HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_T2H_MSG_METHOD_S));\ + } while (0) + +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD_M 0x0000F000 +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD_S 12 +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD_GET(word0) \ + (((word0) & HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD_M) >> \ + HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD_S) +#define HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD_SET(word0, _val) \ + do { \ + HTT_CHECK_SET_VAL(HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD, _val); \ + ((word0) |= ((_val) << HTT_H2T_UMAC_HANG_RECOVERY_PREREQUISITE_SETUP_H2T_MSG_METHOD_S));\ + } while (0) /*=== target -> host messages ===============================================*/ @@ -9661,6 +9902,7 @@ enum htt_t2h_msg_type { HTT_T2H_MSG_TYPE_SAWF_MSDUQ_INFO_IND = 0x2e, HTT_T2H_SAWF_MSDUQ_INFO_IND = 0x2e, /* alias */ HTT_T2H_MSG_TYPE_STREAMING_STATS_IND = 0x2f, + HTT_T2H_PPDU_ID_FMT_IND = 0x30, HTT_T2H_MSG_TYPE_TEST, @@ -18412,4 +18654,282 @@ PREPACK struct htt_t2h_sawf_msduq_event { } while (0) +/** + * @brief target -> PPDU id format indication + * + * MSG_TYPE => HTT_T2H_PPDU_ID_FMT_IND + * + * @details + * The following field definitions describe the format of the HTT target + * to host PPDU ID format indication message. + * hwsch_cmd_id :- A number per ring, increases by one with each HWSCH command. + * ring_id :- HWSCH ring id in which this PPDU was enqueued. + * seq_idx :- Sequence control index of this PPDU. + * link_id :- HW link ID of the link in which the PPDU was enqueued. + * seq_cmd_type:- WHAL_TXSEND_FTYPE (SU Data, MU Data, SGEN frames etc.) + * tqm_cmd:- + * + * |31 27|26 22|21 17| 16 |15 11|10 8|7 6|5 1| 0 | + * |--------------------------------------------------+------------------------| + * | rsvd0 | msg type | + * |-----+----------+----------+---------+-----+----------+----------+---------| + * |rsvd2|ring_id OF|ring_id NB|ring_id V|rsvd1|cmd_id OF |cmd_id NB |cmd_id V | + * |-----+----------+----------+---------+-----+----------+----------+---------| + * |rsvd4|link_id OF|link_id NB|link_id V|rsvd3|seq_idx OF|seq_idx NB|seq_idx V| + * |-----+----------+----------+---------+-----+----------+----------+---------| + * |rsvd6|tqm_cmd OF|tqm_cmd NB|tqm_cmd V|rsvd5|seq_cmd OF|seq_cmd NB|seq_cmd V| + * |-----+----------+----------+---------+-----+----------+----------+---------| + * |rsvd8| crc OF | crc NB | crc V |rsvd7|mac_id OF |mac_id NB |mac_id V | + * |-----+----------+----------+---------+-----+----------+----------+---------| + * Where: OF = bit offset, NB = number of bits, V = valid + * The message is interpreted as follows: + * + * dword0 - b'7:0 - msg_type: This will be set to + * HTT_T2H_PPDU_ID_FMT_IND + * value: 0x30 + * + * dword0 - b'31:8 - reserved + * + * dword1 - b'0:0 - field to indicate whether hwsch_cmd_id is valid or not + * + * dword1 - b'5:1 - number of bits in hwsch_cmd_id + * + * dword1 - b'10:6 - offset of hwsch_cmd_id (in number of bits) + * + * dword1 - b'15:11 - reserved for future use + * + * dword1 - b'16:16 - field to indicate whether ring_id is valid or not + * + * dword1 - b'21:17 - number of bits in ring_id + * + * dword1 - b'26:22 - offset of ring_id (in number of bits) + * + * dword1 - b'31:27 - reserved for future use + * + * dword2 - b'0:0 - field to indicate whether sequence index is valid or not + * + * dword2 - b'5:1 - number of bits in sequence index + * + * dword2 - b'10:6 - offset of sequence index (in number of bits) + * + * dword2 - b'15:11 - reserved for future use + * + * dword2 - b'16:16 - field to indicate whether link_id is valid or not + * + * dword2 - b'21:17 - number of bits in link_id + * + * dword2 - b'26:22 - offset of link_id (in number of bits) + * + * dword2 - b'31:27 - reserved for future use + * + * dword3 - b'0:0 - field to indicate whether seq_cmd_type is valid or not + * + * dword3 - b'5:1 - number of bits in seq_cmd_type + * + * dword3 - b'10:6 - offset of seq_cmd_type (in number of bits) + * + * dword3 - b'15:11 - reserved for future use + * + * dword3 - b'16:16 - field to indicate whether tqm_cmd is valid or not + * + * dword3 - b'21:17 - number of bits in tqm_cmd + * + * dword3 - b'26:22 - offset of tqm_cmd (in number of bits) + * + * dword3 - b'31:27 - reserved for future use + * + * dword4 - b'0:0 - field to indicate whether mac_id is valid or not + * + * dword4 - b'5:1 - number of bits in mac_id + * + * dword4 - b'10:6 - offset of mac_id (in number of bits) + * + * dword4 - b'15:11 - reserved for future use + * + * dword4 - b'16:16 - field to indicate whether crc is valid or not + * + * dword4 - b'21:17 - number of bits in crc + * + * dword4 - b'26:22 - offset of crc (in number of bits) + * + * dword4 - b'31:27 - reserved for future use + * + */ + +#define HTT_PPDU_ID_FMT_IND_VALID_BITS15_0_M 0x00000001 +#define HTT_PPDU_ID_FMT_IND_VALID_BITS15_0_S 0 + +#define HTT_PPDU_ID_FMT_IND_BITS_BITS15_0_M 0x0000003E +#define HTT_PPDU_ID_FMT_IND_BITS_BITS15_0_S 1 + +#define HTT_PPDU_ID_FMT_IND_OFFSET_BITS15_0_M 0x000007C0 +#define HTT_PPDU_ID_FMT_IND_OFFSET_BITS15_0_S 6 + +#define HTT_PPDU_ID_FMT_IND_VALID_BITS31_16_M 0x00010000 +#define HTT_PPDU_ID_FMT_IND_VALID_BITS31_16_S 16 + +#define HTT_PPDU_ID_FMT_IND_BITS_BITS31_16_M 0x003E0000 +#define HTT_PPDU_ID_FMT_IND_BITS_BITS31_16_S 17 + +#define HTT_PPDU_ID_FMT_IND_OFFSET_BITS31_16_M 0x07C00000 +#define HTT_PPDU_ID_FMT_IND_OFFSET_BITS31_16_S 22 + + +/* macros for accessing lower 16 bits in dword */ +#define HTT_PPDU_ID_FMT_IND_VALID_SET_BITS15_0(word, value) \ + do { \ + HTT_CHECK_SET_VAL(HTT_PPDU_ID_FMT_IND_VALID_BITS15_0, value); \ + (word) |= (value) << HTT_PPDU_ID_FMT_IND_VALID_BITS15_0_S; \ + } while (0) +#define HTT_PPDU_ID_FMT_IND_VALID_GET_BITS15_0(word) \ + (((word) & HTT_PPDU_ID_FMT_IND_VALID_BITS15_0_M) >> HTT_PPDU_ID_FMT_IND_VALID_BITS15_0_S) + +#define HTT_PPDU_ID_FMT_IND_BITS_SET_BITS15_0(word, value) \ + do { \ + HTT_CHECK_SET_VAL(HTT_PPDU_ID_FMT_IND_BITS_BITS15_0, value); \ + (word) |= (value) << HTT_PPDU_ID_FMT_IND_BITS_BITS15_0_S; \ + } while (0) +#define HTT_PPDU_ID_FMT_IND_BITS_GET_BITS15_0(word) \ + (((word) & HTT_PPDU_ID_FMT_IND_BITS_BITS15_0_M) >> HTT_PPDU_ID_FMT_IND_BITS_BITS15_0_S) + +#define HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS15_0(word, value) \ + do { \ + HTT_CHECK_SET_VAL(HTT_PPDU_ID_FMT_IND_OFFSET_BITS15_0, value); \ + (word) |= (value) << HTT_PPDU_ID_FMT_IND_OFFSET_BITS15_0_S; \ + } while (0) +#define HTT_PPDU_ID_FMT_IND_OFFSET_GET_BITS15_0(word) \ + (((word) & HTT_PPDU_ID_FMT_IND_OFFSET_BITS15_0_M) >> HTT_PPDU_ID_FMT_IND_OFFSET_BITS15_0_S) + +/* macros for accessing upper 16 bits in dword */ +#define HTT_PPDU_ID_FMT_IND_VALID_SET_BITS31_16(word, value) \ + do { \ + HTT_CHECK_SET_VAL(HTT_PPDU_ID_FMT_IND_VALID_BITS31_16, value); \ + (word) |= (value) << HTT_PPDU_ID_FMT_IND_VALID_BITS31_16_S; \ + } while (0) +#define HTT_PPDU_ID_FMT_IND_VALID_GET_BITS31_16(word) \ + (((word) & HTT_PPDU_ID_FMT_IND_VALID_BITS31_16_M) >> HTT_PPDU_ID_FMT_IND_VALID_BITS31_16_S) + +#define HTT_PPDU_ID_FMT_IND_BITS_SET_BITS31_16(word, value) \ + do { \ + HTT_CHECK_SET_VAL(HTT_PPDU_ID_FMT_IND_BITS_BITS31_16, value); \ + (word) |= (value) << HTT_PPDU_ID_FMT_IND_BITS_BITS31_16_S; \ + } while (0) +#define HTT_PPDU_ID_FMT_IND_BITS_GET_BITS31_16(word) \ + (((word) & HTT_PPDU_ID_FMT_IND_BITS_BITS31_16_M) >> HTT_PPDU_ID_FMT_IND_BITS_BITS31_16_S) + +#define HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS31_16(word, value) \ + do { \ + HTT_CHECK_SET_VAL(HTT_PPDU_ID_FMT_IND_OFFSET_BITS31_16, value); \ + (word) |= (value) << HTT_PPDU_ID_FMT_IND_OFFSET_BITS31_16_S; \ + } while (0) +#define HTT_PPDU_ID_FMT_IND_OFFSET_GET_BITS31_16(word) \ + (((word) & HTT_PPDU_ID_FMT_IND_OFFSET_BITS31_16_M) >> HTT_PPDU_ID_FMT_IND_OFFSET_BITS31_16_S) + + +#define HTT_PPDU_ID_FMT_IND_HWSCH_CMD_ID_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_HWSCH_CMD_ID_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_HWSCH_CMD_ID_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS15_0 + +#define HTT_PPDU_ID_FMT_IND_RING_ID_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_RING_ID_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_RING_ID_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS31_16 + +#define HTT_PPDU_ID_FMT_IND_SEQ_IDX_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_SEQ_IDX_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_SEQ_IDX_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS15_0 + +#define HTT_PPDU_ID_FMT_IND_LINK_ID_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_LINK_ID_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_LINK_ID_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS31_16 + +#define HTT_PPDU_ID_FMT_IND_SEQ_CMD_TYPE_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_SEQ_CMD_TYPE_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_SEQ_CMD_TYPE_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS15_0 + +#define HTT_PPDU_ID_FMT_IND_TQM_CMD_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_TQM_CMD_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_TQM_CMD_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS31_16 + +#define HTT_PPDU_ID_FMT_IND_MAC_ID_TYPE_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_MAC_ID_TYPE_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS15_0 +#define HTT_PPDU_ID_FMT_IND_MAC_ID_TYPE_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS15_0 + +#define HTT_PPDU_ID_FMT_IND_CRC_VALID_SET \ + HTT_PPDU_ID_FMT_IND_VALID_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_CRC_BITS_SET \ + HTT_PPDU_ID_FMT_IND_BITS_SET_BITS31_16 +#define HTT_PPDU_ID_FMT_IND_CRC_OFFSET_SET \ + HTT_PPDU_ID_FMT_IND_OFFSET_SET_BITS31_16 + + +/* offsets in number dwords */ +#define HTT_PPDU_ID_FMT_IND_HWSCH_CMD_ID_OFFSET 1 +#define HTT_PPDU_ID_FMT_IND_RING_ID_OFFSET 1 +#define HTT_PPDU_ID_FMT_IND_SEQ_IDX_OFFSET 2 +#define HTT_PPDU_ID_FMT_IND_LINK_ID_OFFSET 2 +#define HTT_PPDU_ID_FMT_IND_SEQ_CMD_TYPE_OFFSET 3 +#define HTT_PPDU_ID_FMT_IND_TQM_CMD_OFFSET 3 +#define HTT_PPDU_ID_FMT_IND_MAC_ID_OFFSET 4 +#define HTT_PPDU_ID_FMT_IND_CRC_OFFSET 4 + + +typedef struct { + A_UINT32 msg_type: 8, /* bits 7:0 */ + rsvd0: 24;/* bits 31:8 */ + A_UINT32 hwsch_cmd_id_valid: 1, /* bits 0:0 */ + hwsch_cmd_id_bits: 5, /* bits 5:1 */ + hwsch_cmd_id_offset: 5, /* bits 10:6 */ + rsvd1: 5, /* bits 15:11 */ + ring_id_valid: 1, /* bits 16:16 */ + ring_id_bits: 5, /* bits 21:17 */ + ring_id_offset: 5, /* bits 26:22 */ + rsvd2: 5; /* bits 31:27 */ + A_UINT32 seq_idx_valid: 1, /* bits 0:0 */ + seq_idx_bits: 5, /* bits 5:1 */ + seq_idx_offset: 5, /* bits 10:6 */ + rsvd3: 5, /* bits 15:11 */ + link_id_valid: 1, /* bits 16:16 */ + link_id_bits: 5, /* bits 21:17 */ + link_id_offset: 5, /* bits 26:22 */ + rsvd4: 5; /* bits 31:27 */ + A_UINT32 seq_cmd_type_valid: 1, /* bits 0:0 */ + seq_cmd_type_bits: 5, /* bits 5:1 */ + seq_cmd_type_offset: 5, /* bits 10:6 */ + rsvd5: 5, /* bits 15:11 */ + tqm_cmd_valid: 1, /* bits 16:16 */ + tqm_cmd_bits: 5, /* bits 21:17 */ + tqm_cmd_offset: 5, /* bits 26:12 */ + rsvd6: 5; /* bits 31:27 */ + A_UINT32 mac_id_valid: 1, /* bits 0:0 */ + mac_id_bits: 5, /* bits 5:1 */ + mac_id_offset: 5, /* bits 10:6 */ + rsvd8: 5, /* bits 15:11 */ + crc_valid: 1, /* bits 16:16 */ + crc_bits: 5, /* bits 21:17 */ + crc_offset: 5, /* bits 26:12 */ + rsvd9: 5; /* bits 31:27 */ +} htt_t2h_ppdu_id_fmt_ind_t; + + #endif diff --git a/drivers/staging/fw-api/fw/htt_ppdu_stats.h b/drivers/staging/fw-api/fw/htt_ppdu_stats.h index 2e5f4d48053c..50f5f8f26207 100644 --- a/drivers/staging/fw-api/fw/htt_ppdu_stats.h +++ b/drivers/staging/fw-api/fw/htt_ppdu_stats.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2017-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -53,6 +54,8 @@ enum htt_ppdu_stats_tlv_tag { HTT_PPDU_STATS_USR_COMPLTN_BA_BITMAP_1024_TLV,/* htt_ppdu_stats_user_compltn_ba_bitmap_1024_tlv */ HTT_PPDU_STATS_RX_MGMTCTRL_PAYLOAD_TLV, /* htt_ppdu_stats_rx_mgmtctrl_payload_tlv */ HTT_PPDU_STATS_FOR_SMU_TLV, /* htt_ppdu_stats_for_smu_tlv */ + HTT_PPDU_STATS_MLO_TX_RESP_TLV, /* htt_ppdu_stats_mlo_tx_resp_tlv */ + HTT_PPDU_STATS_MLO_TX_NOTIFICATION_TLV, /* htt_ppdu_stats_mlo_tx_notification_tlv */ /* New TLV's are added above to this line */ HTT_PPDU_STATS_MAX_TAG, @@ -2844,4 +2847,117 @@ typedef struct { A_UINT32 ba_bitmap[1]; } htt_ppdu_stats_for_smu_tlv; +typedef struct { + htt_tlv_hdr_t tlv_hdr; + /* + * BIT [ 2 : 0] :- response_reason + * BIT [ 6 : 3] :- mlo_change_t1_cts2self + * BIT [ 10 : 7] :- mlo_change_t1_ppdu + * BIT [ 14 : 11] :- mlo_change_t2_response + * BIT [ 18 : 15] :- mlo_change_t3_r2r + * BIT [ 19 : 19] :- partner_link_info_valid + * BIT [ 22 : 20] :- partner_link_id + * BIT [ 27 : 23] :- partner_link_cmd_ring_id + * BIT [ 28 : 28] :- dot11ax_trigger_frame_embedded + * BIT [ 31 : 29] :- reserved_0a + */ + A_UINT32 response_reason : 3, + mlo_change_t1_cts2self : 4, + mlo_change_t1_ppdu : 4, + mlo_change_t2_response : 4, + mlo_change_t3_r2r : 4, + partner_link_info_valid : 1, + partner_link_id : 3, + partner_link_cmd_ring_id : 5, + dot11ax_trigger_frame_embedded : 1, + reserved_0a : 3; + /* + * BIT [ 15 : 0] :- partner_link_schedule_id + * BIT [ 31 : 16] :- tx_rx_overlap_duration (microsecond units) + */ + A_UINT32 partner_link_schedule_id : 16, + tx_rx_overlap_duration_us : 16; + /* + * BIT [ 15 : 0] :- cts2self_duration (microsecond units) + * BIT [ 31 : 16] :- ppdu_duration (microsecond units) + */ + A_UINT32 cts2self_duration_us : 16, + ppdu_duration_us : 16; + /* + * BIT [ 15 : 0] :- response_duration (microsecond units) + * BIT [ 31 : 16] :- response_to_response_duration (microsecond units) + */ + A_UINT32 response_duration_us : 16, + response_to_response_duration_us : 16; + /* + * BIT [ 15 : 0] :- self_link_schedule_id + * BIT [ 31 : 16] :- hls_branch_debug_code + */ + A_UINT32 self_link_schedule_id : 16, + hls_branch_debug_code : 16; + /* + * BIT [ 31 : 0] :- hls_decision_debug_info + */ + A_UINT32 hls_decision_debug_info : 32; +} htt_ppdu_stats_mlo_tx_resp_tlv; + +typedef struct { + htt_tlv_hdr_t tlv_hdr; + /* + * BIT [ 2 : 0] :- notification_reason + * BIT [ 3 : 3] :- ml_decision + * BIT [ 4 : 4] :- cts2self_padding + * BIT [ 5 : 5] :- initiated_by_truncated_backoff + * BIT [ 8 : 6] :- transmit_start_reason + * BIT [ 14 : 9] :- num_users + * BIT [ 24 : 15] :- nstr_mlo_sta_id + * BIT [ 25 : 25] :- block_self_ml_sync + * BIT [ 26 : 26] :- block_partner_ml_sync + * BIT [ 27 : 27] :- nstr_mlo_sta_id_valid + * BIT [ 31 : 28] :- reserved_0a + */ + A_UINT32 notification_reason : 3, + ml_decision : 1, + cts2self_padding : 1, + initiated_by_truncated_backoff : 1, + transmit_start_reason : 3, + num_users : 6, + nstr_mlo_sta_id : 10, + block_self_ml_sync : 1, + block_partner_ml_sync : 1, + nstr_mlo_sta_id_valid : 1, + reserved_0a : 4; + /* + * BIT [ 15 : 0] :- pdg_ppdu_duration_adjust_value (microsecond units) + * BIT [ 31 : 16] :- mlo_ppdu_duration_adjust_value (microsecond units) + */ + A_UINT32 pdg_ppdu_duration_adjust_value_us : 16, + mlo_ppdu_duration_adjust_value_us : 16; + /* + * BIT [ 15 : 0] :- response_duration (microsecond units) + * BIT [ 31 : 16] :- response_to_response_duration (microsecond units) + */ + A_UINT32 response_duration_us : 16, + response_to_response_duration_us : 16; + /* + * BIT [ 15 : 0] :- schedule_id + * BIT [ 20 : 16] :- cmd_ring_id + * BIT [ 31 : 21] :- reserved_1a + */ + A_UINT32 schedule_id : 16, + cmd_ring_id : 5, + reserved_1a : 11; + /* + * BIT [ 31 : 0] :- mlo_reference_timestamp (microsecond units) + */ + A_UINT32 mlo_reference_timestamp_us : 32; + /* + * BIT [ 15 : 0] :- cts2self_duration (microsecond units) + * BIT [ 31 : 16] :- ppdu_duration (microsecond units) + */ + A_UINT32 cts2self_duration_us : 16, + ppdu_duration_us : 16; +} htt_ppdu_stats_mlo_tx_notification_tlv; + + #endif //__HTT_PPDU_STATS_H__ diff --git a/drivers/staging/fw-api/fw/htt_stats.h b/drivers/staging/fw-api/fw/htt_stats.h index 838e332342ca..04b37f54f27e 100644 --- a/drivers/staging/fw-api/fw/htt_stats.h +++ b/drivers/staging/fw-api/fw/htt_stats.h @@ -4653,6 +4653,8 @@ typedef struct { A_UINT32 ax_su_embedded_trigger_data_ppdu; /** 11AX HE SU data + embedded trigger PPDU failure stats (stats for HETP ack failure PPDU cnt) */ A_UINT32 ax_su_embedded_trigger_data_ppdu_err; + /** sta side trigger stats */ + A_UINT32 trigger_type_11be[HTT_TX_PDEV_STATS_NUM_11BE_TRIGGER_TYPES]; } htt_tx_pdev_rate_stats_tlv; typedef struct { @@ -5210,6 +5212,22 @@ typedef struct { A_UINT32 rx_ulofdma_data_nusers; } htt_rx_pdev_ul_ofdma_user_stats_tlv; +typedef struct { + htt_tlv_hdr_t tlv_hdr; + + A_UINT32 user_index; + /** PPDU level */ + A_UINT32 be_rx_ulofdma_non_data_ppdu; + /** PPDU level */ + A_UINT32 be_rx_ulofdma_data_ppdu; + /** MPDU level */ + A_UINT32 be_rx_ulofdma_mpdu_ok; + /** MPDU level */ + A_UINT32 be_rx_ulofdma_mpdu_fail; + A_UINT32 be_rx_ulofdma_non_data_nusers; + A_UINT32 be_rx_ulofdma_data_nusers; +} htt_rx_pdev_be_ul_ofdma_user_stats_tlv; + typedef struct { htt_tlv_hdr_t tlv_hdr; @@ -5333,6 +5351,8 @@ typedef struct { A_INT8 be_rx_ul_mumimo_fd_rssi[HTT_RX_PDEV_MAX_ULMUMIMO_NUM_USER][HTT_RX_PDEV_STATS_ULMUMIMO_NUM_SPATIAL_STREAMS]; /** Average pilot EVM measued for RX UL TB PPDU */ A_INT8 be_rx_ulmumimo_pilot_evm_dB_mean[HTT_RX_PDEV_MAX_ULMUMIMO_NUM_USER][HTT_RX_PDEV_STATS_ULMUMIMO_NUM_SPATIAL_STREAMS]; + /** Number of times UL MUMIMO TB PPDUs received in a punctured mode */ + A_UINT32 rx_ul_mumimo_punctured_mode[HTT_RX_PDEV_STATS_NUM_PUNCTURED_MODE_COUNTERS]; } htt_rx_pdev_ul_mumimo_trig_be_stats_tlv; /* STATS_TYPE : HTT_DBG_EXT_STATS_PDEV_UL_MUMIMO_TRIG_STATS @@ -6039,15 +6059,25 @@ typedef struct { A_UINT32 sounding[HTT_TX_NUM_OF_SOUNDING_STATS_WORDS]; /* cv upload handler stats */ + /** total times CV nc mismatched */ A_UINT32 cv_nc_mismatch_err; + /** total times CV has FCS error */ A_UINT32 cv_fcs_err; + /** total times CV has invalid NSS index */ A_UINT32 cv_frag_idx_mismatch; + /** total times CV has invalid SW peer ID */ A_UINT32 cv_invalid_peer_id; + /** total times CV rejected because TXBF is not setup in peer */ A_UINT32 cv_no_txbf_setup; + /** total times CV expired while in updating state */ A_UINT32 cv_expiry_in_update; + /** total times Pkt b/w exceeding the cbf_bw */ A_UINT32 cv_pkt_bw_exceed; + /** total times CV DMA not completed */ A_UINT32 cv_dma_not_done_err; + /** total times CV update to peer failed */ A_UINT32 cv_update_failed; + /* cv query stats */ /** total times CV query happened */ A_UINT32 cv_total_query; @@ -6092,6 +6122,16 @@ typedef struct { A_UINT32 cv_found_upload_in_progress; /** Expired CV found during query. */ A_UINT32 cv_expired_during_query; + /** total times CV dma timeout happened */ + A_UINT32 cv_dma_timeout_error; + /** total times CV bufs uploaded for IBF case */ + A_UINT32 cv_buf_ibf_uploads; + /** total times CV bufs uploaded for EBF case */ + A_UINT32 cv_buf_ebf_uploads; + /** total times CV bufs received from IPC ring */ + A_UINT32 cv_buf_received; + /** total times CV bufs fed back to the IPC ring */ + A_UINT32 cv_buf_fed_back; } htt_tx_sounding_stats_tlv; /* STATS_TYPE : HTT_DBG_EXT_STATS_TX_SOUNDING_INFO @@ -6529,6 +6569,7 @@ typedef struct { typedef enum { HTT_STATS_RC_MODE_DLSU = 0, HTT_STATS_RC_MODE_DLMUMIMO = 1, + HTT_STATS_RC_MODE_DLOFDMA = 2, } htt_stats_rc_mode; typedef struct { @@ -6551,6 +6592,12 @@ typedef enum { HTT_RC_MODE_2D_COUNT, } HTT_RC_MODE; +typedef enum { + HTT_STATS_RU_TYPE_INVALID = 0, + HTT_STATS_RU_TYPE_SINGLE_RU_ONLY = 1, + HTT_STATS_RU_TYPE_SINGLE_AND_MULTI_RU = 2, +} htt_stats_ru_type; + typedef struct { htt_tlv_hdr_t tlv_hdr; @@ -6573,6 +6620,9 @@ typedef struct { htt_tx_rate_stats_t per_bw320; A_UINT32 probe_cnt_per_rcmode[HTT_RC_MODE_2D_COUNT]; + + htt_stats_ru_type ru_type; /* refer to htt_stats_ru_type */ + htt_tx_rate_stats_t per_ru[HTT_TX_PDEV_STATS_NUM_BE_RU_SIZE_COUNTERS]; } htt_tx_rate_stats_per_tlv; /* NOTE: @@ -6993,6 +7043,36 @@ typedef enum { HTT_STATS_NO_RESET_SCAN_BACK_TO_SAME_HOME_CHANNEL_CHANGE = 0x00800000, /* No reset, scan to home channel change */ } HTT_STATS_RESET_CAUSE; +typedef enum { + HTT_CHANNEL_RATE_FULL, + HTT_CHANNEL_RATE_HALF, + HTT_CHANNEL_RATE_QUARTER, + + HTT_CHANNEL_RATE_COUNT +} HTT_CHANNEL_RATE; + +typedef enum { + HTT_PHY_BW_IDX_20MHz = 0, + HTT_PHY_BW_IDX_40MHz = 1, + HTT_PHY_BW_IDX_80MHz = 2, + HTT_PHY_BW_IDX_80Plus80 = 3, + HTT_PHY_BW_IDX_160MHz = 4, + HTT_PHY_BW_IDX_10MHz = 5, + HTT_PHY_BW_IDX_5MHz = 6, + HTT_PHY_BW_IDX_165MHz = 7, + +} HTT_PHY_BW_IDX; + +typedef enum { + HTT_WHAL_CONFIG_NONE = 0x00000000, + HTT_WHAL_CONFIG_NF_WAR = 0x00000001, + HTT_WHAL_CONFIG_CAL_WAR = 0x00000002, + HTT_WHAL_CONFIG_DO_NF_CAL = 0x00000004, + HTT_WHAL_CONFIG_SET_WAIT_FOR_NF_CAL = 0x00000008, + HTT_WHAL_CONFIG_FORCED_TX_PWR = 0x00000010, + HTT_WHAL_CONFIG_FORCED_GAIN_IDX = 0x00000020, + HTT_WHAL_CONFIG_FORCED_PER_CHAIN = 0x00000040, +} HTT_WHAL_CONFIG; typedef struct { htt_tlv_hdr_t tlv_hdr; @@ -7149,6 +7229,34 @@ typedef struct { */ A_UINT32 rxdesense_thresh_sw; A_UINT32 rxdesense_thresh_hw; + /** Current PHY Bandwidth - + * values are specified by the HTT_PHY_BW_IDX enum type + */ + A_UINT32 phy_bw_code; + /** Current channel operating rate - + * values are specified by the HTT_CHANNEL_RATE enum type + */ + A_UINT32 phy_rate_mode; + /** current channel operating band + * 0 - 5G; 1 - 2G; 2 -6G + */ + A_UINT32 phy_band_code; + /** microcode processor virtual phy base address - + * provided only for debug + */ + A_UINT32 phy_vreg_base; + /** microcode processor virtual phy base ext address - + * provided only for debug + */ + A_UINT32 phy_vreg_base_ext; + /** HW LUT table configuration for home/scan channel - + * provided only for debug + */ + A_UINT32 cur_table_index; + /** SW configuration flag for PHY reset and Calibrations - + * values are specified by the HTT_WHAL_CONFIG enum type + */ + A_UINT32 whal_config_flag; } htt_phy_reset_stats_tlv; typedef struct { @@ -7168,8 +7276,54 @@ typedef struct { /** phyoff count during rfmode switch */ A_UINT32 rf_mode_switch_phy_off_cnt; + + /** Temperature based recalibration count */ + A_UINT32 temperature_recal_cnt; } htt_phy_reset_counters_tlv; +/* Considering 320 MHz maximum 16 power levels */ +#define HTT_MAX_CH_PWR_INFO_SIZE 16 + +typedef struct { + htt_tlv_hdr_t tlv_hdr; + + /** current pdev_id */ + A_UINT32 pdev_id; + + /** Tranmsit power control scaling related configurations */ + A_UINT32 tx_power_scale; + A_UINT32 tx_power_scale_db; + + /** Minimum negative tx power supported by the target */ + A_INT32 min_negative_tx_power; + + /** current configured CTL domain */ + A_UINT32 reg_ctl_domain; + + /** Regulatory power information for the current channel */ + A_INT32 max_reg_allowed_power[HTT_STATS_MAX_CHAINS]; + A_INT32 max_reg_allowed_power_6g[HTT_STATS_MAX_CHAINS]; + /** channel max regulatory power in 0.5dB */ + A_UINT32 twice_max_rd_power; + + /** current channel and home channel's maximum possible tx power */ + A_INT32 max_tx_power; + A_INT32 home_max_tx_power; + + /** channel's Power Spectral Density */ + A_UINT32 psd_power; + /** channel's EIRP power */ + A_UINT32 eirp_power; + /** 6G channel power mode + * 0-LPI, 1-SP, 2-VLPI and 3-SP_CLIENT power mode + */ + A_UINT32 power_type_6ghz; + + /** sub-band channels and corresponding Tx-power */ + A_UINT32 sub_band_cfreq[HTT_MAX_CH_PWR_INFO_SIZE]; + A_UINT32 sub_band_txpower[HTT_MAX_CH_PWR_INFO_SIZE]; +} htt_phy_tpc_stats_tlv; + /* NOTE: * This structure is for documentation, and cannot be safely used directly. * Instead, use the constituent TLV structures to fill/parse. @@ -7179,6 +7333,7 @@ typedef struct { htt_phy_stats_tlv phy_stats; htt_phy_reset_counters_tlv phy_reset_counters; htt_phy_reset_stats_tlv phy_reset_stats; + htt_phy_tpc_stats_tlv phy_tpc_stats; } htt_phy_counters_and_phy_stats_t; /* NOTE: diff --git a/drivers/staging/fw-api/fw/wlan_module_ids.h b/drivers/staging/fw-api/fw/wlan_module_ids.h index c38ea2abc916..be00dff71882 100644 --- a/drivers/staging/fw-api/fw/wlan_module_ids.h +++ b/drivers/staging/fw-api/fw/wlan_module_ids.h @@ -123,7 +123,18 @@ typedef enum { WLAN_MODULE_STA_MLO_PS, /* 0x59 */ /* MLO PS manager */ WLAN_MODULE_MLO_SYNC_SEQ_NUM, /* 0x5a */ /* sync seq num after rm MPDU */ WLAN_MODULE_PLCMGR, /* 0x5b */ /* Policy Manager */ - + /* OEM module IDs: + * Reserve a small series of module IDs for use in OEM WLAN FW that + * interacts with WLAN FW SDK. + */ + WLAN_MODULE_OEM0, /* 0x5c */ + WLAN_MODULE_OEM1, /* 0x5d */ + WLAN_MODULE_OEM2, /* 0x5e */ + WLAN_MODULE_OEM3, /* 0x5f */ + WLAN_MODULE_OEM4, /* 0x60 */ + WLAN_MODULE_OEM5, /* 0x61 */ + WLAN_MODULE_OEM6, /* 0x62 */ + WLAN_MODULE_OEM7, /* 0x63 */ WLAN_MODULE_ID_MAX, WLAN_MODULE_ID_INVALID = WLAN_MODULE_ID_MAX, diff --git a/drivers/staging/fw-api/fw/wmi_services.h b/drivers/staging/fw-api/fw/wmi_services.h index 89a217faf3f8..d630c46772a9 100644 --- a/drivers/staging/fw-api/fw/wmi_services.h +++ b/drivers/staging/fw-api/fw/wmi_services.h @@ -587,6 +587,10 @@ typedef enum { WMI_SERVICE_HW_TX_POWER_CAPS_SIGNED_SUPPORT = 334, /* Indicates FW supports updating of Tx power capabilities as signed value */ WMI_SERVICE_MULTI_CLIENT_LL_SUPPORT = 335, /* FW supports set param cmd combined for multiple params */ WMI_SERVICE_AFC_PAYLOAD_CLEAR_SUPPORT = 336, /* FW supports clearing the AFC response payload in proxy mode */ + WMI_SERVICE_FW_INI_PARSE_SUPPORT = 337, /* FW supports parsing ini configuration file */ + WMI_SERVICE_TDLS_6GHZ_SUPPORT = 338, /* FW supports 6GHz TDLS both on base channel and offchannel */ + WMI_SERVICE_LINKSPEED_ROAM_TRIGGER_SUPPORT = 339, /* FW supports linkspeed trigger roam */ + WMI_SERVICE_UMAC_HANG_RECOVERY_SUPPORT = 340, /* FW supports recovering system from UMAC hang condition */ WMI_MAX_EXT2_SERVICE diff --git a/drivers/staging/fw-api/fw/wmi_tlv_defs.h b/drivers/staging/fw-api/fw/wmi_tlv_defs.h index acabf2831a55..9f84c672d04e 100644 --- a/drivers/staging/fw-api/fw/wmi_tlv_defs.h +++ b/drivers/staging/fw-api/fw/wmi_tlv_defs.h @@ -1268,6 +1268,13 @@ typedef enum { WMITLV_TAG_STRUC_wmi_tid_to_link_map, WMITLV_TAG_STRUC_wmi_peer_tid_to_link_map_fixed_param, WMITLV_TAG_STRUC_wmi_peer_assoc_tid_to_link_map, + WMITLV_TAG_STRUC_wmi_peer_delete_mlo_params, + WMITLV_TAG_STRUC_wmi_roam_enable_vendor_control_cmd_fixed_param, + WMITLV_TAG_STRUC_wmi_roam_get_vendor_control_param_cmd_fixed_param, + WMITLV_TAG_STRUC_wmi_roam_get_vendor_control_param_event_fixed_param, + WMITLV_TAG_STRUC_wmi_request_halphy_ctrl_path_stats_cmd_fixed_param, + WMITLV_TAG_STRUC_wmi_halphy_ctrl_path_stats_event_fixed_param, + WMITLV_TAG_STRUC_wmi_peer_flush_policy_cmd_fixed_param, } WMITLV_TAG_ID; /* @@ -1762,6 +1769,10 @@ typedef enum { OP(WMI_PMM_SCRATCH_REG_ALLOCATION_CMDID) \ OP(WMI_PEER_TX_FILTER_CMDID) \ OP(WMI_MLO_PEER_TID_TO_LINK_MAP_CMDID) \ + OP(WMI_ROAM_ENABLE_VENDOR_CONTROL_CMDID) \ + OP(WMI_ROAM_GET_VENDOR_CONTROL_PARAM_CMDID) \ + OP(WMI_REQUEST_HALPHY_CTRL_PATH_STATS_CMDID) \ + OP(WMI_PEER_FLUSH_POLICY_CMDID) \ /* add new CMD_LIST elements above this line */ @@ -2048,6 +2059,8 @@ typedef enum { OP(WMI_PMM_AVAILABLE_SCRATCH_REG_EVENTID) \ OP(WMI_PMM_SCRATCH_REG_ALLOCATION_COMPLETE_EVENTID) \ OP(WMI_VDEV_LATENCY_LEVEL_EVENTID) \ + OP(WMI_ROAM_GET_VENDOR_CONTROL_PARAM_EVENTID) \ + OP(WMI_HALPHY_CTRL_PATH_STATS_EVENTID) \ /* add new EVT_LIST elements above this line */ @@ -2073,7 +2086,8 @@ WMITLV_CREATE_PARAM_STRUC(WMI_PEER_CREATE_CMDID); /* Peer delete Cmd */ #define WMITLV_TABLE_WMI_PEER_DELETE_CMDID(id,op,buf,len) \ - WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_peer_delete_cmd_fixed_param, wmi_peer_delete_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_peer_delete_cmd_fixed_param, wmi_peer_delete_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_peer_delete_mlo_params, mlo_params, WMITLV_SIZE_VAR) WMITLV_CREATE_PARAM_STRUC(WMI_PEER_DELETE_CMDID); @@ -4397,6 +4411,14 @@ WMITLV_CREATE_PARAM_STRUC(WMI_REQUEST_PEER_STATS_INFO_CMDID); WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_UINT32, A_UINT32, odd_addr_read_args, WMITLV_SIZE_VAR) WMITLV_CREATE_PARAM_STRUC(WMI_REQUEST_CTRL_PATH_STATS_CMDID); +/* Request Halphy Stats through Ctrl Path */ +#define WMITLV_TABLE_WMI_REQUEST_HALPHY_CTRL_PATH_STATS_CMDID(id,op,buf,len) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_request_halphy_ctrl_path_stats_cmd_fixed_param, wmi_request_halphy_ctrl_path_stats_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX)\ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_UINT32, A_UINT32, pdev_ids, WMITLV_SIZE_VAR)\ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_UINT32, A_UINT32, vdev_ids, WMITLV_SIZE_VAR)\ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_FIXED_STRUC, wmi_mac_addr, mac_addr_list, WMITLV_SIZE_VAR) +WMITLV_CREATE_PARAM_STRUC(WMI_REQUEST_HALPHY_CTRL_PATH_STATS_CMDID); + /* Host sets the current country code */ #define WMITLV_TABLE_WMI_SET_CURRENT_COUNTRY_CMDID(id,op,buf,len) \ WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_set_current_country_cmd_fixed_param, wmi_set_current_country_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) @@ -4983,6 +5005,16 @@ WMITLV_CREATE_PARAM_STRUC(WMI_VDEV_SET_MU_SNIF_CMDID); WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_roam_set_param_cmd_fixed_param, wmi_roam_set_param_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) WMITLV_CREATE_PARAM_STRUC(WMI_ROAM_SET_PARAM_CMDID); +/* Roam enable vendor control Cmd */ +#define WMITLV_TABLE_WMI_ROAM_ENABLE_VENDOR_CONTROL_CMDID(id,op,buf,len) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_roam_enable_vendor_control_cmd_fixed_param, wmi_roam_enable_vendor_control_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) +WMITLV_CREATE_PARAM_STRUC(WMI_ROAM_ENABLE_VENDOR_CONTROL_CMDID); + +/* Roam Get vendor control Param Cmd */ +#define WMITLV_TABLE_WMI_ROAM_GET_VENDOR_CONTROL_PARAM_CMDID(id,op,buf,len) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_roam_get_vendor_control_param_cmd_fixed_param, wmi_roam_get_vendor_control_param_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) +WMITLV_CREATE_PARAM_STRUC(WMI_ROAM_GET_VENDOR_CONTROL_PARAM_CMDID); + #define WMITLV_TABLE_WMI_SAWF_SVC_CLASS_CFG_CMDID(id,op,buf,len) \ WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_sawf_svc_class_cfg_cmd_fixed_param, wmi_sawf_svc_class_cfg_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) WMITLV_CREATE_PARAM_STRUC(WMI_SAWF_SVC_CLASS_CFG_CMDID); @@ -5019,6 +5051,11 @@ WMITLV_CREATE_PARAM_STRUC(WMI_RTT_PASN_DEAUTH_CMD); WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_pmm_scratch_reg_info, scratch_reg_info, WMITLV_SIZE_VAR) WMITLV_CREATE_PARAM_STRUC(WMI_PMM_SCRATCH_REG_ALLOCATION_CMDID); +/* Peer flush Policy Cmd */ +#define WMITLV_TABLE_WMI_PEER_FLUSH_POLICY_CMDID(id,op,buf,len) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_peer_flush_policy_cmd_fixed_param, wmi_peer_flush_policy_cmd_fixed_param, fixed_param, WMITLV_SIZE_FIX) +WMITLV_CREATE_PARAM_STRUC(WMI_PEER_FLUSH_POLICY_CMDID); + /************************** TLV definitions of WMI events *******************************/ @@ -5370,6 +5407,11 @@ WMITLV_CREATE_PARAM_STRUC(WMI_ROAM_SYNCH_FRAME_EVENTID); WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_BYTE, A_UINT8, frame, WMITLV_SIZE_VAR) WMITLV_CREATE_PARAM_STRUC(WMI_ROAM_FRAME_EVENTID); +/* Get Roam Vendor Control Param Event */ +#define WMITLV_TABLE_WMI_ROAM_GET_VENDOR_CONTROL_PARAM_EVENTID(id,op,buf,len) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_roam_get_vendor_control_param_event_fixed_param, wmi_roam_get_vendor_control_param_event_fixed_param, fixed_param, WMITLV_SIZE_FIX) +WMITLV_CREATE_PARAM_STRUC(WMI_ROAM_GET_VENDOR_CONTROL_PARAM_EVENTID); + /* WOW Wakeup Host Event */ /* NOTE: Make sure wow_bitmap_info can be zero or one elements only */ #define WMITLV_TABLE_WMI_WOW_WAKEUP_HOST_EVENTID(id,op,buf,len) \ @@ -6429,6 +6471,23 @@ WMITLV_CREATE_PARAM_STRUC(WMI_PEER_STATS_INFO_EVENTID); WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_ctrl_path_afc_stats_struct, ctrl_path_afc_stats, WMITLV_SIZE_VAR) WMITLV_CREATE_PARAM_STRUC(WMI_CTRL_PATH_STATS_EVENTID); +/* + * Update HALPHY Control Path stats event + * Below definition shows TLV packing of HALPHY ctrl path event + * Special Patch code is needed on host side to have compatible + * HOST Endianness + */ +#define WMITLV_TABLE_WMI_HALPHY_CTRL_PATH_STATS_EVENTID(id, op, buf, len) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_halphy_ctrl_path_stats_event_fixed_param, wmi_halphy_ctrl_path_stats_event_fixed_param, fixed_param, WMITLV_SIZE_FIX) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_tpc_configs, tpc_configs, WMITLV_SIZE_VAR) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_max_reg_power_allowed, regulatory_power, WMITLV_SIZE_VAR) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_INT16, A_INT16, reg_buf, WMITLV_SIZE_VAR) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_tpc_rates_array, tpc_rates, WMITLV_SIZE_VAR) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_INT16, A_UINT16, rates_buf, WMITLV_SIZE_VAR) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_tpc_ctl_pwr_table, ctl_power, WMITLV_SIZE_VAR) \ + WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_BYTE, A_INT8, ctl_buf, WMITLV_SIZE_VAR) +WMITLV_CREATE_PARAM_STRUC(WMI_HALPHY_CTRL_PATH_STATS_EVENTID); + #define WMITLV_TABLE_WMI_RADIO_CHAN_STATS_EVENTID(id, op, buf, len) \ WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_STRUC_wmi_radio_chan_stats_event_fixed_param, wmi_radio_chan_stats_event_fixed_param, fixed_param, WMITLV_SIZE_FIX) \ WMITLV_ELEM(id,op,buf,len, WMITLV_TAG_ARRAY_STRUC, wmi_radio_chan_stats, radio_chan_stats, WMITLV_SIZE_VAR) diff --git a/drivers/staging/fw-api/fw/wmi_unified.h b/drivers/staging/fw-api/fw/wmi_unified.h index 10728000b880..90cb06abe80a 100644 --- a/drivers/staging/fw-api/fw/wmi_unified.h +++ b/drivers/staging/fw-api/fw/wmi_unified.h @@ -706,6 +706,10 @@ typedef enum { /* Mac addr based filtering*/ WMI_PEER_TX_FILTER_CMDID, + /** flush specific tid queues of a peer */ + WMI_PEER_FLUSH_POLICY_CMDID, + + /* beacon/management specific commands */ /** transmit beacon by reference . used for transmitting beacon on low latency interface like pcie */ @@ -845,6 +849,10 @@ typedef enum { WMI_ROAM_MLO_CONFIG_CMDID, /** set roam params **/ WMI_ROAM_SET_PARAM_CMDID, + /** Enable or Disable roam vendor control */ + WMI_ROAM_ENABLE_VENDOR_CONTROL_CMDID, + /** Get roam vendor control params */ + WMI_ROAM_GET_VENDOR_CONTROL_PARAM_CMDID, /** offload scan specific commands */ /** set offload scan AP profile */ @@ -1016,6 +1024,9 @@ typedef enum { /** request for thermal stats */ WMI_REQUEST_THERMAL_STATS_CMDID, + /** request for HALPHY stats through ctrl path */ + WMI_REQUEST_HALPHY_CTRL_PATH_STATS_CMDID, + /** ARP OFFLOAD REQUEST*/ WMI_SET_ARP_NS_OFFLOAD_CMDID = WMI_CMD_GRP_START_ID(WMI_GRP_ARP_NS_OFL), @@ -1854,6 +1865,8 @@ typedef enum { WMI_ROAM_CAPABILITY_REPORT_EVENTID, /** Send AP frame content like beacon/probe resp etc.. */ WMI_ROAM_FRAME_EVENTID, + /** GET Roam Vendor Control Param event */ + WMI_ROAM_GET_VENDOR_CONTROL_PARAM_EVENTID, /** P2P disc found */ WMI_P2P_DISC_EVENTID = WMI_EVT_GRP_START_ID(WMI_GRP_P2P), @@ -1943,6 +1956,11 @@ typedef enum { * and report stats info to host */ WMI_CTRL_PATH_STATS_EVENTID, + /** This event is used to respond to + * WMI_REQUEST_HALPHY_CTRL_PATH_STATS_CMDID and report stats info to host + */ + WMI_HALPHY_CTRL_PATH_STATS_EVENTID, + /* NLO specific events */ /** NLO match event after the first match */ @@ -4588,6 +4606,12 @@ typedef struct { #define WMI_RSRC_CFG_FLAGS2_SAWF_CONFIG_ENABLE_SET(flags2, value) \ WMI_SET_BITS(flags2, 13, 1, value) +#define WMI_RSRC_CFG_FLAGS2_NOTIFY_FRAME_CONFIG_ENABLE_GET(flags2) \ + WMI_GET_BITS(flags2, 14, 1) +#define WMI_RSRC_CFG_FLAGS2_NOTIFY_FRAME_CONFIG_ENABLE_SET(flags2, value) \ + WMI_SET_BITS(flags2, 14, 1, value) + + #define WMI_RSRC_CFG_HOST_SERVICE_FLAG_NAN_IFACE_SUPPORT_GET(host_service_flags) \ WMI_GET_BITS(host_service_flags, 0, 1) #define WMI_RSRC_CFG_HOST_SERVICE_FLAG_NAN_IFACE_SUPPORT_SET(host_service_flags, val) \ @@ -6594,6 +6618,16 @@ typedef struct { * 1 - HT * 2 - VHT * 3 - HE + * 4 - EHT + * + * Rate Bit mask format: + * ... + * + * + * EHT Rate Bit Mask format: + * ... + * + * */ A_UINT32 type; @@ -11495,6 +11529,25 @@ typedef struct { */ } wmi_ctrl_path_stats_event_fixed_param; +typedef struct { + /** TLV tag and len; tag equals + * WMITLV_TAG_STRUC_wmi_halphy_ctrl_path_stats_event_fixed_param + */ + A_UINT32 tlv_header; + + /** Process id requested */ + A_UINT32 request_id; + + /* end_of_event - single event or Multiple Event */ + A_UINT32 end_of_event; + + /* + * event_count - If Multiple Events are send, this is to identify + * particular event out of Multiple Events that are send to host + */ + A_UINT32 event_count; +} wmi_halphy_ctrl_path_stats_event_fixed_param; + typedef struct { /** TLV tag and len; tag equals * WMITLV_TAG_STRUC_wmi_radio_chan_stats */ @@ -15537,12 +15590,24 @@ typedef struct { */ } wmi_peer_create_cmd_fixed_param; +/* this TLV structure is used for providing MLO parameters on peer delete */ +typedef struct { + A_UINT32 tlv_header; /** TLV tag and len */ + A_UINT32 mlo_hw_link_id_bitmap; /* Hardware link id of the link which has crashed or was not created in the first place */ +} wmi_peer_delete_mlo_params; + typedef struct { A_UINT32 tlv_header; /** TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_peer_delete_cmd_fixed_param */ /** unique id identifying the VDEV, generated by the caller */ A_UINT32 vdev_id; /** peer MAC address */ wmi_mac_addr peer_macaddr; +/* The TLVs follows this structure: + * wmi_peer_delete_mlo_params mlo_params[]; <-- MLO flags on peer_delete + * Optional TLV, only present for MLO peers. + * If the peer is non-MLO, the array length should be 0. + * Used for Link peer deleted. + */ } wmi_peer_delete_cmd_fixed_param; #define WMI_VDEV_ALL_PEER_MAX_BITMAP_WORD32 ((WMI_PEER_TYPE_HOST_MAX+31) / 32) @@ -16116,8 +16181,15 @@ typedef struct { typedef struct { A_UINT32 tlv_header; /** TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_eht_rate_set */ - A_UINT32 rx_mcs_set; - A_UINT32 tx_mcs_set; + /* + * B0-B3 indicates max NSS that supports mcs 0-7 + * B4-B7 indicates max NSS that supports mcs 8-9 + * B8-B11 indicates max NSS that supports mcs 10-11 + * B12-B15 indicates max NSS that supports mcs 12-13 + * B16-B31 reserved + */ + A_UINT32 rx_mcs_set; /* Rx max NSS set */ + A_UINT32 tx_mcs_set; /* Tx max NSS set */ } wmi_eht_rate_set; /* @@ -16410,7 +16482,7 @@ typedef struct { * wmi_peer_assoc_mlo_params mlo_params[0,1]; <-- MLO parameters opt. TLV * Only present for MLO peers. * For non-MLO peers the array length should be 0. - * wmi_eht_rate_set_peer_eht_rates; <-- EHT capabilities of the peer + * wmi_eht_rate_set peer_eht_rates; <-- EHT capabilities of the peer * wmi_peer_assoc_mlo_partner_link_params link_info[] <-- partner link info * wmi_peer_assoc_tid_to_link_map[] <-- tid to link_map info */ @@ -18857,6 +18929,10 @@ typedef enum wake_reason_e { WOW_REASON_MDNS_WAKEUP, WOW_REASON_RTT_11AZ, WOW_REASON_P2P_NOA_UPDATE, + /* When Host configured timer elapsed for delayed wakeup */ + WOW_REASON_DELAYED_WAKEUP_HOST_CFG_TIMER_ELAPSED, + /* Data store list is full, so Host wakeup should be triggered */ + WOW_REASON_DELAYED_WAKEUP_DATA_STORE_LIST_FULL, /* add new WOW_REASON_ defs before this line */ WOW_REASON_MAX, @@ -26386,6 +26462,12 @@ typedef struct { A_UINT32 wlan_global_tsf_low; /* high 32 bits of wlan global tsf */ A_UINT32 wlan_global_tsf_high; + /* low 32 bits of tqm timer */ + A_UINT32 tqm_timer_low; + /* high 32 bits of tqm timer */ + A_UINT32 tqm_timer_high; + /* use tqm timer flag */ + A_UINT32 use_tqm_timer; } wmi_vdev_tsf_report_event_fixed_param; /* ie_id values: @@ -29489,6 +29571,11 @@ typedef enum { WMI_REQUEST_CTRL_PATH_AFC_STAT = 11, } wmi_ctrl_path_stats_id; +typedef enum { + /* bit 0 is currently unused / reserved */ + WMI_HALPHY_CTRL_PATH_PDEV_TX_STAT = 1, +} wmi_halphy_ctrl_path_stats_id; + typedef enum { /* * The following stats actions are mutually exclusive. @@ -29500,6 +29587,13 @@ typedef enum { WMI_REQUEST_CTRL_PATH_STAT_STOP = 4, } wmi_ctrl_path_stats_action; +typedef enum { + WMI_HALPHY_CTRL_PATH_SU_STATS = 0, + WMI_HALPHY_CTRL_PATH_SUTXBF_STATS, + WMI_HALPHY_CTRL_PATH_MU_STATS, + WMI_HALPHY_CTRL_PATH_MUTXBF_STATS, +} wmi_halphy_ctrl_path_subid; + typedef struct { /** TLV tag and len; tag equals * WMITLV_TAG_STRUC_wmi_request_ctrl_path_stats_cmd_fixed_param */ @@ -29530,6 +29624,44 @@ typedef struct { */ } wmi_request_ctrl_path_stats_cmd_fixed_param; +typedef struct { + /** TLV tag and len; tag equals + * WMITLV_TAG_STRUC_wmi_request_halphy_ctrl_path_stats_cmd_fixed_param */ + A_UINT32 tlv_header; + + /** Bitmask showing which of stats IDs 0-31 have been requested. + * These stats ids are defined in enum wmi_halphy_ctrl_path_stats_id. + */ + A_UINT32 stats_id_mask; + /** process id requested */ + A_UINT32 request_id; + /** action + * get/reset based on stats id + * defined as a part of wmi_ctrl_path_stats_action + **/ + A_UINT32 action; /* refer to wmi_ctrl_path_stats_action */ + + /** Request Halphy subid stats + * According to the requested stats_id_mask this halphy_subid varies + * For stats_id = 1, the possible values could be enum + * wmi_halphy_ctrl_path_halphyid + */ + A_UINT32 halphy_subid; + + /** The below TLV arrays optionally follow this fixed_param TLV structure: + * 1. A_UINT32 pdev_ids[]; + * If this array is present and non-zero length, stats should only + * be provided from the pdevs identified in the array. + * 2. A_UINT32 vdev_ids[]; + * If this array is present and non-zero length, stats should only + * be provided from the vdevs identified in the array. + * 3. wmi_mac_addr peer_macaddr[]; + * If this array is present and non-zero length, stats should only + * be provided from the peers with the MAC addresses specified + * in the array. + */ +} wmi_request_halphy_ctrl_path_stats_cmd_fixed_param; + typedef enum { WMI_REQUEST_ONE_RADIO_CHAN_STATS = 0x01, /* request stats of one specified channel */ WMI_REQUEST_ALL_RADIO_CHAN_STATS = 0x02, /* request stats of all channels */ @@ -31369,6 +31501,10 @@ static INLINE A_UINT8 *wmi_id_to_name(A_UINT32 wmi_command) WMI_RETURN_STRING(WMI_SET_MULTIPLE_PDEV_VDEV_PARAM_CMDID); WMI_RETURN_STRING(WMI_PMM_SCRATCH_REG_ALLOCATION_CMDID); WMI_RETURN_STRING(WMI_MLO_PEER_TID_TO_LINK_MAP_CMDID); + WMI_RETURN_STRING(WMI_ROAM_ENABLE_VENDOR_CONTROL_CMDID); + WMI_RETURN_STRING(WMI_ROAM_GET_VENDOR_CONTROL_PARAM_CMDID); + WMI_RETURN_STRING(WMI_REQUEST_HALPHY_CTRL_PATH_STATS_CMDID); + WMI_RETURN_STRING(WMI_PEER_FLUSH_POLICY_CMDID); } return (A_UINT8 *) "Invalid WMI cmd"; @@ -33847,6 +33983,9 @@ typedef struct { A_UINT32 btm_req_dialog_token; } wmi_roam_trigger_reason; +#define WMI_GET_BTCONNECT_STATUS(flags) WMI_GET_BITS(flags, 0, 1) +#define WMI_SET_BTCONNECT_STATUS(flags, val) WMI_SET_BITS(flags, 0, 1, val) + typedef struct { A_UINT32 tlv_header; /* TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_roam_scan_info_tlv_param */ /* roam_scan_type: @@ -33867,6 +34006,12 @@ typedef struct { * This timestamp indicates the time when roam scan finished. */ A_UINT32 scan_complete_timestamp; /* milli second units */ + /* + * Flags capturing factors involved during roam scan: + * Bit 0 : Bluetooth connect status, 0(not connected) or 1(connected). + * Bit 1-31 : reserved for future use. + */ + A_UINT32 flags; } wmi_roam_scan_info; typedef struct { @@ -34039,6 +34184,9 @@ typedef struct { A_UINT32 channel; /* Channel frequency in MHz */ } wmi_roam_neighbor_report_channel_info; +#define WMI_GET_ASSOC_ID(frame_info_ext) WMI_GET_BITS(frame_info_ext, 0, 16) +#define WMI_SET_ASSOC_ID(frame_info_ext, val) WMI_SET_BITS(frame_info_ext, 0, 16, val) + typedef struct { A_UINT32 tlv_header; /* TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_roam_frame_info_tlv_param */ /* timestamp is the absolute time w.r.t host timer which is synchronized between the host and target */ @@ -34082,6 +34230,13 @@ typedef struct { */ A_UINT32 retry_count; wmi_mac_addr bssid; /* AP MAC address */ + /* + * frame_info_ext captures below fields: + * Bit 0-15 : (re)assoc id of (re)association response frame, + * section 9.4.1.8 AID field. + * Bit 16~31 : reserved for future use. + */ + A_UINT32 frame_info_ext; } wmi_roam_frame_info; typedef enum { @@ -34219,6 +34374,47 @@ typedef struct { A_UINT32 param_value; } wmi_roam_set_param_cmd_fixed_param; +typedef struct { + A_UINT32 tlv_header; /** TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_vdev_enable_vendor_cmd_fixed_param */ + /** unique id identifying the VDEV, generated by the caller */ + A_UINT32 vdev_id; + /** abstract mechanism for differentiating vendors */ + A_UINT32 vendor_id; + /** 1 - Enable, 0 - Disable */ + A_UINT32 enable; +} wmi_roam_enable_vendor_control_cmd_fixed_param; + +typedef enum { + ROAM_VENDOR_CONTROL_PARAM_TRIGGER = 1, + ROAM_VENDOR_CONTROL_PARAM_DELTA, + ROAM_VENDOR_CONTROL_PARAM_FULL_SCANPERIOD, + ROAM_VENDOR_CONTROL_PARAM_PARTIAL_SCANPERIOD, + ROAM_VENDOR_CONTROL_PARAM_ACTIVE_CH_DWELLTIME, + ROAM_VENDOR_CONTROL_PARAM_PASSIVE_CH_DWELLTIME, + ROAM_VENDOR_CONTROL_PARAM_HOME_CH_TIME, + ROAM_VENDOR_CONTROL_PARAM_AWAY_TIME, +} WMI_ROAM_GET_VENDOR_CONTROL_PARAM_ID; + +typedef struct { + A_UINT32 tlv_header; /** TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_vdev_get_vendor_param_cmd_fixed_param */ + /** unique id identifying the VDEV, generated by the caller */ + A_UINT32 vdev_id; + /** Vendor Control Param ID from enum WMI_ROAM_GET_VENDOR_CONTROL_PARAM_ID */ + A_UINT32 param_id; +} wmi_roam_get_vendor_control_param_cmd_fixed_param; + +/* WMI_ROAM_GET_VENDOR_CONTROL_PARAM_EVENTID - No need to make this event as wakeable event + * Host ensures to take wakelock after sending WMI_ROAM_GET_VENDOR_CONTROL_PARAM_CMDID command */ +typedef struct { + A_UINT32 tlv_header; /** TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_roam_get_vendor_control_param_event_fixed_param */ + /** unique id identifying the VDEV, generated by the caller */ + A_UINT32 vdev_id; + /** Vendor Control Param ID from enum WMI_ROAM_GET_VENDOR_CONTROL_PARAM_ID */ + A_UINT32 param_id; + /** Vendor control param value */ + A_UINT32 param_value; +} wmi_roam_get_vendor_control_param_event_fixed_param; + /** the definition of different ROAM parameters */ typedef enum { /* roam param to configure below roam events @@ -34229,6 +34425,11 @@ typedef enum { * Bit : 2-31 - reserved */ WMI_ROAM_PARAM_ROAM_EVENTS_CONFIG = 0x1, + /* + * Bit : 0 if unset, POOR_LINKSPEED + * Bit : 0 if set, GOOD_LINKSPEED + */ + WMI_ROAM_PARAM_LINKSPEED_STATE = 0x2, /*=== END ROAM_PARAM_PROTOTYPE SECTION ===*/ } WMI_ROAM_PARAM; @@ -36090,7 +36291,11 @@ typedef struct { #define WMI_EHTCAP_MAC_MAXMPDULEN_GET(eht_cap_mac) WMI_GET_BITS(eht_cap_mac[0], 6, 2) #define WMI_EHTCAP_MAC_MAXMPDULEN_SET(eht_cap_mac, value) WMI_SET_BITS(eht_cap_mac[0], 6, 2, value) -/* Bit 8-15: reserved */ +/* Bit 8: Maximum A-MPDU Length Exponent Extension */ +#define WMI_EHTCAP_MAC_MAXAMPDULEN_EXP_GET(eht_cap_mac) WMI_GET_BITS(eht_cap_mac[0], 8, 1) +#define WMI_EHTCAP_MAC_MAXAMPDULEN_EXP_SET(eht_cap_mac, value) WMI_SET_BITS(eht_cap_mac[0], 8, 1, value) + +/* Bit 9-15: reserved */ /****** End of 11BE EHT MAC Capabilities Information field ******/ @@ -38485,6 +38690,30 @@ typedef struct { A_UINT32 is_allocated; } wmi_pmm_scratch_reg_allocation_complete_event_fixed_param; +typedef enum { + WMI_NO_FLUSH, + WMI_TWT_FLUSH, + + /* Add new flush policies above */ + WMI_MAX_FLUSH_POLICY +} wmi_peer_flush_policy; + +typedef struct { + A_UINT32 tlv_header; /** TLV tag and len; tag equals WMITLV_TAG_STRUC_wmi_peer_flush_policy_cmd_fixed_param */ + A_UINT32 vdev_id; + /** peer MAC address */ + wmi_mac_addr peer_macaddr; + /** The tids to flush */ + A_UINT32 peer_tid_bitmap; + /* wmi_peer_flush_policy */ + A_UINT32 flush_policy; + /* n_TWT_SPs_to_expire: + * Expire / drop packets whose age is greater than this specified number + * of TWT service periods. + */ + A_UINT32 n_TWT_SPs_to_expire; +} wmi_peer_flush_policy_cmd_fixed_param; + /* ADD NEW DEFS HERE */ diff --git a/drivers/staging/fw-api/fw/wmi_version.h b/drivers/staging/fw-api/fw/wmi_version.h index 32d4ffb405df..cad40dc5602e 100644 --- a/drivers/staging/fw-api/fw/wmi_version.h +++ b/drivers/staging/fw-api/fw/wmi_version.h @@ -37,7 +37,7 @@ #define __WMI_VER_MINOR_ 0 /** WMI revision number has to be incremented when there is a * change that may or may not break compatibility. */ -#define __WMI_REVISION_ 1148 +#define __WMI_REVISION_ 1158 /** The Version Namespace should not be normally changed. Only * host and firmware of the same WMI namespace will work diff --git a/drivers/staging/fw-api/hw/kiwi/v1/rx_reo_queue_1k.h b/drivers/staging/fw-api/hw/kiwi/v1/rx_reo_queue_1k.h new file mode 100644 index 000000000000..d87020dddf94 --- /dev/null +++ b/drivers/staging/fw-api/hw/kiwi/v1/rx_reo_queue_1k.h @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef _RX_REO_QUEUE_1K_H_ +#define _RX_REO_QUEUE_1K_H_ +#if !defined(__ASSEMBLER__) +#endif + +#include "uniform_descriptor_header.h" +#define NUM_OF_DWORDS_RX_REO_QUEUE_1K 32 + +struct rx_reo_queue_1k { +#ifndef WIFI_BIT_ORDER_BIG_ENDIAN + struct uniform_descriptor_header descriptor_header; + uint32_t rx_bitmap_319_288 : 32; + uint32_t rx_bitmap_351_320 : 32; + uint32_t rx_bitmap_383_352 : 32; + uint32_t rx_bitmap_415_384 : 32; + uint32_t rx_bitmap_447_416 : 32; + uint32_t rx_bitmap_479_448 : 32; + uint32_t rx_bitmap_511_480 : 32; + uint32_t rx_bitmap_543_512 : 32; + uint32_t rx_bitmap_575_544 : 32; + uint32_t rx_bitmap_607_576 : 32; + uint32_t rx_bitmap_639_608 : 32; + uint32_t rx_bitmap_671_640 : 32; + uint32_t rx_bitmap_703_672 : 32; + uint32_t rx_bitmap_735_704 : 32; + uint32_t rx_bitmap_767_736 : 32; + uint32_t rx_bitmap_799_768 : 32; + uint32_t rx_bitmap_831_800 : 32; + uint32_t rx_bitmap_863_832 : 32; + uint32_t rx_bitmap_895_864 : 32; + uint32_t rx_bitmap_927_896 : 32; + uint32_t rx_bitmap_959_928 : 32; + uint32_t rx_bitmap_991_960 : 32; + uint32_t rx_bitmap_1023_992 : 32; + uint32_t reserved_24 : 32; + uint32_t reserved_25 : 32; + uint32_t reserved_26 : 32; + uint32_t reserved_27 : 32; + uint32_t reserved_28 : 32; + uint32_t reserved_29 : 32; + uint32_t reserved_30 : 32; + uint32_t reserved_31 : 32; +#else + struct uniform_descriptor_header descriptor_header; + uint32_t rx_bitmap_319_288 : 32; + uint32_t rx_bitmap_351_320 : 32; + uint32_t rx_bitmap_383_352 : 32; + uint32_t rx_bitmap_415_384 : 32; + uint32_t rx_bitmap_447_416 : 32; + uint32_t rx_bitmap_479_448 : 32; + uint32_t rx_bitmap_511_480 : 32; + uint32_t rx_bitmap_543_512 : 32; + uint32_t rx_bitmap_575_544 : 32; + uint32_t rx_bitmap_607_576 : 32; + uint32_t rx_bitmap_639_608 : 32; + uint32_t rx_bitmap_671_640 : 32; + uint32_t rx_bitmap_703_672 : 32; + uint32_t rx_bitmap_735_704 : 32; + uint32_t rx_bitmap_767_736 : 32; + uint32_t rx_bitmap_799_768 : 32; + uint32_t rx_bitmap_831_800 : 32; + uint32_t rx_bitmap_863_832 : 32; + uint32_t rx_bitmap_895_864 : 32; + uint32_t rx_bitmap_927_896 : 32; + uint32_t rx_bitmap_959_928 : 32; + uint32_t rx_bitmap_991_960 : 32; + uint32_t rx_bitmap_1023_992 : 32; + uint32_t reserved_24 : 32; + uint32_t reserved_25 : 32; + uint32_t reserved_26 : 32; + uint32_t reserved_27 : 32; + uint32_t reserved_28 : 32; + uint32_t reserved_29 : 32; + uint32_t reserved_30 : 32; + uint32_t reserved_31 : 32; +#endif +}; + +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_OFFSET 0x00000000 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_LSB 0 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_MSB 3 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_MASK 0x0000000f + +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_OFFSET 0x00000000 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_LSB 4 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_MSB 7 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_MASK 0x000000f0 + +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_OFFSET 0x00000000 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_LSB 8 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_MSB 31 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_MASK 0xffffff00 + +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_OFFSET 0x00000004 +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_OFFSET 0x00000008 +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_OFFSET 0x0000000c +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_OFFSET 0x00000010 +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_OFFSET 0x00000014 +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_OFFSET 0x00000018 +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_OFFSET 0x0000001c +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_OFFSET 0x00000020 +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_OFFSET 0x00000024 +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_OFFSET 0x00000028 +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_OFFSET 0x0000002c +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_OFFSET 0x00000030 +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_OFFSET 0x00000034 +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_OFFSET 0x00000038 +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_OFFSET 0x0000003c +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_OFFSET 0x00000040 +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_OFFSET 0x00000044 +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_OFFSET 0x00000048 +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_OFFSET 0x0000004c +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_OFFSET 0x00000050 +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_OFFSET 0x00000054 +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_OFFSET 0x00000058 +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_OFFSET 0x0000005c +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_24_OFFSET 0x00000060 +#define RX_REO_QUEUE_1K_RESERVED_24_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_24_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_24_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_25_OFFSET 0x00000064 +#define RX_REO_QUEUE_1K_RESERVED_25_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_25_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_25_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_26_OFFSET 0x00000068 +#define RX_REO_QUEUE_1K_RESERVED_26_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_26_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_26_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_27_OFFSET 0x0000006c +#define RX_REO_QUEUE_1K_RESERVED_27_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_27_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_27_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_28_OFFSET 0x00000070 +#define RX_REO_QUEUE_1K_RESERVED_28_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_28_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_28_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_29_OFFSET 0x00000074 +#define RX_REO_QUEUE_1K_RESERVED_29_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_29_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_29_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_30_OFFSET 0x00000078 +#define RX_REO_QUEUE_1K_RESERVED_30_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_30_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_30_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_31_OFFSET 0x0000007c +#define RX_REO_QUEUE_1K_RESERVED_31_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_31_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_31_MASK 0xffffffff + +#endif diff --git a/drivers/staging/fw-api/hw/kiwi/v2/rx_reo_queue_1k.h b/drivers/staging/fw-api/hw/kiwi/v2/rx_reo_queue_1k.h new file mode 100644 index 000000000000..d87020dddf94 --- /dev/null +++ b/drivers/staging/fw-api/hw/kiwi/v2/rx_reo_queue_1k.h @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef _RX_REO_QUEUE_1K_H_ +#define _RX_REO_QUEUE_1K_H_ +#if !defined(__ASSEMBLER__) +#endif + +#include "uniform_descriptor_header.h" +#define NUM_OF_DWORDS_RX_REO_QUEUE_1K 32 + +struct rx_reo_queue_1k { +#ifndef WIFI_BIT_ORDER_BIG_ENDIAN + struct uniform_descriptor_header descriptor_header; + uint32_t rx_bitmap_319_288 : 32; + uint32_t rx_bitmap_351_320 : 32; + uint32_t rx_bitmap_383_352 : 32; + uint32_t rx_bitmap_415_384 : 32; + uint32_t rx_bitmap_447_416 : 32; + uint32_t rx_bitmap_479_448 : 32; + uint32_t rx_bitmap_511_480 : 32; + uint32_t rx_bitmap_543_512 : 32; + uint32_t rx_bitmap_575_544 : 32; + uint32_t rx_bitmap_607_576 : 32; + uint32_t rx_bitmap_639_608 : 32; + uint32_t rx_bitmap_671_640 : 32; + uint32_t rx_bitmap_703_672 : 32; + uint32_t rx_bitmap_735_704 : 32; + uint32_t rx_bitmap_767_736 : 32; + uint32_t rx_bitmap_799_768 : 32; + uint32_t rx_bitmap_831_800 : 32; + uint32_t rx_bitmap_863_832 : 32; + uint32_t rx_bitmap_895_864 : 32; + uint32_t rx_bitmap_927_896 : 32; + uint32_t rx_bitmap_959_928 : 32; + uint32_t rx_bitmap_991_960 : 32; + uint32_t rx_bitmap_1023_992 : 32; + uint32_t reserved_24 : 32; + uint32_t reserved_25 : 32; + uint32_t reserved_26 : 32; + uint32_t reserved_27 : 32; + uint32_t reserved_28 : 32; + uint32_t reserved_29 : 32; + uint32_t reserved_30 : 32; + uint32_t reserved_31 : 32; +#else + struct uniform_descriptor_header descriptor_header; + uint32_t rx_bitmap_319_288 : 32; + uint32_t rx_bitmap_351_320 : 32; + uint32_t rx_bitmap_383_352 : 32; + uint32_t rx_bitmap_415_384 : 32; + uint32_t rx_bitmap_447_416 : 32; + uint32_t rx_bitmap_479_448 : 32; + uint32_t rx_bitmap_511_480 : 32; + uint32_t rx_bitmap_543_512 : 32; + uint32_t rx_bitmap_575_544 : 32; + uint32_t rx_bitmap_607_576 : 32; + uint32_t rx_bitmap_639_608 : 32; + uint32_t rx_bitmap_671_640 : 32; + uint32_t rx_bitmap_703_672 : 32; + uint32_t rx_bitmap_735_704 : 32; + uint32_t rx_bitmap_767_736 : 32; + uint32_t rx_bitmap_799_768 : 32; + uint32_t rx_bitmap_831_800 : 32; + uint32_t rx_bitmap_863_832 : 32; + uint32_t rx_bitmap_895_864 : 32; + uint32_t rx_bitmap_927_896 : 32; + uint32_t rx_bitmap_959_928 : 32; + uint32_t rx_bitmap_991_960 : 32; + uint32_t rx_bitmap_1023_992 : 32; + uint32_t reserved_24 : 32; + uint32_t reserved_25 : 32; + uint32_t reserved_26 : 32; + uint32_t reserved_27 : 32; + uint32_t reserved_28 : 32; + uint32_t reserved_29 : 32; + uint32_t reserved_30 : 32; + uint32_t reserved_31 : 32; +#endif +}; + +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_OFFSET 0x00000000 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_LSB 0 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_MSB 3 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_OWNER_MASK 0x0000000f + +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_OFFSET 0x00000000 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_LSB 4 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_MSB 7 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_BUFFER_TYPE_MASK 0x000000f0 + +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_OFFSET 0x00000000 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_LSB 8 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_MSB 31 +#define RX_REO_QUEUE_1K_DESCRIPTOR_HEADER_RESERVED_0A_MASK 0xffffff00 + +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_OFFSET 0x00000004 +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_319_288_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_OFFSET 0x00000008 +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_351_320_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_OFFSET 0x0000000c +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_383_352_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_OFFSET 0x00000010 +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_415_384_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_OFFSET 0x00000014 +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_447_416_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_OFFSET 0x00000018 +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_479_448_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_OFFSET 0x0000001c +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_511_480_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_OFFSET 0x00000020 +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_543_512_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_OFFSET 0x00000024 +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_575_544_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_OFFSET 0x00000028 +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_607_576_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_OFFSET 0x0000002c +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_639_608_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_OFFSET 0x00000030 +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_671_640_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_OFFSET 0x00000034 +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_703_672_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_OFFSET 0x00000038 +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_735_704_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_OFFSET 0x0000003c +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_767_736_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_OFFSET 0x00000040 +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_799_768_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_OFFSET 0x00000044 +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_831_800_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_OFFSET 0x00000048 +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_863_832_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_OFFSET 0x0000004c +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_895_864_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_OFFSET 0x00000050 +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_927_896_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_OFFSET 0x00000054 +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_959_928_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_OFFSET 0x00000058 +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_991_960_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_OFFSET 0x0000005c +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_LSB 0 +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_MSB 31 +#define RX_REO_QUEUE_1K_RX_BITMAP_1023_992_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_24_OFFSET 0x00000060 +#define RX_REO_QUEUE_1K_RESERVED_24_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_24_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_24_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_25_OFFSET 0x00000064 +#define RX_REO_QUEUE_1K_RESERVED_25_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_25_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_25_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_26_OFFSET 0x00000068 +#define RX_REO_QUEUE_1K_RESERVED_26_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_26_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_26_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_27_OFFSET 0x0000006c +#define RX_REO_QUEUE_1K_RESERVED_27_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_27_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_27_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_28_OFFSET 0x00000070 +#define RX_REO_QUEUE_1K_RESERVED_28_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_28_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_28_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_29_OFFSET 0x00000074 +#define RX_REO_QUEUE_1K_RESERVED_29_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_29_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_29_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_30_OFFSET 0x00000078 +#define RX_REO_QUEUE_1K_RESERVED_30_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_30_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_30_MASK 0xffffffff + +#define RX_REO_QUEUE_1K_RESERVED_31_OFFSET 0x0000007c +#define RX_REO_QUEUE_1K_RESERVED_31_LSB 0 +#define RX_REO_QUEUE_1K_RESERVED_31_MSB 31 +#define RX_REO_QUEUE_1K_RESERVED_31_MASK 0xffffffff + +#endif diff --git a/drivers/staging/greybus/audio_codec.c b/drivers/staging/greybus/audio_codec.c index 6cbf69a57dfd..806be9d86338 100644 --- a/drivers/staging/greybus/audio_codec.c +++ b/drivers/staging/greybus/audio_codec.c @@ -620,8 +620,8 @@ static int gbcodec_mute_stream(struct snd_soc_dai *dai, int mute, int stream) break; } if (!data) { - dev_err(dai->dev, "%s:%s DATA connection missing\n", - dai->name, module->name); + dev_err(dai->dev, "%s DATA connection missing\n", + dai->name); mutex_unlock(&codec->lock); return -ENODEV; } diff --git a/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_cmn_struct.h b/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_cmn_struct.h index 5978cd8dde58..0ee6ebae6294 100644 --- a/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_cmn_struct.h +++ b/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_cmn_struct.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2011-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -288,6 +289,7 @@ enum cdp_host_txrx_stats { TXRX_SOC_INTERRUPT_STATS = 12, TXRX_SOC_FSE_STATS = 13, TXRX_HAL_REG_WRITE_STATS = 14, + TXRX_SOC_REO_HW_DESC_DUMP = 15, TXRX_HOST_STATS_MAX, }; @@ -2251,6 +2253,7 @@ enum cdp_dp_cfg { cfg_dp_tso_enable, cfg_dp_lro_enable, cfg_dp_gro_enable, + cfg_dp_sg_enable, cfg_dp_tx_flow_start_queue_offset, cfg_dp_tx_flow_stop_queue_threshold, cfg_dp_ipa_uc_tx_buf_size, diff --git a/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_mob_def.h b/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_mob_def.h index 579e6a5f3d34..66db818bcd8b 100644 --- a/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_mob_def.h +++ b/drivers/staging/qca-wifi-host-cmn/dp/inc/cdp_txrx_mob_def.h @@ -314,6 +314,7 @@ struct txrx_pdev_cfg_param_t { bool gro_enable; bool tso_enable; bool lro_enable; + bool sg_enable; bool enable_data_stall_detection; bool enable_flow_steering; bool disable_intra_bss_fwd; diff --git a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_main.c b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_main.c index 83fe7b152980..bec138d008d1 100644 --- a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_main.c +++ b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_main.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -347,6 +348,7 @@ const int dp_stats_mapping_table[][STATS_TYPE_MAX] = { {TXRX_FW_STATS_INVALID, TXRX_SOC_INTERRUPT_STATS}, {TXRX_FW_STATS_INVALID, TXRX_SOC_FSE_STATS}, {TXRX_FW_STATS_INVALID, TXRX_HAL_REG_WRITE_STATS}, + {TXRX_FW_STATS_INVALID, TXRX_SOC_REO_HW_DESC_DUMP}, }; /* MCL specific functions */ @@ -7918,6 +7920,10 @@ dp_print_host_stats(struct dp_vdev *vdev, hal_dump_reg_write_stats(pdev->soc->hal_soc); hal_dump_reg_write_srng_stats(pdev->soc->hal_soc); break; + case TXRX_SOC_REO_HW_DESC_DUMP: + dp_get_rx_reo_queue_info((struct cdp_soc_t *)pdev->soc, + vdev->vdev_id); + break; default: dp_info("Wrong Input For TxRx Host Stats"); dp_txrx_stats_help(); @@ -10169,6 +10175,9 @@ static uint32_t dp_get_cfg(struct cdp_soc_t *soc, enum cdp_dp_cfg cfg) case cfg_dp_gro_enable: value = dpsoc->wlan_cfg_ctx->gro_enabled; break; + case cfg_dp_sg_enable: + value = dpsoc->wlan_cfg_ctx->sg_enabled; + break; case cfg_dp_tx_flow_start_queue_offset: value = dpsoc->wlan_cfg_ctx->tx_flow_start_queue_offset; break; diff --git a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.c b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.c index c95b25a365f6..9cc238688bd8 100644 --- a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.c +++ b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -2014,6 +2015,7 @@ QDF_STATUS dp_rx_tid_setup_wifi3(struct dp_peer *peer, int tid, } else { hw_qdesc_vaddr = rx_tid->hw_qdesc_vaddr_unaligned; } + rx_tid->hw_qdesc_vaddr_aligned = hw_qdesc_vaddr; /* TODO: Ensure that sec_type is set before ADDBA is received. * Currently this is set based on htt indication @@ -3893,3 +3895,85 @@ void dp_peer_flush_frags(struct cdp_soc_t *soc_hdl, uint8_t vdev_id, dp_peer_unref_delete(peer); } + +#ifdef DUMP_REO_QUEUE_INFO_IN_DDR +void dp_dump_rx_reo_queue_info( + struct dp_soc *soc, void *cb_ctxt, union hal_reo_status *reo_status) +{ + struct dp_rx_tid *rx_tid = (struct dp_rx_tid *)cb_ctxt; + + if (!rx_tid) + return; + + if (reo_status->fl_cache_status.header.status != + HAL_REO_CMD_SUCCESS) { + dp_err_rl("Rx tid REO HW desc flush failed(%d)", + reo_status->rx_queue_status.header.status); + return; + } + qdf_spin_lock_bh(&rx_tid->tid_lock); + hal_dump_rx_reo_queue_desc(rx_tid->hw_qdesc_vaddr_aligned); + qdf_spin_unlock_bh(&rx_tid->tid_lock); +} + +void dp_send_cache_flush_for_rx_tid( + struct dp_soc *soc, struct dp_peer *peer) +{ + int i; + struct dp_rx_tid *rx_tid; + struct hal_reo_cmd_params params; + + if (!peer) { + dp_err_rl("Peer is NULL"); + return; + } + + for (i = 0; i < DP_MAX_TIDS; i++) { + rx_tid = &peer->rx_tid[i]; + if (!rx_tid) + continue; + qdf_spin_lock_bh(&rx_tid->tid_lock); + if (rx_tid->hw_qdesc_vaddr_aligned) { + qdf_mem_zero(¶ms, sizeof(params)); + params.std.need_status = 1; + params.std.addr_lo = + rx_tid->hw_qdesc_paddr & 0xffffffff; + params.std.addr_hi = + (uint64_t)(rx_tid->hw_qdesc_paddr) >> 32; + params.u.fl_cache_params.flush_no_inval = 0; + if (QDF_STATUS_SUCCESS != + dp_reo_send_cmd( + soc, CMD_FLUSH_CACHE, + ¶ms, dp_dump_rx_reo_queue_info, + (void *)rx_tid)) { + dp_err_rl("cache flush send failed tid %d", + rx_tid->tid); + qdf_spin_unlock_bh(&rx_tid->tid_lock); + break; + } + } + qdf_spin_unlock_bh(&rx_tid->tid_lock); + } +} + +void dp_get_rx_reo_queue_info( + struct cdp_soc_t *soc_hdl, uint8_t vdev_id) +{ + struct dp_soc *soc = (struct dp_soc *)soc_hdl; + struct dp_vdev *vdev = dp_get_vdev_from_soc_vdev_id_wifi3(soc, vdev_id); + struct dp_peer *peer = NULL; + + if (!vdev) { + dp_err_rl("vdev is null for vdev_id: %u", vdev_id); + return; + } + + peer = vdev->vap_bss_peer; + + if (!peer) { + dp_err_rl("Peer is NULL"); + return; + } + dp_send_cache_flush_for_rx_tid(soc, peer); +} +#endif /* DUMP_REO_QUEUE_INFO_IN_DDR */ diff --git a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.h b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.h index a547426c768c..4992673a76aa 100644 --- a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.h +++ b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_peer.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -22,6 +23,10 @@ #include #include "dp_types.h" +#ifdef DUMP_REO_QUEUE_INFO_IN_DDR +#include "hal_reo.h" +#endif + #define DP_INVALID_PEER_ID 0xffff #define DP_FW_PEER_STATS_CMP_TIMEOUT_MSEC 5000 @@ -338,4 +343,55 @@ dp_peer_update_pkt_capture_params(ol_txrx_soc_handle soc, void dp_rx_tid_delete_cb(struct dp_soc *soc, void *cb_ctxt, union hal_reo_status *reo_status); + +#ifdef DUMP_REO_QUEUE_INFO_IN_DDR +/** + * dp_send_cache_flush_for_rx_tid() - Send cache flush cmd to REO per tid + * @soc : dp_soc handle + * @peer: peer + * + * This function is used to send cache flush cmd to reo and + * to register the callback to handle the dumping of the reo + * queue stas from DDR + * + * Return: none + */ +void dp_send_cache_flush_for_rx_tid( + struct dp_soc *soc, struct dp_peer *peer); + +/** + * dp_get_rx_reo_queue_info() - Handler to get rx tid info + * @soc : cdp_soc_t handle + * @vdev_id: vdev id + * + * Handler to get rx tid info from DDR after h/w cache is + * invalidated first using the cache flush cmd. + * + * Return: none + */ +void dp_get_rx_reo_queue_info( + struct cdp_soc_t *soc_hdl, uint8_t vdev_id); + +/** + * dp_dump_rx_reo_queue_info() - Callback function to dump reo queue stats + * @soc : dp_soc handle + * @cb_ctxt - callback context + * @reo_status: vdev id + * + * This is the callback function registered after sending the reo cmd + * to flush the h/w cache and invalidate it. In the callback the reo + * queue desc info is dumped from DDR. + * + * Return: none + */ +void dp_dump_rx_reo_queue_info( + struct dp_soc *soc, void *cb_ctxt, union hal_reo_status *reo_status); + +#else /* DUMP_REO_QUEUE_INFO_IN_DDR */ + +static inline void dp_get_rx_reo_queue_info( + struct cdp_soc_t *soc_hdl, uint8_t vdev_id) +{ +} +#endif /* DUMP_REO_QUEUE_INFO_IN_DDR */ #endif /* _DP_PEER_H_ */ diff --git a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_rx_err.c b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_rx_err.c index 55ff1bd62a15..1d4488010079 100644 --- a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_rx_err.c +++ b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_rx_err.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -729,12 +730,26 @@ void dp_rx_err_handle_bar(struct dp_soc *soc, start_seq_num); } +/** + * dp_rx_bar_frame_handle() - Function to handle err BAR frames + * @soc: core DP main context + * @ring_desc: Hal ring desc + * @rx_desc: dp rx desc + * @mpdu_desc_info: mpdu desc info + * + * Handle the error BAR frames received. Ensure the SOC level + * stats are updated based on the REO error code. The BAR frames + * are further processed by updating the Rx tids with the start + * sequence number (SSN) and BA window size. Desc is returned + * to the free desc list + * + * Return: none + */ static void dp_rx_bar_frame_handle(struct dp_soc *soc, hal_ring_desc_t ring_desc, struct dp_rx_desc *rx_desc, - struct hal_rx_mpdu_desc_info *mpdu_desc_info, - uint8_t error) + struct hal_rx_mpdu_desc_info *mpdu_desc_info) { qdf_nbuf_t nbuf; struct dp_pdev *pdev; @@ -743,6 +758,7 @@ dp_rx_bar_frame_handle(struct dp_soc *soc, uint16_t peer_id; uint8_t *rx_tlv_hdr; uint32_t tid; + uint8_t reo_err_code; nbuf = rx_desc->nbuf; rx_desc_pool = &soc->rx_desc_buf[rx_desc->pool_id]; @@ -763,25 +779,26 @@ dp_rx_bar_frame_handle(struct dp_soc *soc, if (!peer) goto next; + reo_err_code = HAL_RX_REO_ERROR_GET(ring_desc); dp_info("BAR frame: peer = "QDF_MAC_ADDR_FMT " peer_id = %d" " tid = %u" " SSN = %d" - " error status = %d", + " error code = %d", QDF_MAC_ADDR_REF(peer->mac_addr.raw), peer_id, tid, mpdu_desc_info->mpdu_seq, - error); + reo_err_code); - switch (error) { + switch (reo_err_code) { case HAL_REO_ERR_BAR_FRAME_2K_JUMP: DP_STATS_INC(soc, - rx.err.reo_error[error], 1); + rx.err.reo_error[reo_err_code], 1); case HAL_REO_ERR_BAR_FRAME_OOR: dp_rx_err_handle_bar(soc, peer, nbuf); DP_STATS_INC(soc, - rx.err.reo_error[error], 1); + rx.err.reo_error[reo_err_code], 1); break; default: DP_STATS_INC(soc, rx.bar_frame, 1); @@ -1644,8 +1661,7 @@ dp_rx_err_process(struct dp_intr *int_ctx, struct dp_soc *soc, dp_rx_bar_frame_handle(soc, ring_desc, rx_desc, - &mpdu_desc_info, - error); + &mpdu_desc_info); rx_bufs_reaped[mac_id] += 1; goto next_entry; diff --git a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_types.h b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_types.h index f306e1ba1308..87ebc7dc867b 100644 --- a/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_types.h +++ b/drivers/staging/qca-wifi-host-cmn/dp/wifi3.0/dp_types.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -586,6 +587,7 @@ struct dp_rx_tid { uint8_t pn_size; /* REO TID queue descriptors */ void *hw_qdesc_vaddr_unaligned; + void *hw_qdesc_vaddr_aligned; qdf_dma_addr_t hw_qdesc_paddr_unaligned; qdf_dma_addr_t hw_qdesc_paddr; uint32_t hw_qdesc_alloc_size; diff --git a/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_api.h b/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_api.h index 0bb65493f73d..649852dcac21 100644 --- a/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_api.h +++ b/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_api.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -27,6 +28,10 @@ #include "hif_io32.h" #include "qdf_platform.h" +#ifdef DUMP_REO_QUEUE_INFO_IN_DDR +#include "hal_hw_headers.h" +#endif + /* Ring index for WBM2SW2 release ring */ #define HAL_IPA_TX_COMP_RING_IDX 2 @@ -2116,6 +2121,78 @@ void hal_setup_link_idle_list(hal_soc_handle_t hal_soc_hdl, } +#ifdef DUMP_REO_QUEUE_INFO_IN_DDR +/** + * hal_dump_rx_reo_queue_desc() - Dump reo queue descriptor fields + * @hw_qdesc_vaddr_aligned: Pointer to hw reo queue desc virtual addr + * + * Use the virtual addr pointer to reo h/w queue desc to read + * the values from ddr and log them. + * + * Return: none + */ +static inline void hal_dump_rx_reo_queue_desc( + void *hw_qdesc_vaddr_aligned) +{ + struct rx_reo_queue *hw_qdesc = + (struct rx_reo_queue *)hw_qdesc_vaddr_aligned; + + if (!hw_qdesc) + return; + + hal_info("receive_queue_number %u vld %u window_jump_2k %u" + " hole_count %u ba_window_size %u ignore_ampdu_flag %u" + " svld %u ssn %u current_index %u" + " disable_duplicate_detection %u soft_reorder_enable %u" + " chk_2k_mode %u oor_mode %u mpdu_frames_processed_count %u" + " msdu_frames_processed_count %u total_processed_byte_count %u" + " late_receive_mpdu_count %u seq_2k_error_detected_flag %u" + " pn_error_detected_flag %u current_mpdu_count %u" + " current_msdu_count %u timeout_count %u" + " forward_due_to_bar_count %u duplicate_count %u" + " frames_in_order_count %u bar_received_count %u" + " pn_check_needed %u pn_shall_be_even %u" + " pn_shall_be_uneven %u pn_size %u", + hw_qdesc->receive_queue_number, + hw_qdesc->vld, + hw_qdesc->window_jump_2k, + hw_qdesc->hole_count, + hw_qdesc->ba_window_size, + hw_qdesc->ignore_ampdu_flag, + hw_qdesc->svld, + hw_qdesc->ssn, + hw_qdesc->current_index, + hw_qdesc->disable_duplicate_detection, + hw_qdesc->soft_reorder_enable, + hw_qdesc->chk_2k_mode, + hw_qdesc->oor_mode, + hw_qdesc->mpdu_frames_processed_count, + hw_qdesc->msdu_frames_processed_count, + hw_qdesc->total_processed_byte_count, + hw_qdesc->late_receive_mpdu_count, + hw_qdesc->seq_2k_error_detected_flag, + hw_qdesc->pn_error_detected_flag, + hw_qdesc->current_mpdu_count, + hw_qdesc->current_msdu_count, + hw_qdesc->timeout_count, + hw_qdesc->forward_due_to_bar_count, + hw_qdesc->duplicate_count, + hw_qdesc->frames_in_order_count, + hw_qdesc->bar_received_count, + hw_qdesc->pn_check_needed, + hw_qdesc->pn_shall_be_even, + hw_qdesc->pn_shall_be_uneven, + hw_qdesc->pn_size); +} + +#else /* DUMP_REO_QUEUE_INFO_IN_DDR */ + +static inline void hal_dump_rx_reo_queue_desc( + void *hw_qdesc_vaddr_aligned) +{ +} +#endif /* DUMP_REO_QUEUE_INFO_IN_DDR */ + /** * hal_srng_dump_ring_desc() - Dump ring descriptor info * diff --git a/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.c b/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.c index 6f4f8c905fad..6a0b155a022b 100644 --- a/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.c +++ b/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -590,7 +591,7 @@ inline int hal_reo_cmd_flush_cache(hal_ring_handle_t hal_ring_hdl, BLOCK_CACHE_USAGE_AFTER_FLUSH, cp->block_use_after_flush); HAL_DESC_SET_FIELD(reo_desc, REO_FLUSH_CACHE_2, FLUSH_ENTIRE_CACHE, - cp->flush_all); + cp->flush_entire_cache); if (hif_pm_runtime_get(hal_soc->hif_handle, RTPM_ID_HAL_REO_CMD) == 0) { diff --git a/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.h b/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.h index ba7335e41c22..e6d4ba1fede6 100644 --- a/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.h +++ b/drivers/staging/qca-wifi-host-cmn/hal/wifi3.0/hal_reo.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -179,7 +180,7 @@ struct hal_reo_cmd_flush_queue_params { * @cache_block_res_index: Blocking resource to be used * @flush_no_inval: Flush without invalidatig descriptor * @use_after_flush: Block usage after flush till unblock command - * @flush_all: Flush entire REO cache + * @flush_entire_cache: Flush entire REO cache */ struct hal_reo_cmd_flush_cache_params { bool fwd_mpdus_in_queue; @@ -187,7 +188,7 @@ struct hal_reo_cmd_flush_cache_params { uint8_t cache_block_res_index; bool flush_no_inval; bool block_use_after_flush; - bool flush_all; + bool flush_entire_cache; }; /** diff --git a/drivers/staging/qca-wifi-host-cmn/qdf/inc/qdf_types.h b/drivers/staging/qca-wifi-host-cmn/qdf/inc/qdf_types.h index d60cad720eac..87cf7569553f 100644 --- a/drivers/staging/qca-wifi-host-cmn/qdf/inc/qdf_types.h +++ b/drivers/staging/qca-wifi-host-cmn/qdf/inc/qdf_types.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2014-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -1038,6 +1039,22 @@ struct qdf_ipv6_addr { */ QDF_STATUS qdf_ipv6_parse(const char *ipv6_str, struct qdf_ipv6_addr *out_addr); +/** + * qdf_uint32_array_parse() - parse the given string as uint32 array + * @in_str: the input string to parse + * @out_array: the output uint32 array, populated on success + * @array_size: size of the array + * @out_size: size of the populated array + * + * This API is called to convert string (each value separated by + * a comma) into an uint32 array + * + * Return: QDF_STATUS + */ + +QDF_STATUS qdf_uint32_array_parse(const char *in_str, uint32_t *out_array, + qdf_size_t array_size, qdf_size_t *out_size); + /** * qdf_uint16_array_parse() - parse the given string as uint16 array * @in_str: the input string to parse diff --git a/drivers/staging/qca-wifi-host-cmn/qdf/src/qdf_types.c b/drivers/staging/qca-wifi-host-cmn/qdf/src/qdf_types.c index 8f645c74f347..86325250d25e 100644 --- a/drivers/staging/qca-wifi-host-cmn/qdf/src/qdf_types.c +++ b/drivers/staging/qca-wifi-host-cmn/qdf/src/qdf_types.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -616,6 +617,55 @@ QDF_STATUS qdf_ipv6_parse(const char *ipv6_str, struct qdf_ipv6_addr *out_addr) } qdf_export_symbol(qdf_ipv6_parse); +QDF_STATUS qdf_uint32_array_parse(const char *in_str, uint32_t *out_array, + qdf_size_t array_size, qdf_size_t *out_size) +{ + QDF_STATUS status; + bool negate; + qdf_size_t size = 0; + uint64_t value; + + QDF_BUG(in_str); + if (!in_str) + return QDF_STATUS_E_INVAL; + + QDF_BUG(out_array); + if (!out_array) + return QDF_STATUS_E_INVAL; + + QDF_BUG(out_size); + if (!out_size) + return QDF_STATUS_E_INVAL; + + while (size < array_size) { + status = __qdf_int_parse_lazy(&in_str, &value, &negate); + if (QDF_IS_STATUS_ERROR(status)) + return status; + + if ((uint32_t)value != value || negate) + return QDF_STATUS_E_RANGE; + + in_str = qdf_str_left_trim(in_str); + + switch (in_str[0]) { + case ',': + out_array[size++] = value; + in_str++; + break; + case '\0': + out_array[size++] = value; + *out_size = size; + return QDF_STATUS_SUCCESS; + default: + return QDF_STATUS_E_FAILURE; + } + } + + return QDF_STATUS_E_FAILURE; +} + +qdf_export_symbol(qdf_uint32_array_parse); + QDF_STATUS qdf_uint16_array_parse(const char *in_str, uint16_t *out_array, qdf_size_t array_size, qdf_size_t *out_size) { diff --git a/drivers/staging/qca-wifi-host-cmn/qdf/test/qdf_types_test.c b/drivers/staging/qca-wifi-host-cmn/qdf/test/qdf_types_test.c index 1b351e88ccb1..d833a76e6515 100644 --- a/drivers/staging/qca-wifi-host-cmn/qdf/test/qdf_types_test.c +++ b/drivers/staging/qca-wifi-host-cmn/qdf/test/qdf_types_test.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -234,6 +235,80 @@ static uint32_t qdf_types_ut_uint16_array_parse(void) return errors; } +#define ut_uint32_array_pass(str, max_size, exp_arr, exp_arr_size) \ +__ut_uint32_array(str, QDF_STATUS_SUCCESS, max_size, exp_arr, exp_arr_size) + +#define ut_uint32_array_fail(str, max_size, exp_status, exp_arr, exp_arr_size)\ +__ut_uint32_array(str, exp_status, max_size, exp_arr, exp_arr_size) + +static uint32_t +__ut_uint32_array(const char *str, QDF_STATUS exp_status, + uint8_t max_array_size, uint32_t *exp_array, + uint8_t exp_array_size) +{ + uint32_t parsed_array[10]; + qdf_size_t parsed_array_size; + QDF_STATUS status; + uint8_t i; + + status = qdf_uint32_array_parse(str, parsed_array, max_array_size, + &parsed_array_size); + + if (status != exp_status) { + qdf_nofl_alert("FAIL: qdf_uint32_array_parse(\"%s\") -> status %d; expected status %d", + str, status, exp_status); + return 1; + } + + if (QDF_IS_STATUS_ERROR(status)) + return 0; + + if (parsed_array_size != exp_array_size) { + qdf_nofl_alert("FAIL: qdf_uint32_array_parse(\"%s\") -> parsed_array_size %zu; exp_array_size %d", + str, parsed_array_size, exp_array_size); + return 1; + } + + for (i = 0; i < exp_array_size; i++) + if (parsed_array[i] != exp_array[i]) { + qdf_nofl_alert("FAIL: qdf_uint32_array_parse(\"%s\") -> parsed_array[%d] %d; exp_array[%d] %d", + str, i, parsed_array[i], i, + exp_array[i]); + return 1; + } + + return 0; +} + +static uint32_t qdf_types_ut_uint32_array_parse(void) +{ + uint32_t errors = 0; + uint32_t exp_array_value[10] = { 1, 100, 9997, 899965, 65536, 0, + 4294967295, 268435456, + 2164184149, 999999999}; + + errors += ut_uint32_array_pass( + "1, 100, 9997, 899965, 65536, 0, 4294967295, 268435456, 2164184149, 999999999", + 10, exp_array_value, 10); + errors += ut_uint32_array_pass( + "+1, +100, +9997, +899965, +65536, 0, +4294967295, +268435456, +2164184149, +999999999", + 10, exp_array_value, 10); + errors += ut_uint32_array_fail("1;", 10, QDF_STATUS_E_FAILURE, + exp_array_value, 0); + /* Out of range test where 4294967296 is out of range */ + errors += ut_uint32_array_fail( + "1, 100, 9997, 899965, 65536, 0, 4294967296, 268435456, 2164184149, 999999999", + 10, QDF_STATUS_E_RANGE, exp_array_value, 0); + errors += ut_uint32_array_fail( + "-1, -100, -9997, -899965, -65536, 0, -4294967295, -268435456, -2164184149, -999999999", + 10, QDF_STATUS_E_RANGE, exp_array_value, 0); + errors += ut_uint32_array_fail( + "1, 100, 9997, 899965, 65536, 日本, 0, 4294967295, 268435456, 999999999", + 10, QDF_STATUS_E_FAILURE, exp_array_value, 0); + + return errors; +} + #define ut_uint32_pass(str, exp) __ut_uint32(str, QDF_STATUS_SUCCESS, exp) #define ut_uint32_fail(str, exp_status) __ut_uint32(str, exp_status, 0) @@ -581,6 +656,7 @@ uint32_t qdf_types_unit_test(void) errors += qdf_types_ut_ipv4_parse(); errors += qdf_types_ut_ipv6_parse(); errors += qdf_types_ut_uint16_array_parse(); + errors += qdf_types_ut_uint32_array_parse(); return errors; } diff --git a/drivers/staging/qca-wifi-host-cmn/umac/mlme/vdev_mgr/dispatcher/inc/wlan_vdev_mgr_tgt_if_tx_defs.h b/drivers/staging/qca-wifi-host-cmn/umac/mlme/vdev_mgr/dispatcher/inc/wlan_vdev_mgr_tgt_if_tx_defs.h index b0f041b1d410..51283ab146fd 100644 --- a/drivers/staging/qca-wifi-host-cmn/umac/mlme/vdev_mgr/dispatcher/inc/wlan_vdev_mgr_tgt_if_tx_defs.h +++ b/drivers/staging/qca-wifi-host-cmn/umac/mlme/vdev_mgr/dispatcher/inc/wlan_vdev_mgr_tgt_if_tx_defs.h @@ -331,6 +331,7 @@ struct config_fils_params { * @lower32: Lower 32 bits in the 1st 64-bit value * @higher32: Higher 32 bits in the 1st 64-bit value * @lower32_2: Lower 32 bits in the 2nd 64-bit value + * @higher32_2: Higher 32 bits in the 2nd 64-bit value */ struct config_ratemask_params { uint8_t vdev_id; @@ -338,6 +339,7 @@ struct config_ratemask_params { uint32_t lower32; uint32_t higher32; uint32_t lower32_2; + uint32_t higher32_2; }; /** diff --git a/drivers/staging/qca-wifi-host-cmn/wlan_cfg/cfg_dp.h b/drivers/staging/qca-wifi-host-cmn/wlan_cfg/cfg_dp.h index dd9e4a4329c7..ecc2bff5706e 100644 --- a/drivers/staging/qca-wifi-host-cmn/wlan_cfg/cfg_dp.h +++ b/drivers/staging/qca-wifi-host-cmn/wlan_cfg/cfg_dp.h @@ -578,6 +578,22 @@ CFG_INI_BOOL("LROEnable", WLAN_LRO_ENABLE, \ "DP LRO Enable") +/* + * + * CFG_DP_SG - Enable the SG feature standalonely + * @Min: 0 + * @Max: 1 + * @Default: 1 + * + * This ini entry is used to enable/disable SG feature standalonely. + * Also does Rome support SG on TX, lithium does not. + * For example the lithium does not support SG on UDP frames. + * Which is able to handle SG only for TSO frames(in case TSO is enabled). + * + * Usage: External + * + * + */ #define CFG_DP_SG \ CFG_INI_BOOL("dp_sg_support", false, \ "DP SG Enable") diff --git a/drivers/staging/qca-wifi-host-cmn/wmi/inc/wmi_unified_param.h b/drivers/staging/qca-wifi-host-cmn/wmi/inc/wmi_unified_param.h index 3bfb1f07f481..6384d3d58656 100644 --- a/drivers/staging/qca-wifi-host-cmn/wmi/inc/wmi_unified_param.h +++ b/drivers/staging/qca-wifi-host-cmn/wmi/inc/wmi_unified_param.h @@ -3494,6 +3494,22 @@ enum wmi_host_preamble_type { WMI_HOST_PREAMBLE_HE = 4, }; +/** + * enum wmi_ratemask_type: ratemask type + * @WMI_RATEMASK_TYPE_CCK: CCK rate mask type + * @WMI_RATEMASK_TYPE_HT: HT rate mask type + * @WMI_RATEMASK_TYPE_VHT: VHT rate mask type + * @WMI_RATEMASK_TYPE_HE: HE rate mask type + * + * This is used for 'type' in WMI_VDEV_RATEMASK_CMDID + */ +enum wmi_ratemask_type { + WMI_RATEMASK_TYPE_CCK = 0, + WMI_RATEMASK_TYPE_HT = 1, + WMI_RATEMASK_TYPE_VHT = 2, + WMI_RATEMASK_TYPE_HE = 3, +}; + /** * struct packet_power_info_params - packet power info params * @chainmask: chain mask diff --git a/drivers/staging/qca-wifi-host-cmn/wmi/src/wmi_unified_vdev_tlv.c b/drivers/staging/qca-wifi-host-cmn/wmi/src/wmi_unified_vdev_tlv.c index a49ba9283fbb..7e988e066c3e 100644 --- a/drivers/staging/qca-wifi-host-cmn/wmi/src/wmi_unified_vdev_tlv.c +++ b/drivers/staging/qca-wifi-host-cmn/wmi/src/wmi_unified_vdev_tlv.c @@ -46,6 +46,7 @@ send_vdev_config_ratemask_cmd_tlv(struct wmi_unified *wmi_handle, cmd->mask_lower32 = param->lower32; cmd->mask_higher32 = param->higher32; cmd->mask_lower32_2 = param->lower32_2; + cmd->mask_higher32_2 = param->higher32_2; wmi_mtrace(WMI_VDEV_RATEMASK_CMDID, cmd->vdev_id, 0); if (wmi_unified_cmd_send(wmi_handle, buf, len, diff --git a/drivers/staging/qcacld-3.0/Kbuild b/drivers/staging/qcacld-3.0/Kbuild index cf3aa7892cec..98ee53a384c0 100644 --- a/drivers/staging/qcacld-3.0/Kbuild +++ b/drivers/staging/qcacld-3.0/Kbuild @@ -3324,6 +3324,7 @@ cppflags-$(CONFIG_HIF_CPU_PERF_AFFINE_MASK) += -DHIF_CPU_PERF_AFFINE_MASK cppflags-$(CONFIG_SMMU_S1_UNMAP) += -DCONFIG_SMMU_S1_UNMAP cppflags-$(CONFIG_GENERIC_SHADOW_REGISTER_ACCESS_ENABLE) += -DGENERIC_SHADOW_REGISTER_ACCESS_ENABLE cppflags-$(CONFIG_IPA_SET_RESET_TX_DB_PA) += -DIPA_SET_RESET_TX_DB_PA +cppflags-$(CONFIG_DUMP_REO_QUEUE_INFO_IN_DDR) += -DDUMP_REO_QUEUE_INFO_IN_DDR KBUILD_CPPFLAGS += $(cppflags-y) diff --git a/drivers/staging/qcacld-3.0/components/blacklist_mgr/core/src/wlan_blm_core.c b/drivers/staging/qcacld-3.0/components/blacklist_mgr/core/src/wlan_blm_core.c index c0714d41800f..d228984dd611 100644 --- a/drivers/staging/qcacld-3.0/components/blacklist_mgr/core/src/wlan_blm_core.c +++ b/drivers/staging/qcacld-3.0/components/blacklist_mgr/core/src/wlan_blm_core.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2011-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the diff --git a/drivers/staging/qcacld-3.0/components/mlme/core/inc/wlan_mlme_main.h b/drivers/staging/qcacld-3.0/components/mlme/core/inc/wlan_mlme_main.h index 86f85984a91b..e98e7a9223f0 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/core/inc/wlan_mlme_main.h +++ b/drivers/staging/qcacld-3.0/components/mlme/core/inc/wlan_mlme_main.h @@ -608,4 +608,13 @@ QDF_STATUS mlme_get_cfg_wlm_reset(struct wlan_objmgr_psoc *psoc, */ void mlme_reinit_control_config_lfr_params(struct wlan_objmgr_psoc *psoc, struct wlan_mlme_lfr_cfg *lfr); + +/** + * wlan_is_vdev_id_up() - check if vdev id is in UP state + * @pdev: Pointer to pdev + * @vdev_id: vdev id + * + * Return: if vdev is up + */ +bool wlan_is_vdev_id_up(struct wlan_objmgr_pdev *pdev, uint8_t vdev_id); #endif diff --git a/drivers/staging/qcacld-3.0/components/mlme/core/src/wlan_mlme_main.c b/drivers/staging/qcacld-3.0/components/mlme/core/src/wlan_mlme_main.c index 274005ecd442..9035571d1a0c 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/core/src/wlan_mlme_main.c +++ b/drivers/staging/qcacld-3.0/components/mlme/core/src/wlan_mlme_main.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -234,6 +234,42 @@ static void mlme_init_chainmask_cfg(struct wlan_objmgr_psoc *psoc, cfg_get(psoc, CFG_ENABLE_BT_CHAIN_SEPARATION); } +static void mlme_init_ratemask_cfg(struct wlan_objmgr_psoc *psoc, + struct wlan_mlme_ratemask *ratemask_cfg) +{ + uint32_t masks[CFG_MLME_RATE_MASK_LEN] = { 0 }; + qdf_size_t len = 0; + QDF_STATUS status; + + ratemask_cfg->type = cfg_get(psoc, CFG_RATEMASK_TYPE); + if ((ratemask_cfg->type <= WLAN_MLME_RATEMASK_TYPE_NO_MASK) || + (ratemask_cfg->type >= WLAN_MLME_RATEMASK_TYPE_MAX)) { + mlme_legacy_debug("Ratemask disabled"); + return; + } + + status = qdf_uint32_array_parse(cfg_get(psoc, CFG_RATEMASK_SET), + masks, + CFG_MLME_RATE_MASK_LEN, + &len); + + if (status != QDF_STATUS_SUCCESS || len != CFG_MLME_RATE_MASK_LEN) { + /* Do not enable ratemaks if config is invalid */ + ratemask_cfg->type = WLAN_MLME_RATEMASK_TYPE_NO_MASK; + mlme_legacy_err("Failed to parse ratemask"); + return; + } + + ratemask_cfg->lower32 = masks[0]; + ratemask_cfg->higher32 = masks[1]; + ratemask_cfg->lower32_2 = masks[2]; + ratemask_cfg->higher32_2 = masks[3]; + mlme_legacy_debug("Ratemask type: %d, masks:0x%x, 0x%x, 0x%x, 0x%x", + ratemask_cfg->type, ratemask_cfg->lower32, + ratemask_cfg->higher32, ratemask_cfg->lower32_2, + ratemask_cfg->higher32_2); +} + #ifdef WLAN_FEATURE_11W static void mlme_init_pmf_cfg(struct wlan_objmgr_psoc *psoc, struct wlan_mlme_generic *gen) @@ -1059,13 +1095,13 @@ static void mlme_init_he_cap_in_cfg(struct wlan_objmgr_psoc *psoc, he_caps->dot11_he_cap.rx_full_bw_su_he_mu_non_cmpr_sigb = cfg_default(CFG_HE_RX_FULL_BW_MU_NON_CMPR_SIGB); he_caps->dot11_he_cap.rx_he_mcs_map_lt_80 = - cfg_default(CFG_HE_RX_MCS_MAP_LT_80); + cfg_get(psoc, CFG_HE_RX_MCS_MAP_LT_80); he_caps->dot11_he_cap.tx_he_mcs_map_lt_80 = - cfg_default(CFG_HE_TX_MCS_MAP_LT_80); - value = cfg_default(CFG_HE_RX_MCS_MAP_160); + cfg_get(psoc, CFG_HE_TX_MCS_MAP_LT_80); + value = cfg_get(psoc, CFG_HE_RX_MCS_MAP_160); qdf_mem_copy(he_caps->dot11_he_cap.rx_he_mcs_map_160, &value, sizeof(uint16_t)); - value = cfg_default(CFG_HE_TX_MCS_MAP_160); + value = cfg_get(psoc, CFG_HE_TX_MCS_MAP_160); qdf_mem_copy(he_caps->dot11_he_cap.tx_he_mcs_map_160, &value, sizeof(uint16_t)); value = cfg_default(CFG_HE_RX_MCS_MAP_80_80); @@ -1202,6 +1238,8 @@ static void mlme_init_obss_ht40_cfg(struct wlan_objmgr_psoc *psoc, (bool)cfg_default(CFG_OBSS_DETECTION_OFFLOAD); obss_ht40->obss_color_collision_offload_enabled = (bool)cfg_default(CFG_OBSS_COLOR_COLLISION_OFFLOAD); + obss_ht40->bss_color_collision_det_sta = + cfg_get(psoc, CFG_BSS_CLR_COLLISION_DETCN_STA); } static void mlme_init_threshold_cfg(struct wlan_objmgr_psoc *psoc, @@ -2447,6 +2485,7 @@ QDF_STATUS mlme_cfg_on_psoc_enable(struct wlan_objmgr_psoc *psoc) mlme_init_reg_cfg(psoc, &mlme_cfg->reg); mlme_init_btm_cfg(psoc, &mlme_cfg->btm); mlme_init_roam_score_config(psoc, mlme_cfg); + mlme_init_ratemask_cfg(psoc, &mlme_cfg->ratemask_cfg); return status; } @@ -3076,4 +3115,22 @@ void mlme_set_roam_state(struct wlan_objmgr_psoc *psoc, uint8_t vdev_id, mlme_priv->mlme_roam.roam_sm.state = new_state; wlan_objmgr_vdev_release_ref(vdev, WLAN_MLME_OBJMGR_ID); } + +bool wlan_is_vdev_id_up(struct wlan_objmgr_pdev *pdev, uint8_t vdev_id) +{ + struct wlan_objmgr_vdev *vdev; + bool is_up = false; + + if (!pdev) + return is_up; + + vdev = wlan_objmgr_get_vdev_by_id_from_pdev(pdev, vdev_id, + WLAN_LEGACY_MAC_ID); + if (vdev) { + is_up = QDF_IS_STATUS_SUCCESS(wlan_vdev_is_up(vdev)); + wlan_objmgr_vdev_release_ref(vdev, WLAN_LEGACY_MAC_ID); + } + + return is_up; +} #endif diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_he_caps.h b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_he_caps.h index 7fd251e041e7..ecba16b1e8a4 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_he_caps.h +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_he_caps.h @@ -507,35 +507,158 @@ 0, \ "He Rx Full Bw Mu Non Cmpr Sigb") -#define CFG_HE_RX_MCS_MAP_LT_80 CFG_UINT( \ +/* 11AX related INI configuration */ +/* + * + * he_rx_mcs_map_lt_80 - configure Rx HE-MCS Map for ≤ 80 MHz + * @Min: 0 + * @Max: 0xFFFF + * @Default: 0xFFFA + * + * This ini is used to configure Rx HE-MCS Map for ≤ 80 MHz + * 0:1 Max HE-MCS For 1 SS + * 2:3 Max HE-MCS For 2 SS + * 4:5 Max HE-MCS For 3 SS + * 6:7 Max HE-MCS For 4 SS + * 8:9 Max HE-MCS For 5 SS + * 10:11 Max HE-MCS For 6 SS + * 12:13 Max HE-MCS For 7 SS + * 14:15 Max HE-MCS For 8 SS + * + * 0 indicates support for HE-MCS 0-7 for n spatial streams + * 1 indicates support for HE-MCS 0-9 for n spatial streams + * 2 indicates support for HE-MCS 0-11 for n spatial streams + * 3 indicates that n spatial streams is not supported for HE PPDUs + * + * Related: NA + * + * Supported Feature: 11AX + * + * Usage: External + * + * + */ +#define CFG_HE_RX_MCS_MAP_LT_80 CFG_INI_UINT( \ "he_rx_mcs_map_lt_80", \ 0, \ 0xFFFF, \ - 0xFFF0, \ + 0xFFFA, \ CFG_VALUE_OR_DEFAULT, \ "He Rx Mcs Map Lt 80") -#define CFG_HE_TX_MCS_MAP_LT_80 CFG_UINT( \ +/* 11AX related INI configuration */ +/* + * + * he_tx_mcs_map_lt_80 - configure Tx HE-MCS Map for ≤ 80 MHz + * @Min: 0 + * @Max: 0xFFFF + * @Default: 0xFFFA + * + * This ini is used to configure Tx HE-MCS Map for ≤ 80 MHz + * 0:1 Max HE-MCS For 1 SS + * 2:3 Max HE-MCS For 2 SS + * 4:5 Max HE-MCS For 3 SS + * 6:7 Max HE-MCS For 4 SS + * 8:9 Max HE-MCS For 5 SS + * 10:11 Max HE-MCS For 6 SS + * 12:13 Max HE-MCS For 7 SS + * 14:15 Max HE-MCS For 8 SS + * + * 0 indicates support for HE-MCS 0-7 for n spatial streams + * 1 indicates support for HE-MCS 0-9 for n spatial streams + * 2 indicates support for HE-MCS 0-11 for n spatial streams + * 3 indicates that n spatial streams is not supported for HE PPDUs + * + * Related: NA + * + * Supported Feature: 11AX + * + * Usage: External + * + * + */ +#define CFG_HE_TX_MCS_MAP_LT_80 CFG_INI_UINT( \ "he_tx_mcs_map_lt_80", \ 0, \ 0xFFFF, \ - 0xFFF0, \ + 0xFFFA, \ CFG_VALUE_OR_DEFAULT, \ "He Tx Mcs Map Lt 80") - -#define CFG_HE_RX_MCS_MAP_160 CFG_UINT( \ +/* 11AX related INI configuration */ +/* + * + * he_rx_mcs_map_160 - configure Rx HE-MCS Map for 160 MHz + * @Min: 0 + * @Max: 0xFFFF + * @Default: 0xFFFA + * + * This ini is used to configure Rx HE-MCS Map for 160 MHz + * 0:1 Max HE-MCS For 1 SS + * 2:3 Max HE-MCS For 2 SS + * 4:5 Max HE-MCS For 3 SS + * 6:7 Max HE-MCS For 4 SS + * 8:9 Max HE-MCS For 5 SS + * 10:11 Max HE-MCS For 6 SS + * 12:13 Max HE-MCS For 7 SS + * 14:15 Max HE-MCS For 8 SS + * + * 0 indicates support for HE-MCS 0-7 for n spatial streams + * 1 indicates support for HE-MCS 0-9 for n spatial streams + * 2 indicates support for HE-MCS 0-11 for n spatial streams + * 3 indicates that n spatial streams is not supported for HE PPDUs + * + * Related: NA + * + * Supported Feature: 11AX + * + * Usage: External + * + * + */ +#define CFG_HE_RX_MCS_MAP_160 CFG_INI_UINT( \ "he_rx_mcs_map_160", \ 0, \ 0xFFFF, \ - 0xFFF0, \ + 0xFFFA, \ CFG_VALUE_OR_DEFAULT, \ "He Rx Mcs Map 160") -#define CFG_HE_TX_MCS_MAP_160 CFG_UINT( \ +/* 11AX related INI configuration */ +/* + * + * he_tx_mcs_map_160 - configure Tx HE-MCS Map for 160 MHz + * @Min: 0 + * @Max: 0xFFFF + * @Default: 0xFFFA + * + * This ini is used to configure Tx HE-MCS Map for 160 MHz + * 0:1 Max HE-MCS For 1 SS + * 2:3 Max HE-MCS For 2 SS + * 4:5 Max HE-MCS For 3 SS + * 6:7 Max HE-MCS For 4 SS + * 8:9 Max HE-MCS For 5 SS + * 10:11 Max HE-MCS For 6 SS + * 12:13 Max HE-MCS For 7 SS + * 14:15 Max HE-MCS For 8 SS + * + * 0 indicates support for HE-MCS 0-7 for n spatial streams + * 1 indicates support for HE-MCS 0-9 for n spatial streams + * 2 indicates support for HE-MCS 0-11 for n spatial streams + * 3 indicates that n spatial streams is not supported for HE PPDUs + * + * Related: NA + * + * Supported Feature: 11AX + * + * Usage: External + * + * + */ +#define CFG_HE_TX_MCS_MAP_160 CFG_INI_UINT( \ "he_tx_mcs_map_160", \ 0, \ 0xFFFF, \ - 0xFFF0, \ + 0xFFFA, \ CFG_VALUE_OR_DEFAULT, \ "He Tx Mcs Map 160") diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_obss_ht40.h b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_obss_ht40.h index 284b49fdb91e..cb7a5873ff67 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_obss_ht40.h +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_obss_ht40.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2012-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -243,6 +244,27 @@ 0, \ "Enable obss color collision offload") +/* + * + * bss_color_collision_det_sta - Enables BSS color collision detection in STA + * @Min: 0 + * @Max: 1 + * @Default: 1 + * + * This ini used to enable or disable the BSS color collision detection in + * STA mode if obss_color_collision_offload is enabled. + * + * Supported Feature: STA + * + * Usage: Internal/External + * + * + */ +#define CFG_BSS_CLR_COLLISION_DETCN_STA CFG_INI_BOOL( \ + "bss_color_collision_det_sta", \ + 1, \ + "BSS color collision detection in STA") + #define CFG_OBSS_HT40_ALL \ CFG(CFG_OBSS_HT40_SCAN_ACTIVE_DWELL_TIME) \ CFG(CFG_OBSS_HT40_SCAN_PASSIVE_DWELL_TIME) \ @@ -253,6 +275,7 @@ CFG(CFG_OBSS_HT40_WIDTH_CH_TRANSITION_DELAY) \ CFG(CFG_OBSS_HT40_OVERRIDE_HT40_20_24GHZ) \ CFG(CFG_OBSS_DETECTION_OFFLOAD) \ + CFG(CFG_BSS_CLR_COLLISION_DETCN_STA) \ CFG(CFG_OBSS_COLOR_COLLISION_OFFLOAD) #endif /* CFG_MLME_OBSS_HT40_H__ */ diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_rates.h b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_rates.h index eb886a368d59..04bd31bc1bf7 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_rates.h +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_rates.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2012-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -30,6 +31,7 @@ #define CFG_SUPPORTED_MCS_SET_LEN 16 #define CFG_BASIC_MCS_SET_LEN 16 #define CFG_CURRENT_MCS_SET_LEN 16 +#define CFG_MLME_RATE_MASK_LEN 4 /* * @@ -221,6 +223,78 @@ CFG_CURRENT_MCS_SET_DATA, \ "current MCS set") +/* + * + * ratemask_type - PHY type for the ratemask. + * @Min: 0 No rate mask set defined - disabled the configuration + * @Max: 4 + * @Default: 0 + * + * This ini is used to set the PHY type for ratemask in rate selection. + * + * 0 = Disables the configuration + * 1 = The rate mask specified is for CCK/OFDM configuration + * 2 = The rate mask specified is for HT configuration + * 3 = The rate mask specified is for VHT configuration + * 4 = The rate mask specified is for HE/11ax configuration + * + * Related: CFG_RATEMASK_SET + * + * Usage: External + */ +#define CFG_RATEMASK_TYPE CFG_INI_UINT( \ + "ratemask_type", \ + 0, \ + 4, \ + 0, \ + CFG_VALUE_OR_DEFAULT, \ + "Ratemask type") + +/* + * + * ratemask_set - ratemasks for a PHY type used in rate selection + * @Min: default data length of ratemask in string format + * @Max: default data length of ratemask in string format + * @Default: 0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF, 0xFFFFFFFF + * + * This is used to set the rate mask value to be used in rate selection. + * Each of the four words must be configured. + * A bit value of 1 represents rate is enabled + * A bit value of 0 represents rate is disabled + * + * [b31-b0],[b63-b32],[b95-b64],[b127-b96] + * For HE targets, 12 bits correpond to one NSS setting. Ex: + * b0-13 => NSS1, MCS 0-13 + * b14-27 => NSS2, MCS 0-13 and so on for other NSS. + * Note that the bit representation is continuous. + * + * For VHT targets, 10 bits correspond to one NSS setting. + * b0-9 => NSS1, MCS 0-9 + * b10-19 => NSS2, MCS 0-9 and so on for other NSS. + * + * For HT targets, 8 bits correspond to one NSS setting. + * b0-7 => NSS1, MCS 0-7 + * b8-15 => NSS2, MCS 0-7 and so on for other NSS. + * + * For OFDM/CCK targets, 8 bits correspond to one NSS setting. + * Bit position |-b3-|-b2-|-b1-|-b0-| + * Rates in Mbps |-1 -|-2 -|-5.5|-11-| CCK Rates + * + * Bit position |-b11-|-b10-|-b09-|-b08-|-b07-|-b06-|-b05-|-b04-| + * Rates in Mbps |- 9 -|- 18-|-36 -|-54 -|- 6 -|-12 -| -24-|-48- | OFDM Rates + * + * Related: CFG_RATEMASK_TYPE + * + * Usage: External + */ +#define CFG_RATEMASK_DATA "0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF" +#define CFG_RATEMASK_SET CFG_INI_STRING( \ + "ratemask_set", \ + 0, \ + sizeof(CFG_RATEMASK_DATA) - 1, \ + CFG_RATEMASK_DATA, \ + "Ratemasks for rate selection") + #define CFG_RATES_ALL \ CFG(CFG_MAX_HT_MCS_FOR_TX_DATA) \ CFG(CFG_DISABLE_ABG_RATE_FOR_TX_DATA) \ @@ -232,6 +306,8 @@ CFG(CFG_SUPPORTED_RATES_11A) \ CFG(CFG_SUPPORTED_MCS_SET) \ CFG(CFG_BASIC_MCS_SET) \ - CFG(CFG_CURRENT_MCS_SET) + CFG(CFG_CURRENT_MCS_SET) \ + CFG(CFG_RATEMASK_TYPE) \ + CFG(CFG_RATEMASK_SET) #endif /* __CFG_MLME_RATES_H */ diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_vht_caps.h b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_vht_caps.h index 3c20e6acaaf2..cc4bb01740fc 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_vht_caps.h +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/cfg_mlme_vht_caps.h @@ -597,6 +597,26 @@ 0, \ "Enable subfee in vendor vht ie") +/* + * + * enable_vhtmcs_10_11_support - Enable/Disable vht mcs 10, 11 support + * @Min: 0 + * @Max: 1 + * @Default: 1 + * + * This ini is used to enable/disable mcs 10, 11 support. + * + * Related: NA + * + * Usage: Internal + * + * + */ +#define CFG_ENABLE_VHT_MCS_10_11 CFG_INI_BOOL( \ + "enable_vhtmcs_10_11_support", \ + 1, \ + "Enable/Disable vht mcs 10, 11 support") + #define CFG_VHT_CAPS_ALL \ CFG(CFG_VHT_SUPP_CHAN_WIDTH) \ CFG(CFG_VHT_SU_BEAMFORMEE_CAP) \ @@ -630,6 +650,7 @@ CFG(CFG_ENABLE_SUBFEE_IN_VENDOR_VHTIE) \ CFG(CFG_TX_BF_CAP) \ CFG(CFG_AS_CAP) \ - CFG(CFG_DISABLE_LDPC_WITH_TXBF_AP) + CFG(CFG_DISABLE_LDPC_WITH_TXBF_AP) \ + CFG(CFG_ENABLE_VHT_MCS_10_11) #endif /* __CFG_MLME_VHT_CAPS_H */ diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_public_struct.h b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_public_struct.h index ba509b90a535..adfcd8b68082 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_public_struct.h +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_public_struct.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -1123,6 +1124,42 @@ struct wlan_mlme_chainmask { bool enable_bt_chain_separation; }; +/** + * enum wlan_mlme_ratemask_type: Type of PHY for ratemask + * @WLAN_MLME_RATEMASK_TYPE_NO_MASK: no ratemask set + * @WLAN_MLME_RATEMASK_TYPE_CCK: CCK/OFDM rate + * @WLAN_MLEM_RATEMASK_TYPE_HT: HT rate + * @WLAN_MLME_RATEMASK_TYPE_VHT: VHT rate + * @WLAN_MLME_RATEMASK_TYPE_HE: HE rate + * + * This is used for 'type' values in wlan_mlme_ratemask + */ +enum wlan_mlme_ratemask_type { + WLAN_MLME_RATEMASK_TYPE_NO_MASK = 0, + WLAN_MLME_RATEMASK_TYPE_CCK = 1, + WLAN_MLME_RATEMASK_TYPE_HT = 2, + WLAN_MLME_RATEMASK_TYPE_VHT = 3, + WLAN_MLME_RATEMASK_TYPE_HE = 4, + /* keep this last */ + WLAN_MLME_RATEMASK_TYPE_MAX, +}; + +/** + * struct wlan_mlme_ratemask - ratemask config parameters + * @type: Type of PHY the mask to be applied + * @lower32: Lower 32 bits in the 1st 64-bit value + * @higher32: Higher 32 bits in the 1st 64-bit value + * @lower32_2: Lower 32 bits in the 2nd 64-bit value + * @higher32_2: Higher 32 bits in the 2nd 64-bit value + */ +struct wlan_mlme_ratemask { + enum wlan_mlme_ratemask_type type; + uint32_t lower32; + uint32_t higher32; + uint32_t lower32_2; + uint32_t higher32_2; +}; + /* struct wlan_mlme_generic - Generic CFG config items * * @band_capability: HW Band Capability - Both or 2.4G only or 5G only @@ -1299,6 +1336,7 @@ struct wlan_mlme_cfg_twt { * @is_override_ht20_40_24g: use channel bonding in 2.4 GHz * @obss_detection_offload_enabled: Enable OBSS detection offload * @obss_color_collision_offload_enabled: Enable obss color collision + * @bss_color_collision_det_sta: STA BSS color collision detection offload */ struct wlan_mlme_obss_ht40 { uint32_t active_dwelltime; @@ -1311,6 +1349,7 @@ struct wlan_mlme_obss_ht40 { bool is_override_ht20_40_24g; bool obss_detection_offload_enabled; bool obss_color_collision_offload_enabled; + bool bss_color_collision_det_sta; }; /** @@ -2401,6 +2440,7 @@ struct wlan_mlme_cfg { struct wlan_mlme_reg reg; struct roam_trigger_score_delta trig_score_delta[NUM_OF_ROAM_TRIGGERS]; struct roam_trigger_min_rssi trig_min_rssi[NUM_OF_ROAM_MIN_RSSI]; + struct wlan_mlme_ratemask ratemask_cfg; }; enum pkt_origin { diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_ucfg_api.h b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_ucfg_api.h index ee106928928d..b91b7ea8b5ce 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_ucfg_api.h +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/inc/wlan_mlme_ucfg_api.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -3980,6 +3981,18 @@ QDF_STATUS ucfg_mlme_set_obss_color_collision_offload_enabled( struct wlan_objmgr_psoc *psoc, uint8_t value); +/** + * ucfg_mlme_set_bss_color_collision_det_sta() - Enable bss color + * collision detection offload for STA mode + * @psoc: pointer to psoc object + * @value: enable or disable + * + * Return: QDF Status + */ +QDF_STATUS +ucfg_mlme_set_bss_color_collision_det_sta(struct wlan_objmgr_psoc *psoc, + uint8_t value); + /** * ucfg_mlme_set_restricted_80p80_bw_supp() - Set the restricted 80p80 support * @psoc: pointer to psoc object diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_api.c b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_api.c index ba7694e78860..2fa9d7df4928 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_api.c +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_api.c @@ -537,6 +537,32 @@ QDF_STATUS wlan_mlme_cfg_get_enable_ul_ofdm(struct wlan_objmgr_psoc *psoc, return QDF_STATUS_SUCCESS; } +/* mlme_get_min_rate_cap() - get minimum capability for HE-MCS between + * ini value and fw capability. + * + * Rx HE-MCS Map and Tx HE-MCS Map subfields format where 2-bit indicates + * 0 indicates support for HE-MCS 0-7 for n spatial streams + * 1 indicates support for HE-MCS 0-9 for n spatial streams + * 2 indicates support for HE-MCS 0-11 for n spatial streams + * 3 indicates that n spatial streams is not supported for HE PPDUs + * + */ +static uint16_t mlme_get_min_rate_cap(uint16_t val1, uint16_t val2) +{ + uint16_t ret = 0, i; + + for (i = 0; i < 8; i++) { + if (((val1 >> (2 * i)) & 0x3) == 0x3 || + ((val2 >> (2 * i)) & 0x3) == 0x3) { + ret |= 0x3 << (2 * i); + continue; + } + ret |= QDF_MIN((val1 >> (2 * i)) & 0x3, + (val2 >> (2 * i)) & 0x3) << (2 * i); + } + return ret; +} + QDF_STATUS mlme_update_tgt_he_caps_in_cfg(struct wlan_objmgr_psoc *psoc, struct wma_tgt_cfg *wma_cfg) { @@ -781,8 +807,12 @@ QDF_STATUS mlme_update_tgt_he_caps_in_cfg(struct wlan_objmgr_psoc *psoc, mlme_obj->cfg.he_caps.dot11_he_cap.rx_full_bw_su_he_mu_non_cmpr_sigb = he_cap->rx_full_bw_su_he_mu_non_cmpr_sigb; - tx_mcs_map = he_cap->tx_he_mcs_map_lt_80; - rx_mcs_map = he_cap->rx_he_mcs_map_lt_80; + tx_mcs_map = mlme_get_min_rate_cap( + mlme_obj->cfg.he_caps.dot11_he_cap.tx_he_mcs_map_lt_80, + he_cap->tx_he_mcs_map_lt_80); + rx_mcs_map = mlme_get_min_rate_cap( + mlme_obj->cfg.he_caps.dot11_he_cap.rx_he_mcs_map_lt_80, + he_cap->rx_he_mcs_map_lt_80); if (!mlme_obj->cfg.vht_caps.vht_cap_info.enable2x2) { nss = 2; tx_mcs_map = HE_SET_MCS_4_NSS(tx_mcs_map, HE_MCS_DISABLE, nss); @@ -795,8 +825,12 @@ QDF_STATUS mlme_update_tgt_he_caps_in_cfg(struct wlan_objmgr_psoc *psoc, if (cfg_in_range(CFG_HE_TX_MCS_MAP_LT_80, tx_mcs_map)) mlme_obj->cfg.he_caps.dot11_he_cap.tx_he_mcs_map_lt_80 = tx_mcs_map; - tx_mcs_map = *((uint16_t *)he_cap->tx_he_mcs_map_160); - rx_mcs_map = *((uint16_t *)he_cap->rx_he_mcs_map_160); + tx_mcs_map = mlme_get_min_rate_cap( + *((uint16_t *)mlme_obj->cfg.he_caps.dot11_he_cap.tx_he_mcs_map_160), + *((uint16_t *)he_cap->tx_he_mcs_map_160)); + rx_mcs_map = mlme_get_min_rate_cap( + *((uint16_t *)mlme_obj->cfg.he_caps.dot11_he_cap.rx_he_mcs_map_160), + *((uint16_t *)he_cap->rx_he_mcs_map_160)); if (!mlme_obj->cfg.vht_caps.vht_cap_info.enable2x2) { nss = 2; @@ -3228,8 +3262,11 @@ mlme_update_vht_cap(struct wlan_objmgr_psoc *psoc, struct wma_tgt_vht_cap *cfg) if (vht_cap_info->short_gi_160mhz && !cfg->vht_short_gi_160) vht_cap_info->short_gi_160mhz = cfg->vht_short_gi_160; - vht_cap_info->vht_mcs_10_11_supp = cfg->vht_mcs_10_11_supp; - mlme_legacy_debug(" vht_mcs_10_11_supp %d", cfg->vht_mcs_10_11_supp); + if (cfg_get(psoc, CFG_ENABLE_VHT_MCS_10_11)) + vht_cap_info->vht_mcs_10_11_supp = cfg->vht_mcs_10_11_supp; + + mlme_legacy_debug("vht_mcs_10_11_supp %d", + vht_cap_info->vht_mcs_10_11_supp); return QDF_STATUS_SUCCESS; } diff --git a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_ucfg_api.c b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_ucfg_api.c index 114b98ed51df..bdbc2aaa4243 100644 --- a/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_ucfg_api.c +++ b/drivers/staging/qcacld-3.0/components/mlme/dispatcher/src/wlan_mlme_ucfg_api.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -1783,6 +1784,21 @@ ucfg_mlme_set_obss_detection_offload_enabled(struct wlan_objmgr_psoc *psoc, return QDF_STATUS_SUCCESS; } +QDF_STATUS +ucfg_mlme_set_bss_color_collision_det_sta(struct wlan_objmgr_psoc *psoc, + uint8_t value) +{ + struct wlan_mlme_psoc_ext_obj *mlme_obj; + + mlme_obj = mlme_get_psoc_ext_obj(psoc); + if (!mlme_obj) + return QDF_STATUS_E_INVAL; + + mlme_obj->cfg.obss_ht40.bss_color_collision_det_sta = value; + + return QDF_STATUS_SUCCESS; +} + QDF_STATUS ucfg_mlme_set_obss_color_collision_offload_enabled( struct wlan_objmgr_psoc *psoc, uint8_t value) diff --git a/drivers/staging/qcacld-3.0/components/nan/core/src/nan_main.c b/drivers/staging/qcacld-3.0/components/nan/core/src/nan_main.c index 0de7322a7897..4e806a8a10a6 100644 --- a/drivers/staging/qcacld-3.0/components/nan/core/src/nan_main.c +++ b/drivers/staging/qcacld-3.0/components/nan/core/src/nan_main.c @@ -530,6 +530,7 @@ static QDF_STATUS nan_handle_confirm( struct wlan_objmgr_psoc *psoc; struct nan_psoc_priv_obj *psoc_nan_obj; struct nan_vdev_priv_obj *vdev_nan_obj; + struct wlan_objmgr_peer *peer; QDF_STATUS status; vdev_id = wlan_vdev_get_id(confirm->vdev); @@ -539,6 +540,15 @@ static QDF_STATUS nan_handle_confirm( return QDF_STATUS_E_NULL_VALUE; } + peer = wlan_objmgr_get_peer_by_mac(psoc, + confirm->peer_ndi_mac_addr.bytes, + WLAN_NAN_ID); + if (!peer) { + nan_debug("Drop NDP confirm as peer isn't available"); + return QDF_STATUS_E_NULL_VALUE; + } + wlan_objmgr_peer_release_ref(peer, WLAN_NAN_ID); + psoc_nan_obj = nan_get_psoc_priv_obj(psoc); if (!psoc_nan_obj) { nan_err("psoc_nan_obj is null"); diff --git a/drivers/staging/qcacld-3.0/configs/default_defconfig b/drivers/staging/qcacld-3.0/configs/default_defconfig index 67c6c7003767..af5258d80f8e 100644 --- a/drivers/staging/qcacld-3.0/configs/default_defconfig +++ b/drivers/staging/qcacld-3.0/configs/default_defconfig @@ -46,6 +46,7 @@ ifeq ($(CONFIG_CNSS_QCA6490), y) CONFIG_HASTINGS_BT_WAR := y CONFIG_WDI3_IPA_OVER_GSI :=y CONFIG_GENERIC_SHADOW_REGISTER_ACCESS_ENABLE :=y + CONFIG_DUMP_REO_QUEUE_INFO_IN_DDR :=y endif ifeq ($(CONFIG_CNSS_QCA6750), y) diff --git a/drivers/staging/qcacld-3.0/core/cds/src/cds_api.c b/drivers/staging/qcacld-3.0/core/cds/src/cds_api.c index 37ce2f4c1112..1b8f870cd70f 100644 --- a/drivers/staging/qcacld-3.0/core/cds/src/cds_api.c +++ b/drivers/staging/qcacld-3.0/core/cds/src/cds_api.c @@ -460,6 +460,7 @@ static void cds_cdp_cfg_attach(struct wlan_objmgr_psoc *psoc) cfg_get(psoc, CFG_DP_CE_CLASSIFY_ENABLE); cdp_cfg.tso_enable = cfg_get(psoc, CFG_DP_TSO); cdp_cfg.lro_enable = cfg_get(psoc, CFG_DP_LRO); + cdp_cfg.sg_enable = cfg_get(psoc, CFG_DP_SG); cdp_cfg.enable_data_stall_detection = cfg_get(psoc, CFG_DP_ENABLE_DATA_STALL_DETECTION); cdp_cfg.gro_enable = cfg_get(psoc, CFG_DP_GRO); diff --git a/drivers/staging/qcacld-3.0/core/dp/ol/inc/ol_cfg.h b/drivers/staging/qcacld-3.0/core/dp/ol/inc/ol_cfg.h index 88b9374aac1a..d393655197d3 100644 --- a/drivers/staging/qcacld-3.0/core/dp/ol/inc/ol_cfg.h +++ b/drivers/staging/qcacld-3.0/core/dp/ol/inc/ol_cfg.h @@ -111,6 +111,7 @@ struct txrx_pdev_cfg_t { bool gro_enable; bool tso_enable; bool lro_enable; + bool sg_enable; bool enable_data_stall_detection; bool enable_flow_steering; bool disable_intra_bss_fwd; diff --git a/drivers/staging/qcacld-3.0/core/dp/txrx/ol_cfg.c b/drivers/staging/qcacld-3.0/core/dp/txrx/ol_cfg.c index 9df6b0bae340..31ac0dd93ef5 100644 --- a/drivers/staging/qcacld-3.0/core/dp/txrx/ol_cfg.c +++ b/drivers/staging/qcacld-3.0/core/dp/txrx/ol_cfg.c @@ -208,6 +208,7 @@ struct cdp_cfg *ol_pdev_cfg_attach(qdf_device_t osdev, void *pcfg_param) cfg_ctx->gro_enable = cfg_param->gro_enable; cfg_ctx->tso_enable = cfg_param->tso_enable; cfg_ctx->lro_enable = cfg_param->lro_enable; + cfg_ctx->sg_enable = cfg_param->sg_enable; cfg_ctx->enable_data_stall_detection = cfg_param->enable_data_stall_detection; cfg_ctx->enable_flow_steering = cfg_param->enable_flow_steering; diff --git a/drivers/staging/qcacld-3.0/core/dp/txrx/ol_txrx.c b/drivers/staging/qcacld-3.0/core/dp/txrx/ol_txrx.c index 3bad79a67107..e1e8ba6cddf2 100644 --- a/drivers/staging/qcacld-3.0/core/dp/txrx/ol_txrx.c +++ b/drivers/staging/qcacld-3.0/core/dp/txrx/ol_txrx.c @@ -5904,6 +5904,9 @@ static uint32_t ol_txrx_get_cfg(struct cdp_soc_t *soc_hdl, enum cdp_dp_cfg cfg) case cfg_dp_lro_enable: value = cfg_ctx->lro_enable; break; + case cfg_dp_sg_enable: + value = cfg_ctx->sg_enable; + break; case cfg_dp_gro_enable: value = cfg_ctx->gro_enable; break; diff --git a/drivers/staging/qcacld-3.0/core/hdd/src/wlan_hdd_main.c b/drivers/staging/qcacld-3.0/core/hdd/src/wlan_hdd_main.c index 2376b1ef6524..146591b3ba30 100644 --- a/drivers/staging/qcacld-3.0/core/hdd/src/wlan_hdd_main.c +++ b/drivers/staging/qcacld-3.0/core/hdd/src/wlan_hdd_main.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2012-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -215,6 +216,12 @@ #define MAX_NET_DEV_REF_LEAK_ITERATIONS 10 #define NET_DEV_REF_LEAK_ITERATION_SLEEP_TIME_MS 10 +#ifdef FEATURE_TSO +#define TSO_FEATURE_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_SG) +#else +#define TSO_FEATURE_FLAGS 0 +#endif + int wlan_start_ret_val; static DECLARE_COMPLETION(wlan_start_comp); static qdf_atomic_t wlan_hdd_state_fops_ref; @@ -2478,6 +2485,14 @@ int hdd_update_tgt_cfg(hdd_handle_t hdd_handle, struct wma_tgt_cfg *cfg) if (QDF_IS_STATUS_ERROR(status)) hdd_err("Failed to set WNI_CFG_OBSS_COLOR_COLLISION_OFFLOAD"); + if (!cfg->obss_color_collision_offloaded) { + status = ucfg_mlme_set_bss_color_collision_det_sta( + hdd_ctx->psoc, + cfg->obss_color_collision_offloaded); + if (QDF_IS_STATUS_ERROR(status)) + hdd_err("Failed to set CFG_BSS_CLR_COLLISION_DET_STA"); + } + ucfg_mlme_get_bcast_twt(hdd_ctx->psoc, &bval); if (bval) ucfg_mlme_set_bcast_twt(hdd_ctx->psoc, cfg->bcast_twt_support); @@ -7179,8 +7194,10 @@ void hdd_set_netdev_flags(struct hdd_adapter *adapter) (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); if (cdp_cfg_get(soc, cfg_dp_tso_enable) && enable_csum) - adapter->dev->features |= - (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_SG); + adapter->dev->features |= TSO_FEATURE_FLAGS; + + if (cdp_cfg_get(soc, cfg_dp_sg_enable)) + adapter->dev->features |= NETIF_F_SG; adapter->dev->features |= NETIF_F_RXCSUM; temp = (uint64_t)adapter->dev->features; diff --git a/drivers/staging/qcacld-3.0/core/mac/inc/qwlan_version.h b/drivers/staging/qcacld-3.0/core/mac/inc/qwlan_version.h index bcbf77eaea18..26f67e3408a9 100644 --- a/drivers/staging/qcacld-3.0/core/mac/inc/qwlan_version.h +++ b/drivers/staging/qcacld-3.0/core/mac/inc/qwlan_version.h @@ -32,9 +32,9 @@ #define QWLAN_VERSION_MAJOR 5 #define QWLAN_VERSION_MINOR 2 #define QWLAN_VERSION_PATCH 022 -#define QWLAN_VERSION_EXTRA "T" -#define QWLAN_VERSION_BUILD 10 +#define QWLAN_VERSION_EXTRA "B" +#define QWLAN_VERSION_BUILD 11 -#define QWLAN_VERSIONSTR "5.2.022.10T" +#define QWLAN_VERSIONSTR "5.2.022.11B" #endif /* QWLAN_VERSION_H */ diff --git a/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_assoc_utils.c b/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_assoc_utils.c index ee94873bc587..af5046232677 100644 --- a/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_assoc_utils.c +++ b/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_assoc_utils.c @@ -2492,9 +2492,11 @@ lim_add_sta(struct mac_context *mac_ctx, assoc_req = (tpSirAssocReq) session_entry->parsedAssocReq[aid]; - add_sta_params->wpa_rsn = assoc_req->rsnPresent; - add_sta_params->wpa_rsn |= - (assoc_req->wpaPresent << 1); + if (assoc_req) { + add_sta_params->wpa_rsn = assoc_req->rsnPresent; + add_sta_params->wpa_rsn |= + (assoc_req->wpaPresent << 1); + } } } diff --git a/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_session.c b/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_session.c index df3eeb63e7db..eacb32ec1f93 100644 --- a/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_session.c +++ b/drivers/staging/qcacld-3.0/core/mac/src/pe/lim/lim_session.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2011-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -710,6 +711,9 @@ struct pe_session *pe_create_session(struct mac_context *mac, if (status != QDF_STATUS_SUCCESS) pe_err("cannot create ap_ecsa_timer"); } + if (session_ptr->opmode == QDF_STA_MODE) + session_ptr->is_session_obss_color_collision_det_enabled = + mac->mlme_cfg->obss_ht40.bss_color_collision_det_sta; pe_init_fils_info(session_ptr); pe_init_pmf_comeback_timer(mac, session_ptr); session_ptr->ht_client_cnt = 0; diff --git a/drivers/staging/qcacld-3.0/core/mac/src/pe/nan/nan_datapath.c b/drivers/staging/qcacld-3.0/core/mac/src/pe/nan/nan_datapath.c index f20e53801460..0c4868f3fee8 100644 --- a/drivers/staging/qcacld-3.0/core/mac/src/pe/nan/nan_datapath.c +++ b/drivers/staging/qcacld-3.0/core/mac/src/pe/nan/nan_datapath.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -50,6 +51,11 @@ static QDF_STATUS lim_add_ndi_peer(struct mac_context *mac_ctx, QDF_STATUS status; uint8_t zero_mac_addr[QDF_MAC_ADDR_SIZE] = { 0, 0, 0, 0, 0, 0 }; + if (!wlan_is_vdev_id_up(mac_ctx->pdev, vdev_id)) { + pe_err_rl("NDI vdev is not up"); + return QDF_STATUS_E_FAILURE; + } + if (!qdf_mem_cmp(&zero_mac_addr, &peer_mac_addr.bytes[0], QDF_MAC_ADDR_SIZE)) { pe_err("Failing to add peer with all zero mac addr"); diff --git a/drivers/staging/qcacld-3.0/core/wma/src/wma_dev_if.c b/drivers/staging/qcacld-3.0/core/wma/src/wma_dev_if.c index 4327dcaca226..0e9a8cfd8c93 100644 --- a/drivers/staging/qcacld-3.0/core/wma/src/wma_dev_if.c +++ b/drivers/staging/qcacld-3.0/core/wma/src/wma_dev_if.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2013-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -90,6 +91,7 @@ #include "../../core/src/vdev_mgr_ops.h" #include "wlan_utility.h" #include +#include "wmi_unified_vdev_api.h" QDF_STATUS wma_find_vdev_id_by_addr(tp_wma_handle wma, uint8_t *addr, uint8_t *vdev_id) @@ -1159,6 +1161,36 @@ QDF_STATUS wma_handle_channel_switch_resp(tp_wma_handle wma, return QDF_STATUS_SUCCESS; } +/* + * wma_get_ratemask_type() - convert user input ratemask type to FW type + * @type: User input ratemask type maintained in HDD + * @fwtype: Value return arg for fw ratemask type value + * + * Return: FW configurable ratemask type + */ +static QDF_STATUS wma_get_ratemask_type(enum wlan_mlme_ratemask_type type, + uint8_t *fwtype) +{ + switch (type) { + case WLAN_MLME_RATEMASK_TYPE_CCK: + *fwtype = WMI_RATEMASK_TYPE_CCK; + break; + case WLAN_MLME_RATEMASK_TYPE_HT: + *fwtype = WMI_RATEMASK_TYPE_HT; + break; + case WLAN_MLME_RATEMASK_TYPE_VHT: + *fwtype = WMI_RATEMASK_TYPE_VHT; + break; + case WLAN_MLME_RATEMASK_TYPE_HE: + *fwtype = WMI_RATEMASK_TYPE_HE; + break; + default: + return QDF_STATUS_E_INVAL; + } + + return QDF_STATUS_SUCCESS; +} + QDF_STATUS wma_vdev_start_resp_handler(struct vdev_mlme_obj *vdev_mlme, struct vdev_start_response *rsp) { @@ -1172,6 +1204,9 @@ QDF_STATUS wma_vdev_start_resp_handler(struct vdev_mlme_obj *vdev_mlme, QDF_STATUS status; enum vdev_assoc_type assoc_type = VDEV_ASSOC; struct vdev_mlme_obj *mlme_obj; + struct wlan_mlme_psoc_ext_obj *mlme_psoc_obj; + const struct wlan_mlme_ratemask *ratemask_cfg; + struct config_ratemask_params rparams = {0}; wma = cds_get_context(QDF_MODULE_ID_WMA); if (!wma) { @@ -1184,6 +1219,9 @@ QDF_STATUS wma_vdev_start_resp_handler(struct vdev_mlme_obj *vdev_mlme, return QDF_STATUS_E_FAILURE; } + mlme_psoc_obj = mlme_get_psoc_ext_obj(psoc); + ratemask_cfg = &mlme_psoc_obj->cfg.ratemask_cfg; + #ifdef FEATURE_AP_MCC_CH_AVOIDANCE if (!mac_ctx) { WMA_LOGE("%s: Failed to get mac_ctx", __func__); @@ -1292,6 +1330,38 @@ QDF_STATUS wma_vdev_start_resp_handler(struct vdev_mlme_obj *vdev_mlme, if (iface->type == WMI_VDEV_TYPE_AP && wma_is_vdev_up(rsp->vdev_id)) wma_set_sap_keepalive(wma, rsp->vdev_id); + /* Send ratemask to firmware */ + if ((ratemask_cfg->type > WLAN_MLME_RATEMASK_TYPE_NO_MASK) && + (ratemask_cfg->type < WLAN_MLME_RATEMASK_TYPE_MAX)) { + struct wmi_unified *wmi_handle = wma->wmi_handle; + + if (!wmi_handle) { + wma_err(FL("wmi_handle is null")); + return QDF_STATUS_E_INVAL; + } + + rparams.vdev_id = rsp->vdev_id; + status = wma_get_ratemask_type(ratemask_cfg->type, + &rparams.type); + + if (QDF_IS_STATUS_ERROR(status)) { + wma_err(FL("unable to map ratemask")); + /* don't fail, default rates will still work */ + return QDF_STATUS_SUCCESS; + } + + rparams.lower32 = ratemask_cfg->lower32; + rparams.higher32 = ratemask_cfg->higher32; + rparams.lower32_2 = ratemask_cfg->lower32_2; + rparams.higher32_2 = ratemask_cfg->higher32_2; + + status = wmi_unified_vdev_config_ratemask_cmd_send(wmi_handle, + &rparams); + /* Only log failure. Do not abort */ + if (QDF_IS_STATUS_ERROR(status)) + wma_err(FL("failed to send ratemask")); + } + return QDF_STATUS_SUCCESS; } diff --git a/drivers/staging/qcacld-3.0/core/wma/src/wma_utils.c b/drivers/staging/qcacld-3.0/core/wma/src/wma_utils.c index fa345f5ed838..61fd15bcb4e9 100644 --- a/drivers/staging/qcacld-3.0/core/wma/src/wma_utils.c +++ b/drivers/staging/qcacld-3.0/core/wma/src/wma_utils.c @@ -805,7 +805,6 @@ int wma_profile_data_report_event_handler(void *handle, uint8_t *event_buf, wmi_wlan_profile_t *profile_data; uint32_t i = 0; uint32_t entries; - uint8_t *buf_ptr; char temp_str[150]; param_buf = (WMI_WLAN_PROFILE_DATA_EVENTID_param_tlvs *) event_buf; @@ -813,12 +812,9 @@ int wma_profile_data_report_event_handler(void *handle, uint8_t *event_buf, WMA_LOGE("%s: Invalid profile data event buf", __func__); return -EINVAL; } + profile_ctx = param_buf->profile_ctx; - buf_ptr = (uint8_t *)profile_ctx; - buf_ptr = buf_ptr + sizeof(wmi_wlan_profile_ctx_t) + WMI_TLV_HDR_SIZE; - profile_data = (wmi_wlan_profile_t *) buf_ptr; entries = profile_ctx->bin_count; - if (entries > param_buf->num_profile_data) { WMA_LOGE("FW bin count %d more than data %d in TLV hdr", entries, @@ -847,6 +843,7 @@ int wma_profile_data_report_event_handler(void *handle, uint8_t *event_buf, QDF_TRACE(QDF_MODULE_ID_WMA, QDF_TRACE_LEVEL_ERROR, "Profile ID: Count: TOT: Min: Max: hist_intvl: hist[0]: hist[1]:hist[2]"); + profile_data = param_buf->profile_data; for (i = 0; i < entries; i++) { if (i == WMI_WLAN_PROFILE_MAX_BIN_CNT) break; @@ -1123,9 +1120,9 @@ wma_fill_tx_stats(struct sir_wifi_ll_ext_stats *ll_stats, struct sir_wifi_tx *tx_stats; struct sir_wifi_ll_ext_peer_stats *peer_stats; uint32_t *tx_mpdu_aggr, *tx_succ_mcs, *tx_fail_mcs, *tx_delay; - uint32_t len, dst_len, param_len, tx_mpdu_aggr_array_len, - tx_succ_mcs_array_len, tx_fail_mcs_array_len, - tx_delay_array_len; + uint32_t len, dst_len, param_len, num_entries, + tx_mpdu_aggr_array_len, tx_succ_mcs_array_len, + tx_fail_mcs_array_len, tx_delay_array_len; result = *buf; dst_len = *buf_length; @@ -1204,6 +1201,12 @@ wma_fill_tx_stats(struct sir_wifi_ll_ext_stats *ll_stats, return QDF_STATUS_E_FAILURE; } + num_entries = fix_param->num_peer_ac_tx_stats * WLAN_MAX_AC; + if (num_entries > param_buf->num_tx_stats) { + wma_err("tx stats invalid arg, %d", num_entries); + return QDF_STATUS_E_FAILURE; + } + for (i = 0; i < fix_param->num_peer_ac_tx_stats; i++) { uint32_t peer_id = wmi_peer_tx[i].peer_id; struct sir_wifi_tx *ac; diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c index 919231fec09c..5f1dd4e2d12e 100644 --- a/drivers/staging/rtl8192e/rtllib_softmac.c +++ b/drivers/staging/rtl8192e/rtllib_softmac.c @@ -654,9 +654,9 @@ static void rtllib_beacons_stop(struct rtllib_device *ieee) spin_lock_irqsave(&ieee->beacon_lock, flags); ieee->beacon_txing = 0; - del_timer_sync(&ieee->beacon_timer); spin_unlock_irqrestore(&ieee->beacon_lock, flags); + del_timer_sync(&ieee->beacon_timer); } diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c index 212cc9ccbb96..426966a29d31 100644 --- a/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c @@ -529,9 +529,9 @@ static void ieee80211_beacons_stop(struct ieee80211_device *ieee) spin_lock_irqsave(&ieee->beacon_lock, flags); ieee->beacon_txing = 0; - del_timer_sync(&ieee->beacon_timer); spin_unlock_irqrestore(&ieee->beacon_lock, flags); + del_timer_sync(&ieee->beacon_timer); } void ieee80211_stop_send_beacons(struct ieee80211_device *ieee) diff --git a/drivers/staging/rtl8712/usb_intf.c b/drivers/staging/rtl8712/usb_intf.c index 5e2cdc25401b..2b1ff63913af 100644 --- a/drivers/staging/rtl8712/usb_intf.c +++ b/drivers/staging/rtl8712/usb_intf.c @@ -569,13 +569,13 @@ static int r871xu_drv_init(struct usb_interface *pusb_intf, } else { AutoloadFail = false; } - if (((mac[0] == 0xff) && (mac[1] == 0xff) && + if ((!AutoloadFail) || + ((mac[0] == 0xff) && (mac[1] == 0xff) && (mac[2] == 0xff) && (mac[3] == 0xff) && (mac[4] == 0xff) && (mac[5] == 0xff)) || ((mac[0] == 0x00) && (mac[1] == 0x00) && (mac[2] == 0x00) && (mac[3] == 0x00) && - (mac[4] == 0x00) && (mac[5] == 0x00)) || - (!AutoloadFail)) { + (mac[4] == 0x00) && (mac[5] == 0x00))) { mac[0] = 0x00; mac[1] = 0xe0; mac[2] = 0x4c; diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c index 466d25ccc4bb..40d7130a4909 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c @@ -1359,9 +1359,11 @@ static int rtw_wx_set_scan(struct net_device *dev, struct iw_request_info *a, sec_len = *(pos++); len-= 1; - if (sec_len>0 && sec_len<=len) { + if (sec_len > 0 && + sec_len <= len && + sec_len <= 32) { ssid[ssid_index].SsidLength = sec_len; - memcpy(ssid[ssid_index].Ssid, pos, ssid[ssid_index].SsidLength); + memcpy(ssid[ssid_index].Ssid, pos, sec_len); /* DBG_871X("%s COMBO_SCAN with specific ssid:%s, %d\n", __func__ */ /* , ssid[ssid_index].Ssid, ssid[ssid_index].SsidLength); */ ssid_index++; diff --git a/drivers/tty/goldfish.c b/drivers/tty/goldfish.c index c8c5cdfc5e19..abc84d84f638 100644 --- a/drivers/tty/goldfish.c +++ b/drivers/tty/goldfish.c @@ -407,6 +407,7 @@ static int goldfish_tty_probe(struct platform_device *pdev) err_tty_register_device_failed: free_irq(irq, qtty); err_dec_line_count: + tty_port_destroy(&qtty->port); goldfish_tty_current_line_count--; if (goldfish_tty_current_line_count == 0) goldfish_tty_delete_driver(); @@ -428,6 +429,7 @@ static int goldfish_tty_remove(struct platform_device *pdev) iounmap(qtty->base); qtty->base = NULL; free_irq(qtty->irq, pdev); + tty_port_destroy(&qtty->port); goldfish_tty_current_line_count--; if (goldfish_tty_current_line_count == 0) goldfish_tty_delete_driver(); diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c index 79a4958b3f5c..440023069f4f 100644 --- a/drivers/tty/serial/8250/8250_fintek.c +++ b/drivers/tty/serial/8250/8250_fintek.c @@ -197,12 +197,12 @@ static int fintek_8250_rs485_config(struct uart_port *port, if (!pdata) return -EINVAL; - /* Hardware do not support same RTS level on send and receive */ - if (!(rs485->flags & SER_RS485_RTS_ON_SEND) == - !(rs485->flags & SER_RS485_RTS_AFTER_SEND)) - return -EINVAL; if (rs485->flags & SER_RS485_ENABLED) { + /* Hardware do not support same RTS level on send and receive */ + if (!(rs485->flags & SER_RS485_RTS_ON_SEND) == + !(rs485->flags & SER_RS485_RTS_AFTER_SEND)) + return -EINVAL; memset(rs485->padding, 0, sizeof(rs485->padding)); config |= RS485_URA; } else { diff --git a/drivers/tty/serial/digicolor-usart.c b/drivers/tty/serial/digicolor-usart.c index 4446c13629b1..e06967ca62fa 100644 --- a/drivers/tty/serial/digicolor-usart.c +++ b/drivers/tty/serial/digicolor-usart.c @@ -309,6 +309,8 @@ static void digicolor_uart_set_termios(struct uart_port *port, case CS8: default: config |= UA_CONFIG_CHAR_LEN; + termios->c_cflag &= ~CSIZE; + termios->c_cflag |= CS8; break; } diff --git a/drivers/tty/serial/icom.c b/drivers/tty/serial/icom.c index ad374f7c476d..cb950c78d66d 100644 --- a/drivers/tty/serial/icom.c +++ b/drivers/tty/serial/icom.c @@ -1501,7 +1501,7 @@ static int icom_probe(struct pci_dev *dev, retval = pci_read_config_dword(dev, PCI_COMMAND, &command_reg); if (retval) { dev_err(&dev->dev, "PCI Config read FAILED\n"); - return retval; + goto probe_exit0; } pci_write_config_dword(dev, PCI_COMMAND, diff --git a/drivers/tty/serial/meson_uart.c b/drivers/tty/serial/meson_uart.c index 8a842591b37c..1838d0be3704 100644 --- a/drivers/tty/serial/meson_uart.c +++ b/drivers/tty/serial/meson_uart.c @@ -255,6 +255,14 @@ static const char *meson_uart_type(struct uart_port *port) return (port->type == PORT_MESON) ? "meson_uart" : NULL; } +/* + * This function is called only from probe() using a temporary io mapping + * in order to perform a reset before setting up the device. Since the + * temporarily mapped region was successfully requested, there can be no + * console on this port at this time. Hence it is not necessary for this + * function to acquire the port->lock. (Since there is no console on this + * port at this time, the port->lock is not initialized yet.) + */ static void meson_uart_reset(struct uart_port *port) { u32 val; @@ -269,9 +277,12 @@ static void meson_uart_reset(struct uart_port *port) static int meson_uart_startup(struct uart_port *port) { + unsigned long flags; u32 val; int ret = 0; + spin_lock_irqsave(&port->lock, flags); + val = readl(port->membase + AML_UART_CONTROL); val |= AML_UART_CLEAR_ERR; writel(val, port->membase + AML_UART_CONTROL); @@ -287,6 +298,8 @@ static int meson_uart_startup(struct uart_port *port) val = (AML_UART_RECV_IRQ(1) | AML_UART_XMIT_IRQ(port->fifosize / 2)); writel(val, port->membase + AML_UART_MISC); + spin_unlock_irqrestore(&port->lock, flags); + ret = request_irq(port->irq, meson_uart_interrupt, 0, port->name, port); diff --git a/drivers/tty/serial/msm_serial.c b/drivers/tty/serial/msm_serial.c index 7bb495a7b2fd..931952adbeb8 100644 --- a/drivers/tty/serial/msm_serial.c +++ b/drivers/tty/serial/msm_serial.c @@ -1593,6 +1593,7 @@ static inline struct uart_port *msm_get_port_from_line(unsigned int line) static void __msm_console_write(struct uart_port *port, const char *s, unsigned int count, bool is_uartdm) { + unsigned long flags; int i; int num_newlines = 0; bool replaced = false; @@ -1610,6 +1611,8 @@ static void __msm_console_write(struct uart_port *port, const char *s, num_newlines++; count += num_newlines; + local_irq_save(flags); + if (port->sysrq) locked = 0; else if (oops_in_progress) @@ -1655,6 +1658,8 @@ static void __msm_console_write(struct uart_port *port, const char *s, if (locked) spin_unlock(&port->lock); + + local_irq_restore(flags); } static void msm_console_write(struct console *co, const char *s, diff --git a/drivers/tty/serial/sa1100.c b/drivers/tty/serial/sa1100.c index a399772be3fc..8a65fb1ce568 100644 --- a/drivers/tty/serial/sa1100.c +++ b/drivers/tty/serial/sa1100.c @@ -439,6 +439,8 @@ sa1100_set_termios(struct uart_port *port, struct ktermios *termios, baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk/16); quot = uart_get_divisor(port, baud); + del_timer_sync(&sport->timer); + spin_lock_irqsave(&sport->port.lock, flags); sport->port.read_status_mask &= UTSR0_TO_SM(UTSR0_TFS); @@ -469,8 +471,6 @@ sa1100_set_termios(struct uart_port *port, struct ktermios *termios, UTSR1_TO_SM(UTSR1_ROR); } - del_timer_sync(&sport->timer); - /* * Update the per-port timeout. */ diff --git a/drivers/tty/serial/serial_txx9.c b/drivers/tty/serial/serial_txx9.c index 2f7ef64536a0..8724db39e947 100644 --- a/drivers/tty/serial/serial_txx9.c +++ b/drivers/tty/serial/serial_txx9.c @@ -649,6 +649,8 @@ serial_txx9_set_termios(struct uart_port *port, struct ktermios *termios, case CS6: /* not supported */ case CS8: cval |= TXX9_SILCR_UMODE_8BIT; + termios->c_cflag &= ~CSIZE; + termios->c_cflag |= CS8; break; } diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c index 6f44c5f0ef3a..ba7f0b44b710 100644 --- a/drivers/tty/serial/sh-sci.c +++ b/drivers/tty/serial/sh-sci.c @@ -2382,8 +2382,12 @@ static void sci_set_termios(struct uart_port *port, struct ktermios *termios, int best_clk = -1; unsigned long flags; - if ((termios->c_cflag & CSIZE) == CS7) + if ((termios->c_cflag & CSIZE) == CS7) { smr_val |= SCSMR_CHR; + } else { + termios->c_cflag &= ~CSIZE; + termios->c_cflag |= CS8; + } if (termios->c_cflag & PARENB) smr_val |= SCSMR_PE; if (termios->c_cflag & PARODD) diff --git a/drivers/tty/serial/st-asc.c b/drivers/tty/serial/st-asc.c index 7971997cdead..ce35e3a131b1 100644 --- a/drivers/tty/serial/st-asc.c +++ b/drivers/tty/serial/st-asc.c @@ -540,10 +540,14 @@ static void asc_set_termios(struct uart_port *port, struct ktermios *termios, /* set character length */ if ((cflag & CSIZE) == CS7) { ctrl_val |= ASC_CTL_MODE_7BIT_PAR; + cflag |= PARENB; } else { ctrl_val |= (cflag & PARENB) ? ASC_CTL_MODE_8BIT_PAR : ASC_CTL_MODE_8BIT; + cflag &= ~CSIZE; + cflag |= CS8; } + termios->c_cflag = cflag; /* set stop bit */ ctrl_val |= (cflag & CSTOPB) ? ASC_CTL_STOP_2BIT : ASC_CTL_STOP_1BIT; diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c index ccaaf804df06..bb2f6f02ce23 100644 --- a/drivers/tty/serial/stm32-usart.c +++ b/drivers/tty/serial/stm32-usart.c @@ -688,13 +688,22 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios, * CS8 or (CS7 + parity), 8 bits word aka [M1:M0] = 0b00 * M0 and M1 already cleared by cr1 initialization. */ - if (bits == 9) + if (bits == 9) { cr1 |= USART_CR1_M0; - else if ((bits == 7) && cfg->has_7bits_data) + } else if ((bits == 7) && cfg->has_7bits_data) { cr1 |= USART_CR1_M1; - else if (bits != 8) + } else if (bits != 8) { dev_dbg(port->dev, "Unsupported data bits config: %u bits\n" , bits); + cflag &= ~CSIZE; + cflag |= CS8; + termios->c_cflag = cflag; + bits = 8; + if (cflag & PARENB) { + bits++; + cr1 |= USART_CR1_M0; + } + } if (cflag & PARODD) cr1 |= USART_CR1_PS; diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c index afe34beec720..11c62fcd67f2 100644 --- a/drivers/tty/synclink_gt.c +++ b/drivers/tty/synclink_gt.c @@ -1753,6 +1753,8 @@ static int hdlcdev_init(struct slgt_info *info) */ static void hdlcdev_exit(struct slgt_info *info) { + if (!info->netdev) + return; unregister_hdlc_device(info->netdev); free_netdev(info->netdev); info->netdev = NULL; diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c index 6b0cb633679d..dfe0c8c22cd3 100644 --- a/drivers/tty/tty_buffer.c +++ b/drivers/tty/tty_buffer.c @@ -167,7 +167,8 @@ static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size) have queued and recycle that ? */ if (atomic_read(&port->buf.mem_used) > port->buf.mem_limit) return NULL; - p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC); + p = kmalloc(sizeof(struct tty_buffer) + 2 * size, + GFP_ATOMIC | __GFP_NOWARN); if (p == NULL) return NULL; diff --git a/drivers/usb/core/hcd-pci.c b/drivers/usb/core/hcd-pci.c index 7537681355f6..fed331d786e2 100644 --- a/drivers/usb/core/hcd-pci.c +++ b/drivers/usb/core/hcd-pci.c @@ -605,10 +605,10 @@ const struct dev_pm_ops usb_hcd_pci_pm_ops = { .suspend_noirq = hcd_pci_suspend_noirq, .resume_noirq = hcd_pci_resume_noirq, .resume = hcd_pci_resume, - .freeze = check_root_hub_suspended, + .freeze = hcd_pci_suspend, .freeze_noirq = check_root_hub_suspended, .thaw_noirq = NULL, - .thaw = NULL, + .thaw = hcd_pci_resume, .poweroff = hcd_pci_suspend, .poweroff_noirq = hcd_pci_suspend_noirq, .restore_noirq = hcd_pci_resume_noirq, diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index d5f233fa6f3b..f8f2de7899a9 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -511,6 +511,9 @@ static const struct usb_device_id usb_quirk_list[] = { /* DJI CineSSD */ { USB_DEVICE(0x2ca3, 0x0031), .driver_info = USB_QUIRK_NO_LPM }, + /* DELL USB GEN2 */ + { USB_DEVICE(0x413c, 0xb062), .driver_info = USB_QUIRK_NO_LPM | USB_QUIRK_RESET_RESUME }, + /* VCOM device */ { USB_DEVICE(0x4296, 0x7570), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS }, diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 1e46005929e4..85d25f7e9c27 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -4326,7 +4326,6 @@ static int dwc2_hsotg_udc_start(struct usb_gadget *gadget, WARN_ON(hsotg->driver); - driver->driver.bus = NULL; hsotg->driver = driver; hsotg->gadget.dev.of_node = hsotg->dev->of_node; hsotg->gadget.speed = USB_SPEED_UNKNOWN; diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index ad2cb08b440f..527938eee846 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -205,7 +205,7 @@ static void dwc3_pci_resume_work(struct work_struct *work) int ret; ret = pm_runtime_get_sync(&dwc3->dev); - if (ret) { + if (ret < 0) { pm_runtime_put_sync_autosuspend(&dwc3->dev); return; } diff --git a/drivers/usb/host/isp116x-hcd.c b/drivers/usb/host/isp116x-hcd.c index 74da136d322a..361327012c78 100644 --- a/drivers/usb/host/isp116x-hcd.c +++ b/drivers/usb/host/isp116x-hcd.c @@ -1541,10 +1541,12 @@ static int isp116x_remove(struct platform_device *pdev) iounmap(isp116x->data_reg); res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - release_mem_region(res->start, 2); + if (res) + release_mem_region(res->start, 2); iounmap(isp116x->addr_reg); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(res->start, 2); + if (res) + release_mem_region(res->start, 2); usb_put_hcd(hcd); return 0; diff --git a/drivers/usb/host/oxu210hp-hcd.c b/drivers/usb/host/oxu210hp-hcd.c index 10d97261b433..48533eb707e6 100644 --- a/drivers/usb/host/oxu210hp-hcd.c +++ b/drivers/usb/host/oxu210hp-hcd.c @@ -3476,8 +3476,10 @@ static int oxu_bus_suspend(struct usb_hcd *hcd) } } + spin_unlock_irq(&oxu->lock); /* turn off now-idle HC */ del_timer_sync(&oxu->watchdog); + spin_lock_irq(&oxu->lock); ehci_halt(oxu); hcd->state = HC_STATE_SUSPENDED; diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c index 24e622c05638..5f1c41e95f56 100644 --- a/drivers/usb/musb/omap2430.c +++ b/drivers/usb/musb/omap2430.c @@ -433,6 +433,7 @@ static int omap2430_probe(struct platform_device *pdev) control_node = of_parse_phandle(np, "ctrl-module", 0); if (control_node) { control_pdev = of_find_device_by_node(control_node); + of_node_put(control_node); if (!control_pdev) { dev_err(&pdev->dev, "Failed to get control device\n"); ret = -EINVAL; diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 0699b770e058..427a38ab198b 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -1137,6 +1137,8 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM12, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, 0x0620, 0xff, 0xff, 0x30) }, /* EM160R-GL */ { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, 0x0620, 0xff, 0, 0) }, + { USB_DEVICE_INTERFACE_CLASS(QUECTEL_VENDOR_ID, 0x0700, 0xff), /* BG95 */ + .driver_info = RSVD(3) | ZLP }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0xff, 0x30) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0xff, 0x10), diff --git a/drivers/usb/storage/karma.c b/drivers/usb/storage/karma.c index edcf2be0e0eb..09c8add5108a 100644 --- a/drivers/usb/storage/karma.c +++ b/drivers/usb/storage/karma.c @@ -172,23 +172,24 @@ static void rio_karma_destructor(void *extra) static int rio_karma_init(struct us_data *us) { - int ret = 0; struct karma_data *data = kzalloc(sizeof(struct karma_data), GFP_NOIO); if (!data) - goto out; + return -ENOMEM; data->recv = kmalloc(RIO_RECV_LEN, GFP_NOIO); if (!data->recv) { kfree(data); - goto out; + return -ENOMEM; } us->extra = data; us->extra_destructor = rio_karma_destructor; - ret = rio_karma_send_command(RIO_ENTER_STORAGE, us); - data->in_storage = (ret == 0); -out: - return ret; + if (rio_karma_send_command(RIO_ENTER_STORAGE, us)) + return -EIO; + + data->in_storage = 1; + + return 0; } static struct scsi_host_template karma_host_template; diff --git a/drivers/usb/usbip/stub_dev.c b/drivers/usb/usbip/stub_dev.c index 0081c1073b08..c64964c32cc9 100644 --- a/drivers/usb/usbip/stub_dev.c +++ b/drivers/usb/usbip/stub_dev.c @@ -427,7 +427,6 @@ static int stub_probe(struct usb_device *udev) (struct usb_dev_state *) udev); err_port: dev_set_drvdata(&udev->dev, NULL); - usb_put_dev(udev); /* we already have busid_priv, just lock busid_lock */ spin_lock(&busid_priv->busid_lock); @@ -442,6 +441,7 @@ static int stub_probe(struct usb_device *udev) put_busid_priv(busid_priv); sdev_free: + usb_put_dev(udev); stub_device_free(sdev); return rc; diff --git a/drivers/usb/usbip/stub_rx.c b/drivers/usb/usbip/stub_rx.c index 8c55cd833098..b88eeaee637a 100644 --- a/drivers/usb/usbip/stub_rx.c +++ b/drivers/usb/usbip/stub_rx.c @@ -138,7 +138,9 @@ static int tweak_set_configuration_cmd(struct urb *urb) req = (struct usb_ctrlrequest *) urb->setup_packet; config = le16_to_cpu(req->wValue); + usb_lock_device(sdev->udev); err = usb_set_configuration(sdev->udev, config); + usb_unlock_device(sdev->udev); if (err && err != -ENODEV) dev_err(&sdev->udev->dev, "can't set config #%d, error %d\n", config, err); diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index 97aa9b87e572..6b2efff1c297 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -263,7 +263,7 @@ __vringh_iov(struct vringh *vrh, u16 i, gfp_t gfp, int (*copy)(void *dst, const void *src, size_t len)) { - int err, count = 0, up_next, desc_max; + int err, count = 0, indirect_count = 0, up_next, desc_max; struct vring_desc desc, *descs; struct vringh_range range = { -1ULL, 0 }, slowrange; bool slow = false; @@ -320,7 +320,12 @@ __vringh_iov(struct vringh *vrh, u16 i, continue; } - if (count++ == vrh->vring.num) { + if (up_next == -1) + count++; + else + indirect_count++; + + if (count > vrh->vring.num || indirect_count > desc_max) { vringh_bad("Descriptor loop in %p", descs); err = -ELOOP; goto fail; @@ -382,6 +387,7 @@ __vringh_iov(struct vringh *vrh, u16 i, i = return_from_indirect(vrh, &up_next, &descs, &desc_max); slow = false; + indirect_count = 0; } else break; } diff --git a/drivers/video/fbdev/amba-clcd.c b/drivers/video/fbdev/amba-clcd.c index 38c1f324ce15..549f78e77255 100644 --- a/drivers/video/fbdev/amba-clcd.c +++ b/drivers/video/fbdev/amba-clcd.c @@ -838,12 +838,15 @@ static int clcdfb_of_vram_setup(struct clcd_fb *fb) return -ENODEV; fb->fb.screen_base = of_iomap(memory, 0); - if (!fb->fb.screen_base) + if (!fb->fb.screen_base) { + of_node_put(memory); return -ENOMEM; + } fb->fb.fix.smem_start = of_translate_address(memory, of_get_address(memory, 0, &size, NULL)); fb->fb.fix.smem_len = size; + of_node_put(memory); return 0; } diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index bf7959fdf9f4..7c2582892eab 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -3314,6 +3314,9 @@ static void fbcon_register_existing_fbs(struct work_struct *work) console_lock(); + deferred_takeover = false; + logo_shown = FBCON_LOGO_DONTSHOW; + for_each_registered_fb(i) fbcon_fb_registered(registered_fb[i]); @@ -3331,8 +3334,6 @@ static int fbcon_output_notifier(struct notifier_block *nb, pr_info("fbcon: Taking over console\n"); dummycon_unregister_output_notifier(&fbcon_output_nb); - deferred_takeover = false; - logo_shown = FBCON_LOGO_DONTSHOW; /* We may get called in atomic context */ schedule_work(&fbcon_deferred_takeover_work); diff --git a/drivers/video/fbdev/pxa3xx-gcu.c b/drivers/video/fbdev/pxa3xx-gcu.c index 69cfb337c857..43695a33f062 100644 --- a/drivers/video/fbdev/pxa3xx-gcu.c +++ b/drivers/video/fbdev/pxa3xx-gcu.c @@ -663,6 +663,7 @@ static int pxa3xx_gcu_probe(struct platform_device *pdev) for (i = 0; i < 8; i++) { ret = pxa3xx_gcu_add_buffer(dev, priv); if (ret) { + pxa3xx_gcu_free_buffers(dev, priv); dev_err(dev, "failed to allocate DMA memory\n"); goto err_disable_clk; } @@ -678,15 +679,15 @@ static int pxa3xx_gcu_probe(struct platform_device *pdev) SHARED_SIZE, irq); return 0; -err_free_dma: - dma_free_coherent(dev, SHARED_SIZE, - priv->shared, priv->shared_phys); +err_disable_clk: + clk_disable_unprepare(priv->clk); err_misc_deregister: misc_deregister(&priv->misc_dev); -err_disable_clk: - clk_disable_unprepare(priv->clk); +err_free_dma: + dma_free_coherent(dev, SHARED_SIZE, + priv->shared, priv->shared_phys); return ret; } @@ -699,6 +700,7 @@ static int pxa3xx_gcu_remove(struct platform_device *pdev) pxa3xx_gcu_wait_idle(priv); misc_deregister(&priv->misc_dev); dma_free_coherent(dev, SHARED_SIZE, priv->shared, priv->shared_phys); + clk_disable_unprepare(priv->clk); pxa3xx_gcu_free_buffers(dev, priv); return 0; diff --git a/fs/Kconfig b/fs/Kconfig index 48498e3fcc5d..02fc017e308d 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -260,6 +260,7 @@ source "fs/pstore/Kconfig" source "fs/sysv/Kconfig" source "fs/ufs/Kconfig" source "fs/exofs/Kconfig" +source "fs/erofs/Kconfig" endif # MISC_FILESYSTEMS diff --git a/fs/Makefile b/fs/Makefile index fb3be3f8ec6c..78af7f040d7b 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -134,3 +134,4 @@ obj-y += exofs/ # Multiple modules obj-$(CONFIG_CEPH_FS) += ceph/ obj-$(CONFIG_PSTORE) += pstore/ obj-$(CONFIG_EFIVAR_FS) += efivarfs/ +obj-$(CONFIG_EROFS_FS) += erofs/ diff --git a/fs/afs/dir.c b/fs/afs/dir.c index 54e7f6f1405e..59eb92484051 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -383,8 +383,11 @@ static int afs_dir_iterate_block(struct dir_context *ctx, } /* skip if starts before the current position */ - if (offset < curr) + if (offset < curr) { + if (next > curr) + ctx->pos = blkoff + next * sizeof(union afs_xdr_dirent); continue; + } /* found the next entry */ if (!dir_emit(ctx, dire->u.name, nlen, diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c index e4b59e76afb0..a6f97d86fb80 100644 --- a/fs/binfmt_flat.c +++ b/fs/binfmt_flat.c @@ -408,6 +408,30 @@ static void old_reloc(unsigned long rl) /****************************************************************************/ +static inline u32 __user *skip_got_header(u32 __user *rp) +{ + if (IS_ENABLED(CONFIG_RISCV)) { + /* + * RISC-V has a 16 byte GOT PLT header for elf64-riscv + * and 8 byte GOT PLT header for elf32-riscv. + * Skip the whole GOT PLT header, since it is reserved + * for the dynamic linker (ld.so). + */ + u32 rp_val0, rp_val1; + + if (get_user(rp_val0, rp)) + return rp; + if (get_user(rp_val1, rp + 1)) + return rp; + + if (rp_val0 == 0xffffffff && rp_val1 == 0xffffffff) + rp += 4; + else if (rp_val0 == 0xffffffff) + rp += 2; + } + return rp; +} + static int load_flat_file(struct linux_binprm *bprm, struct lib_info *libinfo, int id, unsigned long *extra_stack) { @@ -745,7 +769,8 @@ static int load_flat_file(struct linux_binprm *bprm, * image. */ if (flags & FLAT_FLAG_GOTPIC) { - for (rp = (u32 __user *)datapos; ; rp++) { + rp = skip_got_header((u32 __user *) datapos); + for (; ; rp++) { u32 addr, rp_val; if (get_user(rp_val, rp)) return -EFAULT; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2ac920bdf4df..c9fd018dcf76 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2855,7 +2855,7 @@ int open_ctree(struct super_block *sb, ~BTRFS_FEATURE_INCOMPAT_SUPP; if (features) { btrfs_err(fs_info, - "cannot mount because of unsupported optional features (%llx)", + "cannot mount because of unsupported optional features (0x%llx)", features); err = -EINVAL; goto fail_alloc; @@ -2915,7 +2915,7 @@ int open_ctree(struct super_block *sb, ~BTRFS_FEATURE_COMPAT_RO_SUPP; if (!sb_rdonly(sb) && features) { btrfs_err(fs_info, - "cannot mount read-write because of unsupported optional features (%llx)", + "cannot mount read-write because of unsupported optional features (0x%llx)", features); err = -EINVAL; goto fail_alloc; diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 8f05e6a472c3..2b4d33b58a68 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -6925,12 +6925,12 @@ int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info) * do another round of validation checks. */ if (total_dev != fs_info->fs_devices->total_devices) { - btrfs_err(fs_info, - "super_num_devices %llu mismatch with num_devices %llu found here", + btrfs_warn(fs_info, +"super block num_devices %llu mismatch with DEV_ITEM count %llu, will be repaired on next transaction commit", btrfs_super_num_devices(fs_info->super_copy), total_dev); - ret = -EINVAL; - goto error; + fs_info->fs_devices->total_devices = total_dev; + btrfs_set_super_num_devices(fs_info->super_copy, total_dev); } if (btrfs_super_total_bytes(fs_info->super_copy) < fs_info->fs_devices->total_rw_bytes) { diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index a09ce27ab220..6fa9a784676b 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -273,6 +273,14 @@ static size_t ceph_vxattrcb_quota_max_files(struct ceph_inode_info *ci, } #define XATTR_RSTAT_FIELD(_type, _name) \ XATTR_NAME_CEPH(_type, _name, VXATTR_FLAG_RSTAT) +#define XATTR_RSTAT_FIELD_UPDATABLE(_type, _name) \ + { \ + .name = CEPH_XATTR_NAME(_type, _name), \ + .name_size = sizeof (CEPH_XATTR_NAME(_type, _name)), \ + .getxattr_cb = ceph_vxattrcb_ ## _type ## _ ## _name, \ + .exists_cb = NULL, \ + .flags = VXATTR_FLAG_RSTAT, \ + } #define XATTR_LAYOUT_FIELD(_type, _name, _field) \ { \ .name = CEPH_XATTR_NAME2(_type, _name, _field), \ @@ -310,7 +318,7 @@ static struct ceph_vxattr ceph_dir_vxattrs[] = { XATTR_RSTAT_FIELD(dir, rfiles), XATTR_RSTAT_FIELD(dir, rsubdirs), XATTR_RSTAT_FIELD(dir, rbytes), - XATTR_RSTAT_FIELD(dir, rctime), + XATTR_RSTAT_FIELD_UPDATABLE(dir, rctime), { .name = "ceph.quota", .name_size = sizeof("ceph.quota"), diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 43478ec6fd67..3485b9bf970f 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -265,6 +265,9 @@ smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon) rc = -EHOSTDOWN; mutex_unlock(&tcon->ses->session_mutex); goto failed; + } else if (rc) { + mutex_unlock(&ses->session_mutex); + goto out; } } if (rc || !tcon->need_reconnect) { diff --git a/fs/dax.c b/fs/dax.c index d09701aa6f2f..7451efc5020c 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -907,7 +907,8 @@ static void dax_mapping_entry_mkclean(struct address_space *mapping, if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp)) goto unlock_pmd; - flush_cache_page(vma, address, pfn); + flush_cache_range(vma, address, + address + HPAGE_PMD_SIZE); pmd = pmdp_invalidate(vma, address, pmdp); pmd = pmd_wrprotect(pmd); pmd = pmd_mkclean(pmd); diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index 26a4847efccc..ac53403e9edb 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -1553,6 +1553,7 @@ static int _remove_from_waiters(struct dlm_lkb *lkb, int mstype, lkb->lkb_wait_type = 0; lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; lkb->lkb_wait_count--; + unhold_lkb(lkb); goto out_del; } @@ -1579,6 +1580,7 @@ static int _remove_from_waiters(struct dlm_lkb *lkb, int mstype, log_error(ls, "remwait error %x reply %d wait_type %d overlap", lkb->lkb_id, mstype, lkb->lkb_wait_type); lkb->lkb_wait_count--; + unhold_lkb(lkb); lkb->lkb_wait_type = 0; } @@ -5314,11 +5316,16 @@ int dlm_recover_waiters_post(struct dlm_ls *ls) lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; lkb->lkb_wait_type = 0; - lkb->lkb_wait_count = 0; + /* drop all wait_count references we still + * hold a reference for this iteration. + */ + while (lkb->lkb_wait_count) { + lkb->lkb_wait_count--; + unhold_lkb(lkb); + } mutex_lock(&ls->ls_waiters_mutex); list_del_init(&lkb->lkb_wait_reply); mutex_unlock(&ls->ls_waiters_mutex); - unhold_lkb(lkb); /* for waiters list */ if (oc || ou) { /* do an unlock or cancel instead of resending */ diff --git a/fs/dlm/plock.c b/fs/dlm/plock.c index c7d5a2ea3d03..e0c05e08d8bf 100644 --- a/fs/dlm/plock.c +++ b/fs/dlm/plock.c @@ -26,11 +26,11 @@ struct plock_op { struct list_head list; int done; struct dlm_plock_info info; + int (*callback)(struct file_lock *fl, int result); }; struct plock_xop { struct plock_op xop; - int (*callback)(struct file_lock *fl, int result); void *fl; void *file; struct file_lock flc; @@ -132,19 +132,18 @@ int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file, /* fl_owner is lockd which doesn't distinguish processes on the nfs client */ op->info.owner = (__u64) fl->fl_pid; - xop->callback = fl->fl_lmops->lm_grant; + op->callback = fl->fl_lmops->lm_grant; locks_init_lock(&xop->flc); locks_copy_lock(&xop->flc, fl); xop->fl = fl; xop->file = file; } else { op->info.owner = (__u64)(long) fl->fl_owner; - xop->callback = NULL; } send_op(op); - if (xop->callback == NULL) { + if (!op->callback) { rv = wait_event_interruptible(recv_wq, (op->done != 0)); if (rv == -ERESTARTSYS) { log_debug(ls, "dlm_posix_lock: wait killed %llx", @@ -206,7 +205,7 @@ static int dlm_plock_callback(struct plock_op *op) file = xop->file; flc = &xop->flc; fl = xop->fl; - notify = xop->callback; + notify = op->callback; if (op->info.rv) { notify(fl, op->info.rv); @@ -439,10 +438,9 @@ static ssize_t dev_write(struct file *file, const char __user *u, size_t count, if (op->info.fsid == info.fsid && op->info.number == info.number && op->info.owner == info.owner) { - struct plock_xop *xop = (struct plock_xop *)op; list_del_init(&op->list); memcpy(&op->info, &info, sizeof(info)); - if (xop->callback) + if (op->callback) do_callback = 1; else op->done = 1; diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig new file mode 100644 index 000000000000..858b3339f381 --- /dev/null +++ b/fs/erofs/Kconfig @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config EROFS_FS + tristate "EROFS filesystem support" + depends on BLOCK + select LIBCRC32C + help + EROFS (Enhanced Read-Only File System) is a lightweight + read-only file system with modern designs (eg. page-sized + blocks, inline xattrs/data, etc.) for scenarios which need + high-performance read-only requirements, e.g. Android OS + for mobile phones and LIVECDs. + + It also provides fixed-sized output compression support, + which improves storage density, keeps relatively higher + compression ratios, which is more useful to achieve high + performance for embedded devices with limited memory. + + If unsure, say N. + +config EROFS_FS_DEBUG + bool "EROFS debugging feature" + depends on EROFS_FS + help + Print debugging messages and enable more BUG_ONs which check + filesystem consistency and find potential issues aggressively, + which can be used for Android eng build, for example. + + For daily use, say N. + +config EROFS_FS_XATTR + bool "EROFS extended attributes" + depends on EROFS_FS + default y + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page, or visit + for details). + + If unsure, say N. + +config EROFS_FS_POSIX_ACL + bool "EROFS Access Control Lists" + depends on EROFS_FS_XATTR + select FS_POSIX_ACL + default y + help + Posix Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + To learn more about Access Control Lists, visit the POSIX ACLs for + Linux website . + + If you don't know what Access Control Lists are, say N. + +config EROFS_FS_SECURITY + bool "EROFS Security Labels" + depends on EROFS_FS_XATTR + default y + help + Security labels provide an access control facility to support Linux + Security Models (LSMs) accepted by AppArmor, SELinux, Smack and TOMOYO + Linux. This option enables an extended attribute handler for file + security labels in the erofs filesystem, so that it requires enabling + the extended attribute support in advance. + + If you are not using a security module, say N. + +config EROFS_FS_ZIP + bool "EROFS Data Compression Support" + depends on EROFS_FS + select LZ4_DECOMPRESS + default y + help + Enable fixed-sized output compression for EROFS. + + If you don't want to enable compression feature, say N. + diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile new file mode 100644 index 000000000000..1f9aced49070 --- /dev/null +++ b/fs/erofs/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_EROFS_FS) += erofs.o +erofs-objs := super.o inode.o data.o namei.o dir.o utils.o pcpubuf.o +erofs-$(CONFIG_EROFS_FS_XATTR) += xattr.o +erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o diff --git a/drivers/staging/erofs/TODO b/fs/erofs/TODO similarity index 100% rename from drivers/staging/erofs/TODO rename to fs/erofs/TODO diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h new file mode 100644 index 000000000000..3701c72bacb2 --- /dev/null +++ b/fs/erofs/compress.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_FS_COMPRESS_H +#define __EROFS_FS_COMPRESS_H + +#include "internal.h" + +enum { + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_RUNTIME_MAX +}; + +struct z_erofs_decompress_req { + struct super_block *sb; + struct page **in, **out; + + unsigned short pageofs_out; + unsigned int inputsize, outputsize; + + /* indicate the algorithm will be used for decompression */ + unsigned int alg; + bool inplace_io, partial_decoding; +}; + +/* some special page->private (unsigned long, see below) */ +#define Z_EROFS_SHORTLIVED_PAGE (-1UL << 2) +#define Z_EROFS_PREALLOCATED_PAGE (-2UL << 2) + +/* + * For all pages in a pcluster, page->private should be one of + * Type Last 2bits page->private + * short-lived page 00 Z_EROFS_SHORTLIVED_PAGE + * preallocated page (tryalloc) 00 Z_EROFS_PREALLOCATED_PAGE + * cached/managed page 00 pointer to z_erofs_pcluster + * online page (file-backed, 01/10/11 sub-index << 2 | count + * some pages can be used for inplace I/O) + * + * page->mapping should be one of + * Type page->mapping + * short-lived page NULL + * preallocated page NULL + * cached/managed page non-NULL or NULL (invalidated/truncated page) + * online page non-NULL + * + * For all managed pages, PG_private should be set with 1 extra refcount, + * which is used for page reclaim / migration. + */ + +/* + * short-lived pages are pages directly from buddy system with specific + * page->private (no need to set PagePrivate since these are non-LRU / + * non-movable pages and bypass reclaim / migration code). + */ +static inline bool z_erofs_is_shortlived_page(struct page *page) +{ + if (page->private != Z_EROFS_SHORTLIVED_PAGE) + return false; + + DBG_BUGON(page->mapping); + return true; +} + +static inline bool z_erofs_put_shortlivedpage(struct list_head *pagepool, + struct page *page) +{ + if (!z_erofs_is_shortlived_page(page)) + return false; + + /* short-lived pages should not be used by others at the same time */ + if (page_ref_count(page) > 1) { + put_page(page); + } else { + /* follow the pcluster rule above. */ + set_page_private(page, 0); + list_add(&page->lru, pagepool); + } + return true; +} + +int z_erofs_decompress(struct z_erofs_decompress_req *rq, + struct list_head *pagepool); + +#endif diff --git a/drivers/staging/erofs/data.c b/fs/erofs/data.c similarity index 57% rename from drivers/staging/erofs/data.c rename to fs/erofs/data.c index 894e60ecebe2..dc264a819dc2 100644 --- a/drivers/staging/erofs/data.c +++ b/fs/erofs/data.c @@ -1,21 +1,14 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* - * linux/drivers/staging/erofs/data.c - * * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. + * https://www.huawei.com/ */ #include "internal.h" #include #include -static inline void read_endio(struct bio *bio) +static void erofs_readendio(struct bio *bio) { int i; struct bio_vec *bvec; @@ -27,7 +20,7 @@ static inline void read_endio(struct bio *bio) /* page is already locked */ DBG_BUGON(PageUptodate(page)); - if (unlikely(err)) + if (err) SetPageError(page); else SetPageUptodate(page); @@ -38,70 +31,35 @@ static inline void read_endio(struct bio *bio) bio_put(bio); } -/* prio -- true is used for dir */ -struct page *erofs_get_meta_page(struct super_block *sb, - erofs_blk_t blkaddr, bool prio) +struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr) { - struct inode *bd_inode = sb->s_bdev->bd_inode; - struct address_space *mapping = bd_inode->i_mapping; + struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping; struct page *page; -repeat: - page = find_or_create_page(mapping, blkaddr, - /* - * Prefer looping in the allocator rather than here, - * at least that code knows what it's doing. - */ - mapping_gfp_constraint(mapping, ~__GFP_FS) | __GFP_NOFAIL); - - BUG_ON(!page || !PageLocked(page)); - - if (!PageUptodate(page)) { - struct bio *bio; - int err; - - bio = prepare_bio(sb, blkaddr, 1, read_endio); - err = bio_add_page(bio, page, PAGE_SIZE, 0); - BUG_ON(err != PAGE_SIZE); - - __submit_bio(bio, REQ_OP_READ, - REQ_META | (prio ? REQ_PRIO : 0)); - + page = read_cache_page_gfp(mapping, blkaddr, + mapping_gfp_constraint(mapping, ~__GFP_FS)); + /* should already be PageUptodate */ + if (!IS_ERR(page)) lock_page(page); - - /* the page has been truncated by others? */ - if (unlikely(page->mapping != mapping)) { - unlock_page(page); - put_page(page); - goto repeat; - } - - /* more likely a read error */ - if (unlikely(!PageUptodate(page))) { - unlock_page(page); - put_page(page); - - page = ERR_PTR(-EIO); - } - } return page; } static int erofs_map_blocks_flatmode(struct inode *inode, - struct erofs_map_blocks *map, - int flags) + struct erofs_map_blocks *map, + int flags) { int err = 0; erofs_blk_t nblocks, lastblk; u64 offset = map->m_la; - struct erofs_vnode *vi = EROFS_V(inode); + struct erofs_inode *vi = EROFS_I(inode); + bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE); trace_erofs_map_blocks_flatmode_enter(inode, map, flags); nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE); - lastblk = nblocks - is_inode_layout_inline(inode); + lastblk = nblocks - tailendpacking; - if (unlikely(offset >= inode->i_size)) { + if (offset >= inode->i_size) { /* leave out-of-bound access unmapped */ map->m_flags = 0; map->m_plen = 0; @@ -114,7 +72,7 @@ static int erofs_map_blocks_flatmode(struct inode *inode, if (offset < blknr_to_addr(lastblk)) { map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la; map->m_plen = blknr_to_addr(lastblk) - offset; - } else if (is_inode_layout_inline(inode)) { + } else if (tailendpacking) { /* 2 - inode inline B: inode, [xattrs], inline last blk... */ struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); @@ -122,17 +80,21 @@ static int erofs_map_blocks_flatmode(struct inode *inode, vi->xattr_isize + erofs_blkoff(map->m_la); map->m_plen = inode->i_size - offset; - /* inline data should locate in one meta block */ + /* inline data should be located in one meta block */ if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) { + erofs_err(inode->i_sb, + "inline data cross block boundary @ nid %llu", + vi->nid); DBG_BUGON(1); - err = -EIO; + err = -EFSCORRUPTED; goto err_out; } map->m_flags |= EROFS_MAP_META; } else { - errln("internal error @ nid: %llu (size %llu), m_la 0x%llx", - vi->nid, inode->i_size, map->m_la); + erofs_err(inode->i_sb, + "internal error @ nid: %llu (size %llu), m_la 0x%llx", + vi->nid, inode->i_size, map->m_la); DBG_BUGON(1); err = -EIO; goto err_out; @@ -146,56 +108,15 @@ static int erofs_map_blocks_flatmode(struct inode *inode, return err; } -#ifdef CONFIG_EROFS_FS_ZIP -extern int z_erofs_map_blocks_iter(struct inode *, - struct erofs_map_blocks *, struct page **, int); -#endif - -int erofs_map_blocks_iter(struct inode *inode, - struct erofs_map_blocks *map, - struct page **mpage_ret, int flags) -{ - /* by default, reading raw data never use erofs_map_blocks_iter */ - if (unlikely(!is_inode_layout_compression(inode))) { - if (*mpage_ret != NULL) - put_page(*mpage_ret); - *mpage_ret = NULL; - - return erofs_map_blocks(inode, map, flags); - } - -#ifdef CONFIG_EROFS_FS_ZIP - return z_erofs_map_blocks_iter(inode, map, mpage_ret, flags); -#else - /* data compression is not available */ - return -ENOTSUPP; -#endif -} - -int erofs_map_blocks(struct inode *inode, - struct erofs_map_blocks *map, int flags) -{ - if (unlikely(is_inode_layout_compression(inode))) { - struct page *mpage = NULL; - int err; - - err = erofs_map_blocks_iter(inode, map, &mpage, flags); - if (mpage != NULL) - put_page(mpage); - return err; - } - return erofs_map_blocks_flatmode(inode, map, flags); -} - -static inline struct bio *erofs_read_raw_page( - struct bio *bio, - struct address_space *mapping, - struct page *page, - erofs_off_t *last_block, - unsigned nblocks, - bool ra) +static inline struct bio *erofs_read_raw_page(struct bio *bio, + struct address_space *mapping, + struct page *page, + erofs_off_t *last_block, + unsigned int nblocks, + bool ra) { - struct inode *inode = mapping->host; + struct inode *const inode = mapping->host; + struct super_block *const sb = inode->i_sb; erofs_off_t current_block = (erofs_off_t)page->index; int err; @@ -206,34 +127,28 @@ static inline struct bio *erofs_read_raw_page( goto has_updated; } - if (cleancache_get_page(page) == 0) { - err = 0; - SetPageUptodate(page); - goto has_updated; - } - /* note that for readpage case, bio also equals to NULL */ - if (bio != NULL && - /* not continuous */ - *last_block + 1 != current_block) { + if (bio && + /* not continuous */ + *last_block + 1 != current_block) { submit_bio_retry: - __submit_bio(bio, REQ_OP_READ, 0); + submit_bio(bio); bio = NULL; } - if (bio == NULL) { + if (!bio) { struct erofs_map_blocks map = { .m_la = blknr_to_addr(current_block), }; erofs_blk_t blknr; - unsigned blkoff; + unsigned int blkoff; - err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); - if (unlikely(err)) + err = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW); + if (err) goto err_out; /* zero out the holed page */ - if (unlikely(!(map.m_flags & EROFS_MAP_MAPPED))) { + if (!(map.m_flags & EROFS_MAP_MAPPED)) { zero_user_segment(page, 0, PAGE_SIZE); SetPageUptodate(page); @@ -254,7 +169,7 @@ static inline struct bio *erofs_read_raw_page( DBG_BUGON(map.m_plen > PAGE_SIZE); - ipage = erofs_get_meta_page(inode->i_sb, blknr, 0); + ipage = erofs_get_meta_page(inode->i_sb, blknr); if (IS_ERR(ipage)) { err = PTR_ERR(ipage); @@ -287,7 +202,13 @@ static inline struct bio *erofs_read_raw_page( if (nblocks > BIO_MAX_PAGES) nblocks = BIO_MAX_PAGES; - bio = prepare_bio(inode->i_sb, blknr, nblocks, read_endio); + bio = bio_alloc(GFP_NOIO, nblocks); + + bio->bi_end_io = erofs_readendio; + bio_set_dev(bio, sb->s_bdev); + bio->bi_iter.bi_sector = (sector_t)blknr << + LOG_SECTORS_PER_BLOCK; + bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0); } err = bio_add_page(bio, page, PAGE_SIZE, 0); @@ -298,7 +219,7 @@ static inline struct bio *erofs_read_raw_page( *last_block = current_block; /* shift in advance in case of it followed by too many gaps */ - if (unlikely(bio->bi_vcnt >= bio->bi_max_vecs)) { + if (bio->bi_iter.bi_size >= bio->bi_max_vecs * PAGE_SIZE) { /* err should reassign to 0 after submitting */ err = 0; goto submit_bio_out; @@ -316,11 +237,10 @@ static inline struct bio *erofs_read_raw_page( unlock_page(page); /* if updated manually, continuous pages has a gap */ - if (bio != NULL) + if (bio) submit_bio_out: - __submit_bio(bio, REQ_OP_READ, 0); - - return unlikely(err) ? ERR_PTR(err) : NULL; + submit_bio(bio); + return err ? ERR_PTR(err) : NULL; } /* @@ -335,7 +255,7 @@ static int erofs_raw_access_readpage(struct file *file, struct page *page) trace_erofs_readpage(page, true); bio = erofs_read_raw_page(NULL, page->mapping, - page, &last_block, 1, false); + page, &last_block, 1, false); if (IS_ERR(bio)) return PTR_ERR(bio); @@ -345,8 +265,9 @@ static int erofs_raw_access_readpage(struct file *file, struct page *page) } static int erofs_raw_access_readpages(struct file *filp, - struct address_space *mapping, - struct list_head *pages, unsigned nr_pages) + struct address_space *mapping, + struct list_head *pages, + unsigned int nr_pages) { erofs_off_t last_block; struct bio *bio = NULL; @@ -363,13 +284,13 @@ static int erofs_raw_access_readpages(struct file *filp, if (!add_to_page_cache_lru(page, mapping, page->index, gfp)) { bio = erofs_read_raw_page(bio, mapping, page, - &last_block, nr_pages, true); + &last_block, nr_pages, true); /* all the page errors are ignored when readahead */ if (IS_ERR(bio)) { pr_err("%s, readahead error at page %lu of nid %llu\n", - __func__, page->index, - EROFS_V(mapping->host)->nid); + __func__, page->index, + EROFS_I(mapping->host)->nid); bio = NULL; } @@ -381,8 +302,28 @@ static int erofs_raw_access_readpages(struct file *filp, DBG_BUGON(!list_empty(pages)); /* the rare case (end in gaps) */ - if (unlikely(bio != NULL)) - __submit_bio(bio, REQ_OP_READ, 0); + if (bio) + submit_bio(bio); + return 0; +} + +static sector_t erofs_bmap(struct address_space *mapping, sector_t block) +{ + struct inode *inode = mapping->host; + struct erofs_map_blocks map = { + .m_la = blknr_to_addr(block), + }; + + if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) { + erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE; + + if (block >> LOG_SECTORS_PER_BLOCK >= blks) + return 0; + } + + if (!erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW)) + return erofs_blknr(map.m_pa); + return 0; } @@ -390,5 +331,5 @@ static int erofs_raw_access_readpages(struct file *filp, const struct address_space_operations erofs_raw_access_aops = { .readpage = erofs_raw_access_readpage, .readpages = erofs_raw_access_readpages, + .bmap = erofs_bmap, }; - diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c new file mode 100644 index 000000000000..ad3f31380e6b --- /dev/null +++ b/fs/erofs/decompressor.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#include "compress.h" +#include +#include + +#ifndef LZ4_DISTANCE_MAX /* history window size */ +#define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1) +#ifndef LZ4_DECOMPRESS_INPLACE_MARGIN +#define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize) (((srcsize) >> 8) + 32) +#endif + +struct z_erofs_decompressor { + /* + * if destpages have sparsed pages, fill them with bounce pages. + * it also check whether destpages indicate continuous physical memory. + */ + int (*prepare_destpages)(struct z_erofs_decompress_req *rq, + struct list_head *pagepool); + int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out); + char *name; +}; + +int z_erofs_load_lz4_config(struct super_block *sb, + struct erofs_super_block *dsb, + struct z_erofs_lz4_cfgs *lz4, int size) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + u16 distance; + + if (lz4) { + if (size < sizeof(struct z_erofs_lz4_cfgs)) { + erofs_err(sb, "invalid lz4 cfgs, size=%u", size); + return -EINVAL; + } + distance = le16_to_cpu(lz4->max_distance); + + sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); + if (!sbi->lz4.max_pclusterblks) { + sbi->lz4.max_pclusterblks = 1; /* reserved case */ + } else if (sbi->lz4.max_pclusterblks > + Z_EROFS_PCLUSTER_MAX_SIZE / EROFS_BLKSIZ) { + erofs_err(sb, "too large lz4 pclusterblks %u", + sbi->lz4.max_pclusterblks); + return -EINVAL; + } else if (sbi->lz4.max_pclusterblks >= 2) { + erofs_info(sb, "EXPERIMENTAL big pcluster feature in use. Use at your own risk!"); + } + } else { + distance = le16_to_cpu(dsb->u1.lz4_max_distance); + sbi->lz4.max_pclusterblks = 1; + } + + sbi->lz4.max_distance_pages = distance ? + DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : + LZ4_MAX_DISTANCE_PAGES; + return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks); +} + +static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, + struct list_head *pagepool) +{ + const unsigned int nr = + PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; + struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; + unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, + BITS_PER_LONG)] = { 0 }; + unsigned int lz4_max_distance_pages = + EROFS_SB(rq->sb)->lz4.max_distance_pages; + void *kaddr = NULL; + unsigned int i, j, top; + + top = 0; + for (i = j = 0; i < nr; ++i, ++j) { + struct page *const page = rq->out[i]; + struct page *victim; + + if (j >= lz4_max_distance_pages) + j = 0; + + /* 'valid' bounced can only be tested after a complete round */ + if (test_bit(j, bounced)) { + DBG_BUGON(i < lz4_max_distance_pages); + DBG_BUGON(top >= lz4_max_distance_pages); + availables[top++] = rq->out[i - lz4_max_distance_pages]; + } + + if (page) { + __clear_bit(j, bounced); + if (kaddr) { + if (kaddr + PAGE_SIZE == page_address(page)) + kaddr += PAGE_SIZE; + else + kaddr = NULL; + } else if (!i) { + kaddr = page_address(page); + } + continue; + } + kaddr = NULL; + __set_bit(j, bounced); + + if (top) { + victim = availables[--top]; + get_page(victim); + } else { + victim = erofs_allocpage(pagepool, + GFP_KERNEL | __GFP_NOFAIL); + set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); + } + rq->out[i] = victim; + } + return kaddr ? 1 : 0; +} + +static void *z_erofs_handle_inplace_io(struct z_erofs_decompress_req *rq, + void *inpage, unsigned int *inputmargin, int *maptype, + bool support_0padding) +{ + unsigned int nrpages_in, nrpages_out; + unsigned int ofull, oend, inputsize, total, i, j; + struct page **in; + void *src, *tmp; + + inputsize = rq->inputsize; + nrpages_in = PAGE_ALIGN(inputsize) >> PAGE_SHIFT; + oend = rq->pageofs_out + rq->outputsize; + ofull = PAGE_ALIGN(oend); + nrpages_out = ofull >> PAGE_SHIFT; + + if (rq->inplace_io) { + if (rq->partial_decoding || !support_0padding || + ofull - oend < LZ4_DECOMPRESS_INPLACE_MARGIN(inputsize)) + goto docopy; + + for (i = 0; i < nrpages_in; ++i) { + DBG_BUGON(rq->in[i] == NULL); + for (j = 0; j < nrpages_out - nrpages_in + i; ++j) + if (rq->out[j] == rq->in[i]) + goto docopy; + } + } + + if (nrpages_in <= 1) { + *maptype = 0; + return inpage; + } + kunmap_atomic(inpage); + might_sleep(); + src = erofs_vm_map_ram(rq->in, nrpages_in); + if (!src) + return ERR_PTR(-ENOMEM); + *maptype = 1; + return src; + +docopy: + /* Or copy compressed data which can be overlapped to per-CPU buffer */ + in = rq->in; + src = erofs_get_pcpubuf(nrpages_in); + if (!src) { + DBG_BUGON(1); + kunmap_atomic(inpage); + return ERR_PTR(-EFAULT); + } + + tmp = src; + total = rq->inputsize; + while (total) { + unsigned int page_copycnt = + min_t(unsigned int, total, PAGE_SIZE - *inputmargin); + + if (!inpage) + inpage = kmap_atomic(*in); + memcpy(tmp, inpage + *inputmargin, page_copycnt); + kunmap_atomic(inpage); + inpage = NULL; + tmp += page_copycnt; + total -= page_copycnt; + ++in; + *inputmargin = 0; + } + *maptype = 2; + return src; +} + +static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out) +{ + unsigned int inputmargin; + u8 *headpage, *src; + bool support_0padding; + int ret, maptype; + + DBG_BUGON(*rq->in == NULL); + headpage = kmap_atomic(*rq->in); + inputmargin = 0; + support_0padding = false; + + /* decompression inplace is only safe when 0padding is enabled */ + if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) { + support_0padding = true; + + while (!headpage[inputmargin & ~PAGE_MASK]) + if (!(++inputmargin & ~PAGE_MASK)) + break; + + if (inputmargin >= rq->inputsize) { + kunmap_atomic(headpage); + return -EIO; + } + } + + rq->inputsize -= inputmargin; + src = z_erofs_handle_inplace_io(rq, headpage, &inputmargin, &maptype, + support_0padding); + if (IS_ERR(src)) + return PTR_ERR(src); + + /* legacy format could compress extra data in a pcluster. */ + if (rq->partial_decoding || !support_0padding) + ret = LZ4_decompress_safe_partial(src + inputmargin, out, + rq->inputsize, rq->outputsize, rq->outputsize); + else + ret = LZ4_decompress_safe(src + inputmargin, out, + rq->inputsize, rq->outputsize); + + if (ret != rq->outputsize) { + erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]", + ret, rq->inputsize, inputmargin, rq->outputsize); + + print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET, + 16, 1, src + inputmargin, rq->inputsize, true); + print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET, + 16, 1, out, rq->outputsize, true); + + if (ret >= 0) + memset(out + ret, 0, rq->outputsize - ret); + ret = -EIO; + } + + if (maptype == 0) { + kunmap_atomic(src); + } else if (maptype == 1) { + vm_unmap_ram(src, PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT); + } else if (maptype == 2) { + erofs_put_pcpubuf(src); + } else { + DBG_BUGON(1); + return -EFAULT; + } + return ret; +} + +static struct z_erofs_decompressor decompressors[] = { + [Z_EROFS_COMPRESSION_SHIFTED] = { + .name = "shifted" + }, + [Z_EROFS_COMPRESSION_LZ4] = { + .prepare_destpages = z_erofs_lz4_prepare_destpages, + .decompress = z_erofs_lz4_decompress, + .name = "lz4" + }, +}; + +static void copy_from_pcpubuf(struct page **out, const char *dst, + unsigned short pageofs_out, + unsigned int outputsize) +{ + const char *end = dst + outputsize; + const unsigned int righthalf = PAGE_SIZE - pageofs_out; + const char *cur = dst - pageofs_out; + + while (cur < end) { + struct page *const page = *out++; + + if (page) { + char *buf = kmap_atomic(page); + + if (cur >= dst) { + memcpy(buf, cur, min_t(uint, PAGE_SIZE, + end - cur)); + } else { + memcpy(buf + pageofs_out, cur + pageofs_out, + min_t(uint, righthalf, end - cur)); + } + kunmap_atomic(buf); + } + cur += PAGE_SIZE; + } +} + +static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq, + struct list_head *pagepool) +{ + const unsigned int nrpages_out = + PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; + const struct z_erofs_decompressor *alg = decompressors + rq->alg; + unsigned int dst_maptype; + void *dst; + int ret; + + /* two optimized fast paths only for non bigpcluster cases yet */ + if (rq->inputsize <= PAGE_SIZE) { + if (nrpages_out == 1 && !rq->inplace_io) { + DBG_BUGON(!*rq->out); + dst = kmap_atomic(*rq->out); + dst_maptype = 0; + goto dstmap_out; + } + + /* + * For the case of small output size (especially much less + * than PAGE_SIZE), memcpy the decompressed data rather than + * compressed data is preferred. + */ + if (rq->outputsize <= PAGE_SIZE * 7 / 8) { + dst = erofs_get_pcpubuf(1); + if (IS_ERR(dst)) + return PTR_ERR(dst); + + rq->inplace_io = false; + ret = alg->decompress(rq, dst); + if (!ret) + copy_from_pcpubuf(rq->out, dst, rq->pageofs_out, + rq->outputsize); + + erofs_put_pcpubuf(dst); + return ret; + } + } + + /* general decoding path which can be used for all cases */ + ret = alg->prepare_destpages(rq, pagepool); + if (ret < 0) + return ret; + if (ret) { + dst = page_address(*rq->out); + dst_maptype = 1; + goto dstmap_out; + } + + dst = erofs_vm_map_ram(rq->out, nrpages_out); + if (!dst) + return -ENOMEM; + dst_maptype = 2; + +dstmap_out: + ret = alg->decompress(rq, dst + rq->pageofs_out); + + if (!dst_maptype) + kunmap_atomic(dst); + else if (dst_maptype == 2) + vm_unmap_ram(dst, nrpages_out); + return ret; +} + +static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq, + struct list_head *pagepool) +{ + const unsigned int nrpages_out = + PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; + const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out; + unsigned char *src, *dst; + + if (nrpages_out > 2) { + DBG_BUGON(1); + return -EIO; + } + + if (rq->out[0] == *rq->in) { + DBG_BUGON(nrpages_out != 1); + return 0; + } + + src = kmap_atomic(*rq->in); + if (rq->out[0]) { + dst = kmap_atomic(rq->out[0]); + memcpy(dst + rq->pageofs_out, src, righthalf); + kunmap_atomic(dst); + } + + if (nrpages_out == 2) { + DBG_BUGON(!rq->out[1]); + if (rq->out[1] == *rq->in) { + memmove(src, src + righthalf, rq->pageofs_out); + } else { + dst = kmap_atomic(rq->out[1]); + memcpy(dst, src + righthalf, rq->pageofs_out); + kunmap_atomic(dst); + } + } + kunmap_atomic(src); + return 0; +} + +int z_erofs_decompress(struct z_erofs_decompress_req *rq, + struct list_head *pagepool) +{ + if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) + return z_erofs_shifted_transform(rq, pagepool); + return z_erofs_decompress_generic(rq, pagepool); +} diff --git a/drivers/staging/erofs/dir.c b/fs/erofs/dir.c similarity index 62% rename from drivers/staging/erofs/dir.c rename to fs/erofs/dir.c index fe6683effd05..722b653c7a77 100644 --- a/drivers/staging/erofs/dir.c +++ b/fs/erofs/dir.c @@ -1,14 +1,7 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* - * linux/drivers/staging/erofs/dir.c - * * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. + * https://www.huawei.com/ */ #include "internal.h" @@ -33,19 +26,18 @@ static void debug_one_dentry(unsigned char d_type, const char *de_name, memcpy(dbg_namebuf, de_name, de_namelen); dbg_namebuf[de_namelen] = '\0'; - debugln("found dirent %s de_len %u d_type %d", dbg_namebuf, - de_namelen, d_type); + erofs_dbg("found dirent %s de_len %u d_type %d", dbg_namebuf, + de_namelen, d_type); #endif } -static int erofs_fill_dentries(struct dir_context *ctx, - void *dentry_blk, unsigned *ofs, - unsigned nameoff, unsigned maxsize) +static int erofs_fill_dentries(struct inode *dir, struct dir_context *ctx, + void *dentry_blk, unsigned int *ofs, + unsigned int nameoff, unsigned int maxsize) { - struct erofs_dirent *de = dentry_blk; + struct erofs_dirent *de = dentry_blk + *ofs; const struct erofs_dirent *end = dentry_blk + nameoff; - de = dentry_blk + *ofs; while (de < end) { const char *de_name; unsigned int de_namelen; @@ -66,16 +58,18 @@ static int erofs_fill_dentries(struct dir_context *ctx, de_namelen = le16_to_cpu(de[1].nameoff) - nameoff; /* a corrupted entry is found */ - if (unlikely(nameoff + de_namelen > maxsize || - de_namelen > EROFS_NAME_LEN)) { + if (nameoff + de_namelen > maxsize || + de_namelen > EROFS_NAME_LEN) { + erofs_err(dir->i_sb, "bogus dirent @ nid %llu", + EROFS_I(dir)->nid); DBG_BUGON(1); - return -EIO; + return -EFSCORRUPTED; } debug_one_dentry(d_type, de_name, de_namelen); if (!dir_emit(ctx, de_name, de_namelen, - le64_to_cpu(de->nid), d_type)) - /* stoped by some reason */ + le64_to_cpu(de->nid), d_type)) + /* stopped by some reason */ return 1; ++de; *ofs += sizeof(struct erofs_dirent); @@ -89,62 +83,63 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) struct inode *dir = file_inode(f); struct address_space *mapping = dir->i_mapping; const size_t dirsize = i_size_read(dir); - unsigned i = ctx->pos / EROFS_BLKSIZ; - unsigned ofs = ctx->pos % EROFS_BLKSIZ; + unsigned int i = ctx->pos / EROFS_BLKSIZ; + unsigned int ofs = ctx->pos % EROFS_BLKSIZ; int err = 0; bool initial = true; while (ctx->pos < dirsize) { struct page *dentry_page; struct erofs_dirent *de; - unsigned nameoff, maxsize; + unsigned int nameoff, maxsize; dentry_page = read_mapping_page(mapping, i, NULL); if (dentry_page == ERR_PTR(-ENOMEM)) { err = -ENOMEM; break; } else if (IS_ERR(dentry_page)) { - errln("fail to readdir of logical block %u of nid %llu", - i, EROFS_V(dir)->nid); - err = PTR_ERR(dentry_page); + erofs_err(dir->i_sb, + "fail to readdir of logical block %u of nid %llu", + i, EROFS_I(dir)->nid); + err = -EFSCORRUPTED; break; } - lock_page(dentry_page); de = (struct erofs_dirent *)kmap(dentry_page); nameoff = le16_to_cpu(de->nameoff); - if (unlikely(nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE)) { - errln("%s, invalid de[0].nameoff %u", - __func__, nameoff); - - err = -EIO; + if (nameoff < sizeof(struct erofs_dirent) || + nameoff >= PAGE_SIZE) { + erofs_err(dir->i_sb, + "invalid de[0].nameoff %u @ nid %llu", + nameoff, EROFS_I(dir)->nid); + err = -EFSCORRUPTED; goto skip_this; } - maxsize = min_t(unsigned, dirsize - ctx->pos + ofs, PAGE_SIZE); + maxsize = min_t(unsigned int, + dirsize - ctx->pos + ofs, PAGE_SIZE); /* search dirents at the arbitrary position */ - if (unlikely(initial)) { + if (initial) { initial = false; ofs = roundup(ofs, sizeof(struct erofs_dirent)); - if (unlikely(ofs >= nameoff)) + if (ofs >= nameoff) goto skip_this; } - err = erofs_fill_dentries(ctx, de, &ofs, nameoff, maxsize); + err = erofs_fill_dentries(dir, ctx, de, &ofs, + nameoff, maxsize); skip_this: kunmap(dentry_page); - unlock_page(dentry_page); put_page(dentry_page); ctx->pos = blknr_to_addr(i) + ofs; - if (unlikely(err)) + if (err) break; ++i; ofs = 0; @@ -155,6 +150,5 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) const struct file_operations erofs_dir_fops = { .llseek = generic_file_llseek, .read = generic_read_dir, - .iterate = erofs_readdir, + .iterate_shared = erofs_readdir, }; - diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h new file mode 100644 index 000000000000..32f79b14ef50 --- /dev/null +++ b/fs/erofs/erofs_fs.h @@ -0,0 +1,357 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR Apache-2.0 */ +/* + * EROFS (Enhanced ROM File System) on-disk format definition + * + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_FS_H +#define __EROFS_FS_H + +#define EROFS_SUPER_MAGIC_V1 0xE0F5E1E2 +#define EROFS_SUPER_OFFSET 1024 + +#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 + +/* + * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should + * be incompatible with this kernel version. + */ +#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001 +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 +#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 +#define EROFS_ALL_FEATURE_INCOMPAT \ + (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ + EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ + EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER) + +#define EROFS_SB_EXTSLOT_SIZE 16 + +/* erofs on-disk super block (currently 128 bytes) */ +struct erofs_super_block { + __le32 magic; /* file system magic number */ + __le32 checksum; /* crc32c(super_block) */ + __le32 feature_compat; + __u8 blkszbits; /* support block_size == PAGE_SIZE only */ + __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */ + + __le16 root_nid; /* nid of root directory */ + __le64 inos; /* total valid ino # (== f_files - f_favail) */ + + __le64 build_time; /* inode v1 time derivation */ + __le32 build_time_nsec; /* inode v1 time derivation in nano scale */ + __le32 blocks; /* used for statfs */ + __le32 meta_blkaddr; /* start block address of metadata area */ + __le32 xattr_blkaddr; /* start block address of shared xattr area */ + __u8 uuid[16]; /* 128-bit uuid for volume */ + __u8 volume_name[16]; /* volume name */ + __le32 feature_incompat; + union { + /* bitmap for available compression algorithms */ + __le16 available_compr_algs; + /* customized sliding window size instead of 64k by default */ + __le16 lz4_max_distance; + } __packed u1; + __u8 reserved2[42]; +}; + +/* + * erofs inode datalayout (i_format in on-disk inode): + * 0 - inode plain without inline data A: + * inode, [xattrs], ... | ... | no-holed data + * 1 - inode VLE compression B (legacy): + * inode, [xattrs], extents ... | ... + * 2 - inode plain with inline data C: + * inode, [xattrs], last_inline_data, ... | ... | no-holed data + * 3 - inode compression D: + * inode, [xattrs], map_header, extents ... | ... + * 4~7 - reserved + */ +enum { + EROFS_INODE_FLAT_PLAIN = 0, + EROFS_INODE_FLAT_COMPRESSION_LEGACY = 1, + EROFS_INODE_FLAT_INLINE = 2, + EROFS_INODE_FLAT_COMPRESSION = 3, + EROFS_INODE_DATALAYOUT_MAX +}; + +static inline bool erofs_inode_is_data_compressed(unsigned int datamode) +{ + return datamode == EROFS_INODE_FLAT_COMPRESSION || + datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY; +} + +/* bit definitions of inode i_advise */ +#define EROFS_I_VERSION_BITS 1 +#define EROFS_I_DATALAYOUT_BITS 3 + +#define EROFS_I_VERSION_BIT 0 +#define EROFS_I_DATALAYOUT_BIT 1 + +#define EROFS_I_ALL \ + ((1 << (EROFS_I_DATALAYOUT_BIT + EROFS_I_DATALAYOUT_BITS)) - 1) + +/* 32-byte reduced form of an ondisk inode */ +struct erofs_inode_compact { + __le16 i_format; /* inode format hints */ + +/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ + __le16 i_xattr_icount; + __le16 i_mode; + __le16 i_nlink; + __le32 i_size; + __le32 i_reserved; + union { + /* file total compressed blocks for data mapping 1 */ + __le32 compressed_blocks; + __le32 raw_blkaddr; + + /* for device files, used to indicate old/new device # */ + __le32 rdev; + } i_u; + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le16 i_uid; + __le16 i_gid; + __le32 i_reserved2; +}; + +/* 32 bytes on-disk inode */ +#define EROFS_INODE_LAYOUT_COMPACT 0 +/* 64 bytes on-disk inode */ +#define EROFS_INODE_LAYOUT_EXTENDED 1 + +/* 64-byte complete form of an ondisk inode */ +struct erofs_inode_extended { + __le16 i_format; /* inode format hints */ + +/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ + __le16 i_xattr_icount; + __le16 i_mode; + __le16 i_reserved; + __le64 i_size; + union { + /* file total compressed blocks for data mapping 1 */ + __le32 compressed_blocks; + __le32 raw_blkaddr; + + /* for device files, used to indicate old/new device # */ + __le32 rdev; + } i_u; + + /* only used for 32-bit stat compatibility */ + __le32 i_ino; + + __le32 i_uid; + __le32 i_gid; + __le64 i_ctime; + __le32 i_ctime_nsec; + __le32 i_nlink; + __u8 i_reserved2[16]; +}; + +#define EROFS_MAX_SHARED_XATTRS (128) +/* h_shared_count between 129 ... 255 are special # */ +#define EROFS_SHARED_XATTR_EXTENT (255) + +/* + * inline xattrs (n == i_xattr_icount): + * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes + * 12 bytes / \ + * / \ + * /-----------------------\ + * | erofs_xattr_entries+ | + * +-----------------------+ + * inline xattrs must starts in erofs_xattr_ibody_header, + * for read-only fs, no need to introduce h_refcount + */ +struct erofs_xattr_ibody_header { + __le32 h_reserved; + __u8 h_shared_count; + __u8 h_reserved2[7]; + __le32 h_shared_xattrs[0]; /* shared xattr id array */ +}; + +/* Name indexes */ +#define EROFS_XATTR_INDEX_USER 1 +#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EROFS_XATTR_INDEX_TRUSTED 4 +#define EROFS_XATTR_INDEX_LUSTRE 5 +#define EROFS_XATTR_INDEX_SECURITY 6 + +/* xattr entry (for both inline & shared xattrs) */ +struct erofs_xattr_entry { + __u8 e_name_len; /* length of name */ + __u8 e_name_index; /* attribute name index */ + __le16 e_value_size; /* size of attribute value */ + /* followed by e_name and e_value */ + char e_name[0]; /* attribute name */ +}; + +static inline unsigned int erofs_xattr_ibody_size(__le16 i_xattr_icount) +{ + if (!i_xattr_icount) + return 0; + + return sizeof(struct erofs_xattr_ibody_header) + + sizeof(__u32) * (le16_to_cpu(i_xattr_icount) - 1); +} + +#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry)) + +static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e) +{ + return EROFS_XATTR_ALIGN(sizeof(struct erofs_xattr_entry) + + e->e_name_len + le16_to_cpu(e->e_value_size)); +} + +/* maximum supported size of a physical compression cluster */ +#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) + +/* available compression algorithm types (for h_algorithmtype) */ +enum { + Z_EROFS_COMPRESSION_LZ4 = 0, + Z_EROFS_COMPRESSION_MAX +}; +#define Z_EROFS_ALL_COMPR_ALGS (1 << (Z_EROFS_COMPRESSION_MAX - 1)) + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lz4_cfgs { + __le16 max_distance; + __le16 max_pclusterblks; + u8 reserved[10]; +} __packed; + +/* + * bit 0 : COMPACTED_2B indexes (0 - off; 1 - on) + * e.g. for 4k logical cluster size, 4B if compacted 2B is off; + * (4B) + 2B + (4B) if compacted 2B is on. + * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) + * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) + */ +#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 + +struct z_erofs_map_header { + __le32 h_reserved1; + __le16 h_advise; + /* + * bit 0-3 : algorithm type of head 1 (logical cluster type 01); + * bit 4-7 : algorithm type of head 2 (logical cluster type 11). + */ + __u8 h_algorithmtype; + /* + * bit 0-2 : logical cluster bits - 12, e.g. 0 for 4096; + * bit 3-7 : reserved. + */ + __u8 h_clusterbits; +}; + +#define Z_EROFS_VLE_LEGACY_HEADER_PADDING 8 + +/* + * Fixed-sized output compression ondisk Logical Extent cluster type: + * 0 - literal (uncompressed) cluster + * 1 - compressed cluster (for the head logical cluster) + * 2 - compressed cluster (for the other logical clusters) + * + * In detail, + * 0 - literal (uncompressed) cluster, + * di_advise = 0 + * di_clusterofs = the literal data offset of the cluster + * di_blkaddr = the blkaddr of the literal cluster + * + * 1 - compressed cluster (for the head logical cluster) + * di_advise = 1 + * di_clusterofs = the decompressed data offset of the cluster + * di_blkaddr = the blkaddr of the compressed cluster + * + * 2 - compressed cluster (for the other logical clusters) + * di_advise = 2 + * di_clusterofs = + * the decompressed data offset in its own head cluster + * di_u.delta[0] = distance to its corresponding head cluster + * di_u.delta[1] = distance to its corresponding tail cluster + * (di_advise could be 0, 1 or 2) + */ +enum { + Z_EROFS_VLE_CLUSTER_TYPE_PLAIN = 0, + Z_EROFS_VLE_CLUSTER_TYPE_HEAD = 1, + Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD = 2, + Z_EROFS_VLE_CLUSTER_TYPE_RESERVED = 3, + Z_EROFS_VLE_CLUSTER_TYPE_MAX +}; + +#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS 2 +#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT 0 + +/* + * D0_CBLKCNT will be marked _only_ at the 1st non-head lcluster to store the + * compressed block count of a compressed extent (in logical clusters, aka. + * block count of a pcluster). + */ +#define Z_EROFS_VLE_DI_D0_CBLKCNT (1 << 11) + +struct z_erofs_vle_decompressed_index { + __le16 di_advise; + /* where to decompress in the head cluster */ + __le16 di_clusterofs; + + union { + /* for the head cluster */ + __le32 blkaddr; + /* + * for the rest clusters + * eg. for 4k page-sized cluster, maximum 4K*64k = 256M) + * [0] - pointing to the head cluster + * [1] - pointing to the tail cluster + */ + __le16 delta[2]; + } di_u; +}; + +#define Z_EROFS_VLE_LEGACY_INDEX_ALIGN(size) \ + (round_up(size, sizeof(struct z_erofs_vle_decompressed_index)) + \ + sizeof(struct z_erofs_map_header) + Z_EROFS_VLE_LEGACY_HEADER_PADDING) + +/* dirent sorts in alphabet order, thus we can do binary search */ +struct erofs_dirent { + __le64 nid; /* node number */ + __le16 nameoff; /* start offset of file name */ + __u8 file_type; /* file type */ + __u8 reserved; /* reserved */ +} __packed; + +/* file types used in inode_info->flags */ +enum { + EROFS_FT_UNKNOWN, + EROFS_FT_REG_FILE, + EROFS_FT_DIR, + EROFS_FT_CHRDEV, + EROFS_FT_BLKDEV, + EROFS_FT_FIFO, + EROFS_FT_SOCK, + EROFS_FT_SYMLINK, + EROFS_FT_MAX +}; + +#define EROFS_NAME_LEN 255 + +/* check the EROFS on-disk layout strictly at compile time */ +static inline void erofs_check_ondisk_layout_definitions(void) +{ + BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128); + BUILD_BUG_ON(sizeof(struct erofs_inode_compact) != 32); + BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64); + BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12); + BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4); + BUILD_BUG_ON(sizeof(struct z_erofs_map_header) != 8); + BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8); + BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12); + + BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) < + Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); +} + +#endif diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c new file mode 100644 index 000000000000..5f9c6fb79ac0 --- /dev/null +++ b/fs/erofs/inode.c @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#include "xattr.h" + +#include + +/* + * if inode is successfully read, return its inode page (or sometimes + * the inode payload page if it's an extended inode) in order to fill + * inline data if possible. + */ +static struct page *erofs_read_inode(struct inode *inode, + unsigned int *ofs) +{ + struct super_block *sb = inode->i_sb; + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_inode *vi = EROFS_I(inode); + const erofs_off_t inode_loc = iloc(sbi, vi->nid); + + erofs_blk_t blkaddr, nblks = 0; + struct page *page; + struct erofs_inode_compact *dic; + struct erofs_inode_extended *die, *copied = NULL; + unsigned int ifmt; + int err; + + blkaddr = erofs_blknr(inode_loc); + *ofs = erofs_blkoff(inode_loc); + + erofs_dbg("%s, reading inode nid %llu at %u of blkaddr %u", + __func__, vi->nid, *ofs, blkaddr); + + page = erofs_get_meta_page(sb, blkaddr); + if (IS_ERR(page)) { + erofs_err(sb, "failed to get inode (nid: %llu) page, err %ld", + vi->nid, PTR_ERR(page)); + return page; + } + + dic = page_address(page) + *ofs; + ifmt = le16_to_cpu(dic->i_format); + + if (ifmt & ~EROFS_I_ALL) { + erofs_err(inode->i_sb, "unsupported i_format %u of nid %llu", + ifmt, vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + + vi->datalayout = erofs_inode_datalayout(ifmt); + if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) { + erofs_err(inode->i_sb, "unsupported datalayout %u of nid %llu", + vi->datalayout, vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + + switch (erofs_inode_version(ifmt)) { + case EROFS_INODE_LAYOUT_EXTENDED: + vi->inode_isize = sizeof(struct erofs_inode_extended); + /* check if the inode acrosses page boundary */ + if (*ofs + vi->inode_isize <= PAGE_SIZE) { + *ofs += vi->inode_isize; + die = (struct erofs_inode_extended *)dic; + } else { + const unsigned int gotten = PAGE_SIZE - *ofs; + + copied = kmalloc(vi->inode_isize, GFP_NOFS); + if (!copied) { + err = -ENOMEM; + goto err_out; + } + memcpy(copied, dic, gotten); + unlock_page(page); + put_page(page); + + page = erofs_get_meta_page(sb, blkaddr + 1); + if (IS_ERR(page)) { + erofs_err(sb, "failed to get inode payload page (nid: %llu), err %ld", + vi->nid, PTR_ERR(page)); + kfree(copied); + return page; + } + *ofs = vi->inode_isize - gotten; + memcpy((u8 *)copied + gotten, page_address(page), *ofs); + die = copied; + } + vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount); + + inode->i_mode = le16_to_cpu(die->i_mode); + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + case S_IFDIR: + case S_IFLNK: + vi->raw_blkaddr = le32_to_cpu(die->i_u.raw_blkaddr); + break; + case S_IFCHR: + case S_IFBLK: + inode->i_rdev = + new_decode_dev(le32_to_cpu(die->i_u.rdev)); + break; + case S_IFIFO: + case S_IFSOCK: + inode->i_rdev = 0; + break; + default: + goto bogusimode; + } + i_uid_write(inode, le32_to_cpu(die->i_uid)); + i_gid_write(inode, le32_to_cpu(die->i_gid)); + set_nlink(inode, le32_to_cpu(die->i_nlink)); + + /* extended inode has its own timestamp */ + inode->i_ctime.tv_sec = le64_to_cpu(die->i_ctime); + inode->i_ctime.tv_nsec = le32_to_cpu(die->i_ctime_nsec); + + inode->i_size = le64_to_cpu(die->i_size); + + /* total blocks for compressed files */ + if (erofs_inode_is_data_compressed(vi->datalayout)) + nblks = le32_to_cpu(die->i_u.compressed_blocks); + + kfree(copied); + break; + case EROFS_INODE_LAYOUT_COMPACT: + vi->inode_isize = sizeof(struct erofs_inode_compact); + *ofs += vi->inode_isize; + vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount); + + inode->i_mode = le16_to_cpu(dic->i_mode); + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + case S_IFDIR: + case S_IFLNK: + vi->raw_blkaddr = le32_to_cpu(dic->i_u.raw_blkaddr); + break; + case S_IFCHR: + case S_IFBLK: + inode->i_rdev = + new_decode_dev(le32_to_cpu(dic->i_u.rdev)); + break; + case S_IFIFO: + case S_IFSOCK: + inode->i_rdev = 0; + break; + default: + goto bogusimode; + } + i_uid_write(inode, le16_to_cpu(dic->i_uid)); + i_gid_write(inode, le16_to_cpu(dic->i_gid)); + set_nlink(inode, le16_to_cpu(dic->i_nlink)); + + /* use build time for compact inodes */ + inode->i_ctime.tv_sec = sbi->build_time; + inode->i_ctime.tv_nsec = sbi->build_time_nsec; + + inode->i_size = le32_to_cpu(dic->i_size); + if (erofs_inode_is_data_compressed(vi->datalayout)) + nblks = le32_to_cpu(dic->i_u.compressed_blocks); + break; + default: + erofs_err(inode->i_sb, + "unsupported on-disk inode version %u of nid %llu", + erofs_inode_version(ifmt), vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + + inode->i_mtime.tv_sec = inode->i_ctime.tv_sec; + inode->i_atime.tv_sec = inode->i_ctime.tv_sec; + inode->i_mtime.tv_nsec = inode->i_ctime.tv_nsec; + inode->i_atime.tv_nsec = inode->i_ctime.tv_nsec; + + if (!nblks) + /* measure inode.i_blocks as generic filesystems */ + inode->i_blocks = roundup(inode->i_size, EROFS_BLKSIZ) >> 9; + else + inode->i_blocks = nblks << LOG_SECTORS_PER_BLOCK; + return page; + +bogusimode: + erofs_err(inode->i_sb, "bogus i_mode (%o) @ nid %llu", + inode->i_mode, vi->nid); + err = -EFSCORRUPTED; +err_out: + DBG_BUGON(1); + kfree(copied); + unlock_page(page); + put_page(page); + return ERR_PTR(err); +} + +static int erofs_fill_symlink(struct inode *inode, void *data, + unsigned int m_pofs) +{ + struct erofs_inode *vi = EROFS_I(inode); + char *lnk; + + /* if it cannot be handled with fast symlink scheme */ + if (vi->datalayout != EROFS_INODE_FLAT_INLINE || + inode->i_size >= PAGE_SIZE) { + inode->i_op = &erofs_symlink_iops; + return 0; + } + + lnk = kmalloc(inode->i_size + 1, GFP_KERNEL); + if (!lnk) + return -ENOMEM; + + m_pofs += vi->xattr_isize; + /* inline symlink data shouldn't cross page boundary as well */ + if (m_pofs + inode->i_size > PAGE_SIZE) { + kfree(lnk); + erofs_err(inode->i_sb, + "inline data cross block boundary @ nid %llu", + vi->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; + } + + memcpy(lnk, data + m_pofs, inode->i_size); + lnk[inode->i_size] = '\0'; + + inode->i_link = lnk; + inode->i_op = &erofs_fast_symlink_iops; + return 0; +} + +static int erofs_fill_inode(struct inode *inode, int isdir) +{ + struct erofs_inode *vi = EROFS_I(inode); + struct page *page; + unsigned int ofs; + int err = 0; + + trace_erofs_fill_inode(inode, isdir); + + /* read inode base data from disk */ + page = erofs_read_inode(inode, &ofs); + if (IS_ERR(page)) + return PTR_ERR(page); + + /* setup the new inode */ + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + inode->i_op = &erofs_generic_iops; + inode->i_fop = &generic_ro_fops; + break; + case S_IFDIR: + inode->i_op = &erofs_dir_iops; + inode->i_fop = &erofs_dir_fops; + break; + case S_IFLNK: + err = erofs_fill_symlink(inode, page_address(page), ofs); + if (err) + goto out_unlock; + inode_nohighmem(inode); + break; + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + inode->i_op = &erofs_generic_iops; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + goto out_unlock; + default: + err = -EFSCORRUPTED; + goto out_unlock; + } + + if (erofs_inode_is_data_compressed(vi->datalayout)) { + err = z_erofs_fill_inode(inode); + goto out_unlock; + } + inode->i_mapping->a_ops = &erofs_raw_access_aops; + +out_unlock: + unlock_page(page); + put_page(page); + return err; +} + +/* + * erofs nid is 64bits, but i_ino is 'unsigned long', therefore + * we should do more for 32-bit platform to find the right inode. + */ +static int erofs_ilookup_test_actor(struct inode *inode, void *opaque) +{ + const erofs_nid_t nid = *(erofs_nid_t *)opaque; + + return EROFS_I(inode)->nid == nid; +} + +static int erofs_iget_set_actor(struct inode *inode, void *opaque) +{ + const erofs_nid_t nid = *(erofs_nid_t *)opaque; + + inode->i_ino = erofs_inode_hash(nid); + return 0; +} + +static inline struct inode *erofs_iget_locked(struct super_block *sb, + erofs_nid_t nid) +{ + const unsigned long hashval = erofs_inode_hash(nid); + + return iget5_locked(sb, hashval, erofs_ilookup_test_actor, + erofs_iget_set_actor, &nid); +} + +struct inode *erofs_iget(struct super_block *sb, + erofs_nid_t nid, + bool isdir) +{ + struct inode *inode = erofs_iget_locked(sb, nid); + + if (!inode) + return ERR_PTR(-ENOMEM); + + if (inode->i_state & I_NEW) { + int err; + struct erofs_inode *vi = EROFS_I(inode); + + vi->nid = nid; + + err = erofs_fill_inode(inode, isdir); + if (!err) + unlock_new_inode(inode); + else { + iget_failed(inode); + inode = ERR_PTR(err); + } + } + return inode; +} + +int erofs_getattr(const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *const inode = d_inode(path->dentry); + + if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) + stat->attributes |= STATX_ATTR_COMPRESSED; + + stat->attributes |= STATX_ATTR_IMMUTABLE; + stat->attributes_mask |= (STATX_ATTR_COMPRESSED | + STATX_ATTR_IMMUTABLE); + + generic_fillattr(inode, stat); + return 0; +} + +const struct inode_operations erofs_generic_iops = { + .getattr = erofs_getattr, + .listxattr = erofs_listxattr, + .get_acl = erofs_get_acl, +}; + +const struct inode_operations erofs_symlink_iops = { + .get_link = page_get_link, + .getattr = erofs_getattr, + .listxattr = erofs_listxattr, + .get_acl = erofs_get_acl, +}; + +const struct inode_operations erofs_fast_symlink_iops = { + .get_link = simple_get_link, + .getattr = erofs_getattr, + .listxattr = erofs_listxattr, + .get_acl = erofs_get_acl, +}; diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h new file mode 100644 index 000000000000..c8f904bbe985 --- /dev/null +++ b/fs/erofs/internal.h @@ -0,0 +1,469 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_INTERNAL_H +#define __EROFS_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "erofs_fs.h" + +/* redefine pr_fmt "erofs: " */ +#undef pr_fmt +#define pr_fmt(fmt) "erofs: " fmt + +__printf(3, 4) void _erofs_err(struct super_block *sb, + const char *function, const char *fmt, ...); +#define erofs_err(sb, fmt, ...) \ + _erofs_err(sb, __func__, fmt "\n", ##__VA_ARGS__) +__printf(3, 4) void _erofs_info(struct super_block *sb, + const char *function, const char *fmt, ...); +#define erofs_info(sb, fmt, ...) \ + _erofs_info(sb, __func__, fmt "\n", ##__VA_ARGS__) +#ifdef CONFIG_EROFS_FS_DEBUG +#define erofs_dbg(x, ...) pr_debug(x "\n", ##__VA_ARGS__) +#define DBG_BUGON BUG_ON +#else +#define erofs_dbg(x, ...) ((void)0) +#define DBG_BUGON(x) ((void)(x)) +#endif /* !CONFIG_EROFS_FS_DEBUG */ + +/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */ +#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1 + +typedef u64 erofs_nid_t; +typedef u64 erofs_off_t; +/* data type for filesystem-wide blocks number */ +typedef u32 erofs_blk_t; + +/* all filesystem-wide lz4 configurations */ +struct erofs_sb_lz4_info { + /* # of pages needed for EROFS lz4 rolling decompression */ + u16 max_distance_pages; + /* maximum possible blocks for pclusters in the filesystem */ + u16 max_pclusterblks; +}; + +struct erofs_sb_info { +#ifdef CONFIG_EROFS_FS_ZIP + /* list for all registered superblocks, mainly for shrinker */ + struct list_head list; + struct mutex umount_mutex; + + /* the dedicated workstation for compression */ + struct radix_tree_root workstn_tree; + + /* strategy of sync decompression (false - auto, true - force on) */ + bool readahead_sync_decompress; + + /* threshold for decompression synchronously */ + unsigned int max_sync_decompress_pages; + + unsigned int shrinker_run_no; + u16 available_compr_algs; + + /* current strategy of how to use managed cache */ + unsigned char cache_strategy; + + /* pseudo inode to manage cached pages */ + struct inode *managed_cache; + + struct erofs_sb_lz4_info lz4; +#endif /* CONFIG_EROFS_FS_ZIP */ + u32 blocks; + u32 meta_blkaddr; +#ifdef CONFIG_EROFS_FS_XATTR + u32 xattr_blkaddr; +#endif + + /* inode slot unit size in bit shift */ + unsigned char islotbits; + + u32 sb_size; /* total superblock size */ + u32 build_time_nsec; + u64 build_time; + + /* what we really care is nid, rather than ino.. */ + erofs_nid_t root_nid; + /* used for statfs, f_files - f_favail */ + u64 inos; + + u8 uuid[16]; /* 128-bit uuid for volume */ + u8 volume_name[16]; /* volume name */ + u32 feature_compat; + u32 feature_incompat; + + unsigned int mount_opt; +}; + +#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) +#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info) + +/* Mount flags set via mount options or defaults */ +#define EROFS_MOUNT_XATTR_USER 0x00000010 +#define EROFS_MOUNT_POSIX_ACL 0x00000020 + +#define clear_opt(sbi, option) ((sbi)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(sbi, option) ((sbi)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(sbi, option) ((sbi)->mount_opt & EROFS_MOUNT_##option) + +#ifdef CONFIG_EROFS_FS_ZIP +enum { + EROFS_ZIP_CACHE_DISABLED, + EROFS_ZIP_CACHE_READAHEAD, + EROFS_ZIP_CACHE_READAROUND +}; + +#define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL) + +/* basic unit of the workstation of a super_block */ +struct erofs_workgroup { + /* the workgroup index in the workstation */ + pgoff_t index; + + /* overall workgroup reference count */ + atomic_t refcount; +}; + +#if defined(CONFIG_SMP) +static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp, + int val) +{ + preempt_disable(); + if (val != atomic_cmpxchg(&grp->refcount, val, EROFS_LOCKED_MAGIC)) { + preempt_enable(); + return false; + } + return true; +} + +static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp, + int orig_val) +{ + /* + * other observers should notice all modifications + * in the freezing period. + */ + smp_mb(); + atomic_set(&grp->refcount, orig_val); + preempt_enable(); +} + +static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp) +{ + return atomic_cond_read_relaxed(&grp->refcount, + VAL != EROFS_LOCKED_MAGIC); +} +#else +static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp, + int val) +{ + preempt_disable(); + /* no need to spin on UP platforms, let's just disable preemption. */ + if (val != atomic_read(&grp->refcount)) { + preempt_enable(); + return false; + } + return true; +} + +static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp, + int orig_val) +{ + preempt_enable(); +} + +static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp) +{ + int v = atomic_read(&grp->refcount); + + /* workgroup is never freezed on uniprocessor systems */ + DBG_BUGON(v == EROFS_LOCKED_MAGIC); + return v; +} +#endif /* !CONFIG_SMP */ +#endif /* !CONFIG_EROFS_FS_ZIP */ + +/* we strictly follow PAGE_SIZE and no buffer head yet */ +#define LOG_BLOCK_SIZE PAGE_SHIFT + +#undef LOG_SECTORS_PER_BLOCK +#define LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9) + +#undef SECTORS_PER_BLOCK +#define SECTORS_PER_BLOCK (1 << SECTORS_PER_BLOCK) + +#define EROFS_BLKSIZ (1 << LOG_BLOCK_SIZE) + +#if (EROFS_BLKSIZ % 4096 || !EROFS_BLKSIZ) +#error erofs cannot be used in this platform +#endif + +#define ROOT_NID(sb) ((sb)->root_nid) + +#define erofs_blknr(addr) ((addr) / EROFS_BLKSIZ) +#define erofs_blkoff(addr) ((addr) % EROFS_BLKSIZ) +#define blknr_to_addr(nr) ((erofs_off_t)(nr) * EROFS_BLKSIZ) + +static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits); +} + +#define EROFS_FEATURE_FUNCS(name, compat, feature) \ +static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \ +{ \ + return sbi->feature_##compat & EROFS_FEATURE_##feature; \ +} + +EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) +EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) +EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) + +/* atomic flag definitions */ +#define EROFS_I_EA_INITED_BIT 0 +#define EROFS_I_Z_INITED_BIT 1 + +/* bitlock definitions (arranged in reverse order) */ +#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1) +#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2) + +struct erofs_inode { + erofs_nid_t nid; + + /* atomic flags (including bitlocks) */ + unsigned long flags; + + unsigned char datalayout; + unsigned char inode_isize; + unsigned short xattr_isize; + + unsigned int xattr_shared_count; + unsigned int *xattr_shared_xattrs; + + union { + erofs_blk_t raw_blkaddr; +#ifdef CONFIG_EROFS_FS_ZIP + struct { + unsigned short z_advise; + unsigned char z_algorithmtype[2]; + unsigned char z_logical_clusterbits; + }; +#endif /* CONFIG_EROFS_FS_ZIP */ + }; + /* the corresponding vfs inode */ + struct inode vfs_inode; +}; + +#define EROFS_I(ptr) \ + container_of(ptr, struct erofs_inode, vfs_inode) + +static inline unsigned long erofs_inode_datablocks(struct inode *inode) +{ + /* since i_size cannot be changed */ + return DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); +} + +static inline unsigned int erofs_bitrange(unsigned int value, unsigned int bit, + unsigned int bits) +{ + + return (value >> bit) & ((1 << bits) - 1); +} + + +static inline unsigned int erofs_inode_version(unsigned int value) +{ + return erofs_bitrange(value, EROFS_I_VERSION_BIT, + EROFS_I_VERSION_BITS); +} + +static inline unsigned int erofs_inode_datalayout(unsigned int value) +{ + return erofs_bitrange(value, EROFS_I_DATALAYOUT_BIT, + EROFS_I_DATALAYOUT_BITS); +} + +extern const struct super_operations erofs_sops; + +extern const struct address_space_operations erofs_raw_access_aops; +extern const struct address_space_operations z_erofs_aops; + +/* + * Logical to physical block mapping + * + * Different with other file systems, it is used for 2 access modes: + * + * 1) RAW access mode: + * + * Users pass a valid (m_lblk, m_lofs -- usually 0) pair, + * and get the valid m_pblk, m_pofs and the longest m_len(in bytes). + * + * Note that m_lblk in the RAW access mode refers to the number of + * the compressed ondisk block rather than the uncompressed + * in-memory block for the compressed file. + * + * m_pofs equals to m_lofs except for the inline data page. + * + * 2) Normal access mode: + * + * If the inode is not compressed, it has no difference with + * the RAW access mode. However, if the inode is compressed, + * users should pass a valid (m_lblk, m_lofs) pair, and get + * the needed m_pblk, m_pofs, m_len to get the compressed data + * and the updated m_lblk, m_lofs which indicates the start + * of the corresponding uncompressed data in the file. + */ +enum { + BH_Zipped = BH_PrivateStart, + BH_FullMapped, +}; + +/* Has a disk mapping */ +#define EROFS_MAP_MAPPED (1 << BH_Mapped) +/* Located in metadata (could be copied from bd_inode) */ +#define EROFS_MAP_META (1 << BH_Meta) +/* The extent has been compressed */ +#define EROFS_MAP_ZIPPED (1 << BH_Zipped) +/* The length of extent is full */ +#define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped) + +struct erofs_map_blocks { + erofs_off_t m_pa, m_la; + u64 m_plen, m_llen; + + unsigned int m_flags; + + struct page *mpage; +}; + +/* Flags used by erofs_map_blocks_flatmode() */ +#define EROFS_GET_BLOCKS_RAW 0x0001 + +/* zmap.c */ +#ifdef CONFIG_EROFS_FS_ZIP +int z_erofs_fill_inode(struct inode *inode); +int z_erofs_map_blocks_iter(struct inode *inode, + struct erofs_map_blocks *map, + int flags); +#else +static inline int z_erofs_fill_inode(struct inode *inode) { return -EOPNOTSUPP; } +static inline int z_erofs_map_blocks_iter(struct inode *inode, + struct erofs_map_blocks *map, + int flags) +{ + return -EOPNOTSUPP; +} +#endif /* !CONFIG_EROFS_FS_ZIP */ + +/* data.c */ +struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr); + +/* inode.c */ +static inline unsigned long erofs_inode_hash(erofs_nid_t nid) +{ +#if BITS_PER_LONG == 32 + return (nid >> 32) ^ (nid & 0xffffffff); +#else + return nid; +#endif +} + +extern const struct inode_operations erofs_generic_iops; +extern const struct inode_operations erofs_symlink_iops; +extern const struct inode_operations erofs_fast_symlink_iops; + +struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir); +int erofs_getattr(const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags); + +/* namei.c */ +extern const struct inode_operations erofs_dir_iops; + +int erofs_namei(struct inode *dir, struct qstr *name, + erofs_nid_t *nid, unsigned int *d_type); + +/* dir.c */ +extern const struct file_operations erofs_dir_fops; + +static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count) +{ + int retried = 0; + + while (1) { + void *p = vm_map_ram(pages, count, -1, PAGE_KERNEL); + + /* retry two more times (totally 3 times) */ + if (p || ++retried >= 3) + return p; + vm_unmap_aliases(); + } + return NULL; +} + +/* pcpubuf.c */ +void *erofs_get_pcpubuf(unsigned int requiredpages); +void erofs_put_pcpubuf(void *ptr); +int erofs_pcpubuf_growsize(unsigned int nrpages); +void erofs_pcpubuf_init(void); +void erofs_pcpubuf_exit(void); + +/* utils.c / zdata.c */ +struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp); + +#ifdef CONFIG_EROFS_FS_ZIP +int erofs_workgroup_put(struct erofs_workgroup *grp); +struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb, + pgoff_t index); +int erofs_register_workgroup(struct super_block *sb, + struct erofs_workgroup *grp); +void erofs_workgroup_free_rcu(struct erofs_workgroup *grp); +void erofs_shrinker_register(struct super_block *sb); +void erofs_shrinker_unregister(struct super_block *sb); +int __init erofs_init_shrinker(void); +void erofs_exit_shrinker(void); +int __init z_erofs_init_zip_subsystem(void); +void z_erofs_exit_zip_subsystem(void); +int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, + struct erofs_workgroup *egrp); +int erofs_try_to_free_cached_page(struct address_space *mapping, + struct page *page); +int z_erofs_load_lz4_config(struct super_block *sb, + struct erofs_super_block *dsb, + struct z_erofs_lz4_cfgs *lz4, int len); +#else +static inline void erofs_shrinker_register(struct super_block *sb) {} +static inline void erofs_shrinker_unregister(struct super_block *sb) {} +static inline int erofs_init_shrinker(void) { return 0; } +static inline void erofs_exit_shrinker(void) {} +static inline int z_erofs_init_zip_subsystem(void) { return 0; } +static inline void z_erofs_exit_zip_subsystem(void) {} +static inline int z_erofs_load_lz4_config(struct super_block *sb, + struct erofs_super_block *dsb, + struct z_erofs_lz4_cfgs *lz4, int len) +{ + if (lz4 || dsb->u1.lz4_max_distance) { + erofs_err(sb, "lz4 algorithm isn't enabled"); + return -EINVAL; + } + return 0; +} +#endif /* !CONFIG_EROFS_FS_ZIP */ + +#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ + +#ifndef lru_to_page +#define lru_to_page(head) (list_entry((head)->prev, struct page, lru)) +#endif + +#endif /* __EROFS_INTERNAL_H */ diff --git a/drivers/staging/erofs/namei.c b/fs/erofs/namei.c similarity index 77% rename from drivers/staging/erofs/namei.c rename to fs/erofs/namei.c index 023f64fa2c87..c5c0dd9d9033 100644 --- a/drivers/staging/erofs/namei.c +++ b/fs/erofs/namei.c @@ -1,16 +1,8 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* - * linux/drivers/staging/erofs/namei.c - * * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. + * https://www.huawei.com/ */ -#include "internal.h" #include "xattr.h" #include @@ -21,9 +13,9 @@ struct erofs_qstr { }; /* based on the end of qn is accurate and it must have the trailing '\0' */ -static inline int dirnamecmp(const struct erofs_qstr *qn, - const struct erofs_qstr *qd, - unsigned int *matched) +static inline int erofs_dirnamecmp(const struct erofs_qstr *qn, + const struct erofs_qstr *qd, + unsigned int *matched) { unsigned int i = *matched; @@ -71,16 +63,16 @@ static struct erofs_dirent *find_target_dirent(struct erofs_qstr *name, unsigned int matched = min(startprfx, endprfx); struct erofs_qstr dname = { .name = data + nameoff, - .end = unlikely(mid >= ndirents - 1) ? + .end = mid >= ndirents - 1 ? data + dirblksize : data + nameoff_from_disk(de[mid + 1].nameoff, dirblksize) }; /* string comparison without already matched prefix */ - int ret = dirnamecmp(name, &dname, &matched); + int ret = erofs_dirnamecmp(name, &dname, &matched); - if (unlikely(!ret)) { + if (!ret) { return de + mid; } else if (ret > 0) { head = mid + 1; @@ -105,7 +97,7 @@ static struct page *find_target_block_classic(struct inode *dir, startprfx = endprfx = 0; head = 0; - back = inode_datablocks(dir) - 1; + back = erofs_inode_datablocks(dir) - 1; while (head <= back) { const int mid = head + (back - head) / 2; @@ -120,11 +112,14 @@ static struct page *find_target_block_classic(struct inode *dir, unsigned int matched; struct erofs_qstr dname; - if (unlikely(!ndirents)) { - DBG_BUGON(1); + if (!ndirents) { kunmap_atomic(de); put_page(page); - page = ERR_PTR(-EIO); + erofs_err(dir->i_sb, + "corrupted dir block %d @ nid %llu", + mid, EROFS_I(dir)->nid); + DBG_BUGON(1); + page = ERR_PTR(-EFSCORRUPTED); goto out; } @@ -139,17 +134,17 @@ static struct page *find_target_block_classic(struct inode *dir, EROFS_BLKSIZ); /* string comparison without already matched prefix */ - diff = dirnamecmp(name, &dname, &matched); + diff = erofs_dirnamecmp(name, &dname, &matched); kunmap_atomic(de); - if (unlikely(!diff)) { + if (!diff) { *_ndirents = 0; goto out; } else if (diff > 0) { head = mid + 1; startprfx = matched; - if (likely(!IS_ERR(candidate))) + if (!IS_ERR(candidate)) put_page(candidate); candidate = page; *_ndirents = ndirents; @@ -179,7 +174,7 @@ int erofs_namei(struct inode *dir, struct erofs_dirent *de; struct erofs_qstr qn; - if (unlikely(!dir->i_size)) + if (!dir->i_size) return -ENOENT; qn.name = name->name; @@ -188,7 +183,7 @@ int erofs_namei(struct inode *dir, ndirents = 0; page = find_target_block_classic(dir, &qn, &ndirents); - if (unlikely(IS_ERR(page))) + if (IS_ERR(page)) return PTR_ERR(page); data = kmap_atomic(page); @@ -198,7 +193,7 @@ int erofs_namei(struct inode *dir, else de = (struct erofs_dirent *)data; - if (likely(!IS_ERR(de))) { + if (!IS_ERR(de)) { *nid = le64_to_cpu(de->nid); *d_type = de->file_type; } @@ -211,11 +206,12 @@ int erofs_namei(struct inode *dir, /* NOTE: i_mutex is already held by vfs */ static struct dentry *erofs_lookup(struct inode *dir, - struct dentry *dentry, unsigned int flags) + struct dentry *dentry, + unsigned int flags) { int err; erofs_nid_t nid; - unsigned d_type; + unsigned int d_type; struct inode *inode; DBG_BUGON(!d_really_is_negative(dentry)); @@ -225,7 +221,7 @@ static struct dentry *erofs_lookup(struct inode *dir, trace_erofs_lookup(dir, dentry, flags); /* file name exceeds fs limit */ - if (unlikely(dentry->d_name.len > EROFS_NAME_LEN)) + if (dentry->d_name.len > EROFS_NAME_LEN) return ERR_PTR(-ENAMETOOLONG); /* false uninitialized warnings on gcc 4.8.x */ @@ -234,29 +230,19 @@ static struct dentry *erofs_lookup(struct inode *dir, if (err == -ENOENT) { /* negative dentry */ inode = NULL; - goto negative_out; - } else if (unlikely(err)) - return ERR_PTR(err); - - debugln("%s, %s (nid %llu) found, d_type %u", __func__, - dentry->d_name.name, nid, d_type); - - inode = erofs_iget(dir->i_sb, nid, d_type == EROFS_FT_DIR); - if (IS_ERR(inode)) - return ERR_CAST(inode); - -negative_out: + } else if (err) { + inode = ERR_PTR(err); + } else { + erofs_dbg("%s, %s (nid %llu) found, d_type %u", __func__, + dentry->d_name.name, nid, d_type); + inode = erofs_iget(dir->i_sb, nid, d_type == EROFS_FT_DIR); + } return d_splice_alias(inode, dentry); } const struct inode_operations erofs_dir_iops = { .lookup = erofs_lookup, -}; - -const struct inode_operations erofs_dir_xattr_iops = { - .lookup = erofs_lookup, -#ifdef CONFIG_EROFS_FS_XATTR + .getattr = erofs_getattr, .listxattr = erofs_listxattr, -#endif + .get_acl = erofs_get_acl, }; - diff --git a/fs/erofs/pcpubuf.c b/fs/erofs/pcpubuf.c new file mode 100644 index 000000000000..6c885575128a --- /dev/null +++ b/fs/erofs/pcpubuf.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Gao Xiang + * + * For low-latency decompression algorithms (e.g. lz4), reserve consecutive + * per-CPU virtual memory (in pages) in advance to store such inplace I/O + * data if inplace decompression is failed (due to unmet inplace margin for + * example). + */ +#include "internal.h" + +struct erofs_pcpubuf { + raw_spinlock_t lock; + void *ptr; + struct page **pages; + unsigned int nrpages; +}; + +static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb); + +void *erofs_get_pcpubuf(unsigned int requiredpages) + __acquires(pcb->lock) +{ + struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb); + + raw_spin_lock(&pcb->lock); + /* check if the per-CPU buffer is too small */ + if (requiredpages > pcb->nrpages) { + raw_spin_unlock(&pcb->lock); + put_cpu_var(erofs_pcb); + /* (for sparse checker) pretend pcb->lock is still taken */ + __acquire(pcb->lock); + return NULL; + } + return pcb->ptr; +} + +void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock) +{ + struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id()); + + DBG_BUGON(pcb->ptr != ptr); + raw_spin_unlock(&pcb->lock); + put_cpu_var(erofs_pcb); +} + +/* the next step: support per-CPU page buffers hotplug */ +int erofs_pcpubuf_growsize(unsigned int nrpages) +{ + static DEFINE_MUTEX(pcb_resize_mutex); + static unsigned int pcb_nrpages; + LIST_HEAD(pagepool); + int delta, cpu, ret, i; + + mutex_lock(&pcb_resize_mutex); + delta = nrpages - pcb_nrpages; + ret = 0; + /* avoid shrinking pcpubuf, since no idea how many fses rely on */ + if (delta <= 0) + goto out; + + for_each_possible_cpu(cpu) { + struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); + struct page **pages, **oldpages; + void *ptr, *old_ptr; + + pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL); + if (!pages) { + ret = -ENOMEM; + break; + } + + for (i = 0; i < nrpages; ++i) { + pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL); + if (!pages[i]) { + ret = -ENOMEM; + oldpages = pages; + goto free_pagearray; + } + } + ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL); + if (!ptr) { + ret = -ENOMEM; + oldpages = pages; + goto free_pagearray; + } + raw_spin_lock(&pcb->lock); + old_ptr = pcb->ptr; + pcb->ptr = ptr; + oldpages = pcb->pages; + pcb->pages = pages; + i = pcb->nrpages; + pcb->nrpages = nrpages; + raw_spin_unlock(&pcb->lock); + + if (!oldpages) { + DBG_BUGON(old_ptr); + continue; + } + + if (old_ptr) + vunmap(old_ptr); +free_pagearray: + while (i) + list_add(&oldpages[--i]->lru, &pagepool); + kfree(oldpages); + if (ret) + break; + } + pcb_nrpages = nrpages; + put_pages_list(&pagepool); +out: + mutex_unlock(&pcb_resize_mutex); + return ret; +} + +void erofs_pcpubuf_init(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); + + raw_spin_lock_init(&pcb->lock); + } +} + +void erofs_pcpubuf_exit(void) +{ + int cpu, i; + + for_each_possible_cpu(cpu) { + struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); + + if (pcb->ptr) { + vunmap(pcb->ptr); + pcb->ptr = NULL; + } + if (!pcb->pages) + continue; + + for (i = 0; i < pcb->nrpages; ++i) + if (pcb->pages[i]) + put_page(pcb->pages[i]); + kfree(pcb->pages); + pcb->pages = NULL; + } +} diff --git a/fs/erofs/super.c b/fs/erofs/super.c new file mode 100644 index 000000000000..f5c0be845bd7 --- /dev/null +++ b/fs/erofs/super.c @@ -0,0 +1,791 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#include +#include +#include +#include +#include +#include +#include "xattr.h" + +#define CREATE_TRACE_POINTS +#include + +static struct kmem_cache *erofs_inode_cachep __read_mostly; + +void _erofs_err(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("(device %s): %s: %pV", sb->s_id, function, &vaf); + va_end(args); +} + +void _erofs_info(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_info("(device %s): %pV", sb->s_id, &vaf); + va_end(args); +} + +static int erofs_superblock_csum_verify(struct super_block *sb, void *sbdata) +{ + struct erofs_super_block *dsb; + u32 expected_crc, crc; + + dsb = kmemdup(sbdata + EROFS_SUPER_OFFSET, + EROFS_BLKSIZ - EROFS_SUPER_OFFSET, GFP_KERNEL); + if (!dsb) + return -ENOMEM; + + expected_crc = le32_to_cpu(dsb->checksum); + dsb->checksum = 0; + /* to allow for x86 boot sectors and other oddities. */ + crc = crc32c(~0, dsb, EROFS_BLKSIZ - EROFS_SUPER_OFFSET); + kfree(dsb); + + if (crc != expected_crc) { + erofs_err(sb, "invalid checksum 0x%08x, 0x%08x expected", + crc, expected_crc); + return -EBADMSG; + } + return 0; +} + +static void erofs_inode_init_once(void *ptr) +{ + struct erofs_inode *vi = ptr; + + inode_init_once(&vi->vfs_inode); +} + +static struct inode *erofs_alloc_inode(struct super_block *sb) +{ + struct erofs_inode *vi = + kmem_cache_alloc(erofs_inode_cachep, GFP_KERNEL); + + if (!vi) + return NULL; + + /* zero out everything except vfs_inode */ + memset(vi, 0, offsetof(struct erofs_inode, vfs_inode)); + return &vi->vfs_inode; +} + +static void i_callback(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + struct erofs_inode *vi = EROFS_I(inode); + + /* be careful of RCU symlink path */ + if (inode->i_op == &erofs_fast_symlink_iops) + kfree(inode->i_link); + kfree(vi->xattr_shared_xattrs); + + kmem_cache_free(erofs_inode_cachep, vi); +} + +static void erofs_destroy_inode(struct inode *inode) +{ + call_rcu(&inode->i_rcu, i_callback); +} + +static bool check_layout_compatibility(struct super_block *sb, + struct erofs_super_block *dsb) +{ + const unsigned int feature = le32_to_cpu(dsb->feature_incompat); + + EROFS_SB(sb)->feature_incompat = feature; + + /* check if current kernel meets all mandatory requirements */ + if (feature & (~EROFS_ALL_FEATURE_INCOMPAT)) { + erofs_err(sb, + "unidentified incompatible feature %x, please upgrade kernel version", + feature & ~EROFS_ALL_FEATURE_INCOMPAT); + return false; + } + return true; +} + +#ifdef CONFIG_EROFS_FS_ZIP +/* read variable-sized metadata, offset will be aligned by 4-byte */ +static void *erofs_read_metadata(struct super_block *sb, struct page **pagep, + erofs_off_t *offset, int *lengthp) +{ + struct page *page = *pagep; + u8 *buffer, *ptr; + int len, i, cnt; + erofs_blk_t blk; + + *offset = round_up(*offset, 4); + blk = erofs_blknr(*offset); + + if (!page || page->index != blk) { + if (page) { + unlock_page(page); + put_page(page); + } + page = erofs_get_meta_page(sb, blk); + if (IS_ERR(page)) + goto err_nullpage; + } + + ptr = kmap(page); + len = le16_to_cpu(*(__le16 *)&ptr[erofs_blkoff(*offset)]); + if (!len) + len = U16_MAX + 1; + buffer = kmalloc(len, GFP_KERNEL); + if (!buffer) { + buffer = ERR_PTR(-ENOMEM); + goto out; + } + *offset += sizeof(__le16); + *lengthp = len; + + for (i = 0; i < len; i += cnt) { + cnt = min(EROFS_BLKSIZ - (int)erofs_blkoff(*offset), len - i); + blk = erofs_blknr(*offset); + + if (!page || page->index != blk) { + if (page) { + kunmap(page); + unlock_page(page); + put_page(page); + } + page = erofs_get_meta_page(sb, blk); + if (IS_ERR(page)) { + kfree(buffer); + goto err_nullpage; + } + ptr = kmap(page); + } + memcpy(buffer + i, ptr + erofs_blkoff(*offset), cnt); + *offset += cnt; + } +out: + kunmap(page); + *pagep = page; + return buffer; +err_nullpage: + *pagep = NULL; + return page; +} + +static int erofs_load_compr_cfgs(struct super_block *sb, + struct erofs_super_block *dsb) +{ + struct erofs_sb_info *sbi; + struct page *page; + unsigned int algs, alg; + erofs_off_t offset; + int size, ret; + + sbi = EROFS_SB(sb); + sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs); + + if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) { + erofs_err(sb, "try to load compressed fs with unsupported algorithms %x", + sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS); + return -EINVAL; + } + + offset = EROFS_SUPER_OFFSET + sbi->sb_size; + page = NULL; + alg = 0; + ret = 0; + + for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) { + void *data; + + if (!(algs & 1)) + continue; + + data = erofs_read_metadata(sb, &page, &offset, &size); + if (IS_ERR(data)) { + ret = PTR_ERR(data); + goto err; + } + + switch (alg) { + case Z_EROFS_COMPRESSION_LZ4: + ret = z_erofs_load_lz4_config(sb, dsb, data, size); + break; + default: + DBG_BUGON(1); + ret = -EFAULT; + } + kfree(data); + if (ret) + goto err; + } +err: + if (page) { + unlock_page(page); + put_page(page); + } + return ret; +} +#else +static int erofs_load_compr_cfgs(struct super_block *sb, + struct erofs_super_block *dsb) +{ + if (dsb->u1.available_compr_algs) { + erofs_err(sb, "try to load compressed fs when compression is disabled"); + return -EINVAL; + } + return 0; +} +#endif + +static int erofs_read_superblock(struct super_block *sb) +{ + struct erofs_sb_info *sbi; + struct page *page; + struct erofs_super_block *dsb; + unsigned int blkszbits; + void *data; + int ret; + + page = read_mapping_page(sb->s_bdev->bd_inode->i_mapping, 0, NULL); + if (IS_ERR(page)) { + erofs_err(sb, "cannot read erofs superblock"); + return PTR_ERR(page); + } + + sbi = EROFS_SB(sb); + + data = kmap(page); + dsb = (struct erofs_super_block *)(data + EROFS_SUPER_OFFSET); + + ret = -EINVAL; + if (le32_to_cpu(dsb->magic) != EROFS_SUPER_MAGIC_V1) { + erofs_err(sb, "cannot find valid erofs superblock"); + goto out; + } + + sbi->feature_compat = le32_to_cpu(dsb->feature_compat); + if (erofs_sb_has_sb_chksum(sbi)) { + ret = erofs_superblock_csum_verify(sb, data); + if (ret) + goto out; + } + + ret = -EINVAL; + blkszbits = dsb->blkszbits; + /* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */ + if (blkszbits != LOG_BLOCK_SIZE) { + erofs_err(sb, "blkszbits %u isn't supported on this platform", + blkszbits); + goto out; + } + + if (!check_layout_compatibility(sb, dsb)) + goto out; + + sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE; + if (sbi->sb_size > EROFS_BLKSIZ) { + erofs_err(sb, "invalid sb_extslots %u (more than a fs block)", + sbi->sb_size); + goto out; + } + sbi->blocks = le32_to_cpu(dsb->blocks); + sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr); +#ifdef CONFIG_EROFS_FS_XATTR + sbi->xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr); +#endif + sbi->islotbits = ilog2(sizeof(struct erofs_inode_compact)); + sbi->root_nid = le16_to_cpu(dsb->root_nid); + sbi->inos = le64_to_cpu(dsb->inos); + + sbi->build_time = le64_to_cpu(dsb->build_time); + sbi->build_time_nsec = le32_to_cpu(dsb->build_time_nsec); + + memcpy(&sb->s_uuid, dsb->uuid, sizeof(dsb->uuid)); + + ret = strscpy(sbi->volume_name, dsb->volume_name, + sizeof(dsb->volume_name)); + if (ret < 0) { /* -E2BIG */ + erofs_err(sb, "bad volume name without NIL terminator"); + ret = -EFSCORRUPTED; + goto out; + } + + /* parse on-disk compression configurations */ + if (erofs_sb_has_compr_cfgs(sbi)) + ret = erofs_load_compr_cfgs(sb, dsb); + else + ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0); +out: + kunmap(page); + put_page(page); + return ret; +} + +#ifdef CONFIG_EROFS_FS_ZIP +static int erofs_build_cache_strategy(struct super_block *sb, + substring_t *args) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + const char *cs = match_strdup(args); + int err = 0; + + if (!cs) { + erofs_err(sb, "Not enough memory to store cache strategy"); + return -ENOMEM; + } + + if (!strcmp(cs, "disabled")) { + sbi->cache_strategy = EROFS_ZIP_CACHE_DISABLED; + } else if (!strcmp(cs, "readahead")) { + sbi->cache_strategy = EROFS_ZIP_CACHE_READAHEAD; + } else if (!strcmp(cs, "readaround")) { + sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND; + } else { + erofs_err(sb, "Unrecognized cache strategy \"%s\"", cs); + err = -EINVAL; + } + kfree(cs); + return err; +} +#else +static int erofs_build_cache_strategy(struct super_block *sb, + substring_t *args) +{ + erofs_info(sb, "EROFS compression is disabled, so cache strategy is ignored"); + return 0; +} +#endif + +/* set up default EROFS parameters */ +static void erofs_default_options(struct erofs_sb_info *sbi) +{ +#ifdef CONFIG_EROFS_FS_ZIP + sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND; + sbi->max_sync_decompress_pages = 3; + sbi->readahead_sync_decompress = false; +#endif +#ifdef CONFIG_EROFS_FS_XATTR + set_opt(sbi, XATTR_USER); +#endif +#ifdef CONFIG_EROFS_FS_POSIX_ACL + set_opt(sbi, POSIX_ACL); +#endif +} + +enum { + Opt_user_xattr, + Opt_nouser_xattr, + Opt_acl, + Opt_noacl, + Opt_cache_strategy, + Opt_err +}; + +static match_table_t erofs_tokens = { + {Opt_user_xattr, "user_xattr"}, + {Opt_nouser_xattr, "nouser_xattr"}, + {Opt_acl, "acl"}, + {Opt_noacl, "noacl"}, + {Opt_cache_strategy, "cache_strategy=%s"}, + {Opt_err, NULL} +}; + +static int erofs_parse_options(struct super_block *sb, char *options) +{ + substring_t args[MAX_OPT_ARGS]; + char *p; + int err; + + if (!options) + return 0; + + while ((p = strsep(&options, ","))) { + int token; + + if (!*p) + continue; + + args[0].to = args[0].from = NULL; + token = match_token(p, erofs_tokens, args); + + switch (token) { +#ifdef CONFIG_EROFS_FS_XATTR + case Opt_user_xattr: + set_opt(EROFS_SB(sb), XATTR_USER); + break; + case Opt_nouser_xattr: + clear_opt(EROFS_SB(sb), XATTR_USER); + break; +#else + case Opt_user_xattr: + erofs_info(sb, "user_xattr options not supported"); + break; + case Opt_nouser_xattr: + erofs_info(sb, "nouser_xattr options not supported"); + break; +#endif +#ifdef CONFIG_EROFS_FS_POSIX_ACL + case Opt_acl: + set_opt(EROFS_SB(sb), POSIX_ACL); + break; + case Opt_noacl: + clear_opt(EROFS_SB(sb), POSIX_ACL); + break; +#else + case Opt_acl: + erofs_info(sb, "acl options not supported"); + break; + case Opt_noacl: + erofs_info(sb, "noacl options not supported"); + break; +#endif + case Opt_cache_strategy: + err = erofs_build_cache_strategy(sb, args); + if (err) + return err; + break; + default: + erofs_err(sb, "Unrecognized mount option \"%s\" or missing value", p); + return -EINVAL; + } + } + return 0; +} + +#ifdef CONFIG_EROFS_FS_ZIP +static const struct address_space_operations managed_cache_aops; + +static int erofs_managed_cache_releasepage(struct page *page, gfp_t gfp_mask) +{ + int ret = 1; /* 0 - busy */ + struct address_space *const mapping = page->mapping; + + DBG_BUGON(!PageLocked(page)); + DBG_BUGON(mapping->a_ops != &managed_cache_aops); + + if (PagePrivate(page)) + ret = erofs_try_to_free_cached_page(mapping, page); + + return ret; +} + +static void erofs_managed_cache_invalidatepage(struct page *page, + unsigned int offset, + unsigned int length) +{ + const unsigned int stop = length + offset; + + DBG_BUGON(!PageLocked(page)); + + /* Check for potential overflow in debug mode */ + DBG_BUGON(stop > PAGE_SIZE || stop < length); + + if (offset == 0 && stop == PAGE_SIZE) + while (!erofs_managed_cache_releasepage(page, GFP_NOFS)) + cond_resched(); +} + +static const struct address_space_operations managed_cache_aops = { + .releasepage = erofs_managed_cache_releasepage, + .invalidatepage = erofs_managed_cache_invalidatepage, +}; + +static int erofs_init_managed_cache(struct super_block *sb) +{ + struct erofs_sb_info *const sbi = EROFS_SB(sb); + struct inode *const inode = new_inode(sb); + + if (!inode) + return -ENOMEM; + + set_nlink(inode, 1); + inode->i_size = OFFSET_MAX; + + inode->i_mapping->a_ops = &managed_cache_aops; + mapping_set_gfp_mask(inode->i_mapping, + GFP_NOFS | __GFP_HIGHMEM | __GFP_MOVABLE); + sbi->managed_cache = inode; + return 0; +} +#else +static int erofs_init_managed_cache(struct super_block *sb) { return 0; } +#endif + +static int erofs_fill_super(struct super_block *sb, void *data, int silent) +{ + struct inode *inode; + struct erofs_sb_info *sbi; + int err; + + sb->s_magic = EROFS_SUPER_MAGIC; + + if (!sb_set_blocksize(sb, EROFS_BLKSIZ)) { + erofs_err(sb, "failed to set erofs blksize"); + return -EINVAL; + } + + sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); + if (!sbi) + return -ENOMEM; + + sb->s_fs_info = sbi; + err = erofs_read_superblock(sb); + if (err) + return err; + + sb->s_flags |= SB_RDONLY | SB_NOATIME; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_time_gran = 1; + + sb->s_op = &erofs_sops; + sb->s_xattr = erofs_xattr_handlers; + + /* set erofs default mount options */ + erofs_default_options(sbi); + + err = erofs_parse_options(sb, data); + if (err) + return err; + + if (test_opt(sbi, POSIX_ACL)) + sb->s_flags |= SB_POSIXACL; + else + sb->s_flags &= ~SB_POSIXACL; + +#ifdef CONFIG_EROFS_FS_ZIP + INIT_RADIX_TREE(&sbi->workstn_tree, GFP_ATOMIC); +#endif + + /* get the root inode */ + inode = erofs_iget(sb, ROOT_NID(sbi), true); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (!S_ISDIR(inode->i_mode)) { + erofs_err(sb, "rootino(nid %llu) is not a directory(i_mode %o)", + ROOT_NID(sbi), inode->i_mode); + iput(inode); + return -EINVAL; + } + + sb->s_root = d_make_root(inode); + if (!sb->s_root) + return -ENOMEM; + + erofs_shrinker_register(sb); + /* sb->s_umount is already locked, SB_ACTIVE and SB_BORN are not set */ + err = erofs_init_managed_cache(sb); + if (err) + return err; + + erofs_info(sb, "mounted with opts: %s, root inode @ nid %llu.", + (char *)data, ROOT_NID(sbi)); + return 0; +} + +static struct dentry *erofs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) +{ + return mount_bdev(fs_type, flags, dev_name, data, erofs_fill_super); +} + +/* + * could be triggered after deactivate_locked_super() + * is called, thus including umount and failed to initialize. + */ +static void erofs_kill_sb(struct super_block *sb) +{ + struct erofs_sb_info *sbi; + + WARN_ON(sb->s_magic != EROFS_SUPER_MAGIC); + + kill_block_super(sb); + + sbi = EROFS_SB(sb); + if (!sbi) + return; + kfree(sbi); + sb->s_fs_info = NULL; +} + +/* called when ->s_root is non-NULL */ +static void erofs_put_super(struct super_block *sb) +{ + struct erofs_sb_info *const sbi = EROFS_SB(sb); + + DBG_BUGON(!sbi); + + erofs_shrinker_unregister(sb); +#ifdef CONFIG_EROFS_FS_ZIP + iput(sbi->managed_cache); + sbi->managed_cache = NULL; +#endif +} + +static struct file_system_type erofs_fs_type = { + .owner = THIS_MODULE, + .name = "erofs", + .mount = erofs_mount, + .kill_sb = erofs_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("erofs"); + +static int __init erofs_module_init(void) +{ + int err; + + erofs_check_ondisk_layout_definitions(); + + erofs_inode_cachep = kmem_cache_create("erofs_inode", + sizeof(struct erofs_inode), 0, + SLAB_RECLAIM_ACCOUNT, + erofs_inode_init_once); + if (!erofs_inode_cachep) { + err = -ENOMEM; + goto icache_err; + } + + err = erofs_init_shrinker(); + if (err) + goto shrinker_err; + + erofs_pcpubuf_init(); + err = z_erofs_init_zip_subsystem(); + if (err) + goto zip_err; + + err = register_filesystem(&erofs_fs_type); + if (err) + goto fs_err; + + return 0; + +fs_err: + z_erofs_exit_zip_subsystem(); +zip_err: + erofs_exit_shrinker(); +shrinker_err: + kmem_cache_destroy(erofs_inode_cachep); +icache_err: + return err; +} + +static void __exit erofs_module_exit(void) +{ + unregister_filesystem(&erofs_fs_type); + z_erofs_exit_zip_subsystem(); + erofs_exit_shrinker(); + + /* Ensure all RCU free inodes are safe before cache is destroyed. */ + rcu_barrier(); + kmem_cache_destroy(erofs_inode_cachep); + erofs_pcpubuf_exit(); +} + +/* get filesystem statistics */ +static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct erofs_sb_info *sbi = EROFS_SB(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = sb->s_magic; + buf->f_bsize = EROFS_BLKSIZ; + buf->f_blocks = sbi->blocks; + buf->f_bfree = buf->f_bavail = 0; + + buf->f_files = ULLONG_MAX; + buf->f_ffree = ULLONG_MAX - sbi->inos; + + buf->f_namelen = EROFS_NAME_LEN; + + buf->f_fsid.val[0] = (u32)id; + buf->f_fsid.val[1] = (u32)(id >> 32); + return 0; +} + +static int erofs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct erofs_sb_info *sbi __maybe_unused = EROFS_SB(root->d_sb); + +#ifdef CONFIG_EROFS_FS_XATTR + if (test_opt(sbi, XATTR_USER)) + seq_puts(seq, ",user_xattr"); + else + seq_puts(seq, ",nouser_xattr"); +#endif +#ifdef CONFIG_EROFS_FS_POSIX_ACL + if (test_opt(sbi, POSIX_ACL)) + seq_puts(seq, ",acl"); + else + seq_puts(seq, ",noacl"); +#endif +#ifdef CONFIG_EROFS_FS_ZIP + if (sbi->cache_strategy == EROFS_ZIP_CACHE_DISABLED) { + seq_puts(seq, ",cache_strategy=disabled"); + } else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAHEAD) { + seq_puts(seq, ",cache_strategy=readahead"); + } else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAROUND) { + seq_puts(seq, ",cache_strategy=readaround"); + } +#endif + return 0; +} + +static int erofs_remount(struct super_block *sb, int *flags, char *data) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + unsigned int org_mnt_opt = sbi->mount_opt; + int err; + + DBG_BUGON(!sb_rdonly(sb)); + err = erofs_parse_options(sb, data); + if (err) + goto out; + + if (test_opt(sbi, POSIX_ACL)) + sb->s_flags |= SB_POSIXACL; + else + sb->s_flags &= ~SB_POSIXACL; + + *flags |= SB_RDONLY; + return 0; +out: + sbi->mount_opt = org_mnt_opt; + return err; +} + +const struct super_operations erofs_sops = { + .put_super = erofs_put_super, + .alloc_inode = erofs_alloc_inode, + .destroy_inode = erofs_destroy_inode, + .statfs = erofs_statfs, + .show_options = erofs_show_options, + .remount_fs = erofs_remount, +}; + +module_init(erofs_module_init); +module_exit(erofs_module_exit); + +MODULE_DESCRIPTION("Enhanced ROM File System"); +MODULE_AUTHOR("Gao Xiang, Chao Yu, Miao Xie, CONSUMER BG, HUAWEI Inc."); +MODULE_LICENSE("GPL"); diff --git a/drivers/staging/erofs/include/linux/tagptr.h b/fs/erofs/tagptr.h similarity index 93% rename from drivers/staging/erofs/include/linux/tagptr.h rename to fs/erofs/tagptr.h index ccd106dbd48e..64ceb7270b5c 100644 --- a/drivers/staging/erofs/include/linux/tagptr.h +++ b/fs/erofs/tagptr.h @@ -1,11 +1,9 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * Tagged pointer implementation - * - * Copyright (C) 2018 Gao Xiang +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * A tagged pointer implementation */ -#ifndef _LINUX_TAGPTR_H -#define _LINUX_TAGPTR_H +#ifndef __EROFS_FS_TAGPTR_H +#define __EROFS_FS_TAGPTR_H #include #include @@ -106,5 +104,4 @@ tagptr_init(o, cmpxchg(&ptptr->v, o.v, n.v)); }) ptptr->v &= ~tags; \ *ptptr; }) -#endif - +#endif /* __EROFS_FS_TAGPTR_H */ diff --git a/drivers/staging/erofs/utils.c b/fs/erofs/utils.c similarity index 51% rename from drivers/staging/erofs/utils.c rename to fs/erofs/utils.c index 4de9c39535eb..160554accdfb 100644 --- a/drivers/staging/erofs/utils.c +++ b/fs/erofs/utils.c @@ -1,16 +1,8 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* - * linux/drivers/staging/erofs/utils.c - * * Copyright (C) 2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. + * https://www.huawei.com/ */ - #include "internal.h" #include @@ -20,43 +12,55 @@ struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp) if (!list_empty(pool)) { page = lru_to_page(pool); + DBG_BUGON(page_ref_count(page) != 1); list_del(&page->lru); } else { - page = alloc_pages(gfp | __GFP_NOFAIL, 0); + page = alloc_page(gfp); } return page; } +#ifdef CONFIG_EROFS_FS_ZIP /* global shrink count (for all mounted EROFS instances) */ static atomic_long_t erofs_global_shrink_cnt; -#ifdef CONFIG_EROFS_FS_ZIP +#define __erofs_workgroup_get(grp) atomic_inc(&(grp)->refcount) +#define __erofs_workgroup_put(grp) atomic_dec(&(grp)->refcount) + +static int erofs_workgroup_get(struct erofs_workgroup *grp) +{ + int o; + +repeat: + o = erofs_wait_on_workgroup_freezed(grp); + if (o <= 0) + return -1; + + if (atomic_cmpxchg(&grp->refcount, o, o + 1) != o) + goto repeat; -/* radix_tree and the future XArray both don't use tagptr_t yet */ -struct erofs_workgroup *erofs_find_workgroup( - struct super_block *sb, pgoff_t index, bool *tag) + /* decrease refcount paired by erofs_workgroup_put */ + if (o == 1) + atomic_long_dec(&erofs_global_shrink_cnt); + return 0; +} + +struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb, + pgoff_t index) { struct erofs_sb_info *sbi = EROFS_SB(sb); struct erofs_workgroup *grp; - int oldcount; repeat: rcu_read_lock(); grp = radix_tree_lookup(&sbi->workstn_tree, index); - if (grp != NULL) { - *tag = radix_tree_exceptional_entry(grp); - grp = (void *)((unsigned long)grp & - ~RADIX_TREE_EXCEPTIONAL_ENTRY); - - if (erofs_workgroup_get(grp, &oldcount)) { + if (grp) { + if (erofs_workgroup_get(grp)) { /* prefer to relax rcu read side */ rcu_read_unlock(); goto repeat; } - /* decrease refcount added by erofs_workgroup_put */ - if (unlikely(oldcount == 1)) - atomic_long_dec(&erofs_global_shrink_cnt); DBG_BUGON(index != grp->index); } rcu_read_unlock(); @@ -64,14 +68,13 @@ struct erofs_workgroup *erofs_find_workgroup( } int erofs_register_workgroup(struct super_block *sb, - struct erofs_workgroup *grp, - bool tag) + struct erofs_workgroup *grp) { struct erofs_sb_info *sbi; int err; /* grp shouldn't be broken or used before */ - if (unlikely(atomic_read(&grp->refcount) != 1)) { + if (atomic_read(&grp->refcount) != 1) { DBG_BUGON(1); return -EINVAL; } @@ -81,35 +84,28 @@ int erofs_register_workgroup(struct super_block *sb, return err; sbi = EROFS_SB(sb); - erofs_workstn_lock(sbi); - - if (tag) - grp = (void *)((unsigned long)grp | - 1UL << RADIX_TREE_EXCEPTIONAL_SHIFT); + xa_lock(&sbi->workstn_tree); /* * Bump up reference count before making this workgroup * visible to other users in order to avoid potential UAF - * without serialized by erofs_workstn_lock. + * without serialized by workstn_lock. */ __erofs_workgroup_get(grp); - err = radix_tree_insert(&sbi->workstn_tree, - grp->index, grp); - if (unlikely(err)) + err = radix_tree_insert(&sbi->workstn_tree, grp->index, grp); + if (err) /* * it's safe to decrease since the workgroup isn't visible * and refcount >= 2 (cannot be freezed). */ __erofs_workgroup_put(grp); - erofs_workstn_unlock(sbi); + xa_unlock(&sbi->workstn_tree); radix_tree_preload_end(); return err; } -extern void erofs_workgroup_free_rcu(struct erofs_workgroup *grp); - static void __erofs_workgroup_free(struct erofs_workgroup *grp) { atomic_long_dec(&erofs_global_shrink_cnt); @@ -127,33 +123,22 @@ int erofs_workgroup_put(struct erofs_workgroup *grp) return count; } -#ifdef EROFS_FS_HAS_MANAGED_CACHE -/* for cache-managed case, customized reclaim paths exist */ -static void erofs_workgroup_unfreeze_final(struct erofs_workgroup *grp) +static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, + struct erofs_workgroup *grp) { - erofs_workgroup_unfreeze(grp, 0); - __erofs_workgroup_free(grp); -} - -bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, - struct erofs_workgroup *grp, - bool cleanup) -{ - void *entry; - /* - * for managed cache enabled, the refcount of workgroups - * themselves could be < 0 (freezed). So there is no guarantee - * that all refcount > 0 if managed cache is enabled. + * If managed cache is on, refcount of workgroups + * themselves could be < 0 (freezed). In other words, + * there is no guarantee that all refcounts > 0. */ if (!erofs_workgroup_try_to_freeze(grp, 1)) return false; /* - * note that all cached pages should be unlinked - * before delete it from the radix tree. - * Otherwise some cached pages of an orphan old workgroup - * could be still linked after the new one is available. + * Note that all cached pages should be unattached + * before deleted from the radix tree. Otherwise some + * cached pages could be still attached to the orphan + * old workgroup when the new one is available in the tree. */ if (erofs_try_to_free_all_cached_pages(sbi, grp)) { erofs_workgroup_unfreeze(grp, 1); @@ -161,87 +146,52 @@ bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, } /* - * it is impossible to fail after the workgroup is freezed, + * It's impossible to fail after the workgroup is freezed, * however in order to avoid some race conditions, add a * DBG_BUGON to observe this in advance. */ - entry = radix_tree_delete(&sbi->workstn_tree, grp->index); - DBG_BUGON((void *)((unsigned long)entry & - ~RADIX_TREE_EXCEPTIONAL_ENTRY) != grp); + DBG_BUGON(radix_tree_delete(&sbi->workstn_tree, grp->index) != grp); - /* - * if managed cache is enable, the last refcount - * should indicate the related workstation. - */ - erofs_workgroup_unfreeze_final(grp); - return true; -} - -#else -/* for nocache case, no customized reclaim path at all */ -bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, - struct erofs_workgroup *grp, - bool cleanup) -{ - int cnt = atomic_read(&grp->refcount); - void *entry; - - DBG_BUGON(cnt <= 0); - DBG_BUGON(cleanup && cnt != 1); - - if (cnt > 1) - return false; - - entry = radix_tree_delete(&sbi->workstn_tree, grp->index); - DBG_BUGON((void *)((unsigned long)entry & - ~RADIX_TREE_EXCEPTIONAL_ENTRY) != grp); - - /* (rarely) could be grabbed again when freeing */ - erofs_workgroup_put(grp); + /* last refcount should be connected with its managed pslot. */ + erofs_workgroup_unfreeze(grp, 0); + __erofs_workgroup_free(grp); return true; } -#endif - -unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi, - unsigned long nr_shrink, - bool cleanup) +static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi, + unsigned long nr_shrink) { pgoff_t first_index = 0; void *batch[PAGEVEC_SIZE]; - unsigned freed = 0; + unsigned int freed = 0; int i, found; repeat: - erofs_workstn_lock(sbi); + xa_lock(&sbi->workstn_tree); found = radix_tree_gang_lookup(&sbi->workstn_tree, - batch, first_index, PAGEVEC_SIZE); + batch, first_index, PAGEVEC_SIZE); for (i = 0; i < found; ++i) { - struct erofs_workgroup *grp = (void *) - ((unsigned long)batch[i] & - ~RADIX_TREE_EXCEPTIONAL_ENTRY); + struct erofs_workgroup *grp = batch[i]; first_index = grp->index + 1; /* try to shrink each valid workgroup */ - if (!erofs_try_to_release_workgroup(sbi, grp, cleanup)) + if (!erofs_try_to_release_workgroup(sbi, grp)) continue; ++freed; - if (unlikely(!--nr_shrink)) + if (!--nr_shrink) break; } - erofs_workstn_unlock(sbi); + xa_unlock(&sbi->workstn_tree); if (i && nr_shrink) goto repeat; return freed; } -#endif - /* protected by 'erofs_sb_list_lock' */ static unsigned int shrinker_run_no; @@ -249,7 +199,7 @@ static unsigned int shrinker_run_no; static DEFINE_SPINLOCK(erofs_sb_list_lock); static LIST_HEAD(erofs_sb_list); -void erofs_register_super(struct super_block *sb) +void erofs_shrinker_register(struct super_block *sb) { struct erofs_sb_info *sbi = EROFS_SB(sb); @@ -260,21 +210,28 @@ void erofs_register_super(struct super_block *sb) spin_unlock(&erofs_sb_list_lock); } -void erofs_unregister_super(struct super_block *sb) +void erofs_shrinker_unregister(struct super_block *sb) { + struct erofs_sb_info *const sbi = EROFS_SB(sb); + + mutex_lock(&sbi->umount_mutex); + /* clean up all remaining workgroups in memory */ + erofs_shrink_workstation(sbi, ~0UL); + spin_lock(&erofs_sb_list_lock); - list_del(&EROFS_SB(sb)->list); + list_del(&sbi->list); spin_unlock(&erofs_sb_list_lock); + mutex_unlock(&sbi->umount_mutex); } -unsigned long erofs_shrink_count(struct shrinker *shrink, - struct shrink_control *sc) +static unsigned long erofs_shrink_count(struct shrinker *shrink, + struct shrink_control *sc) { return atomic_long_read(&erofs_global_shrink_cnt); } -unsigned long erofs_shrink_scan(struct shrinker *shrink, - struct shrink_control *sc) +static unsigned long erofs_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc) { struct erofs_sb_info *sbi; struct list_head *p; @@ -284,9 +241,9 @@ unsigned long erofs_shrink_scan(struct shrinker *shrink, unsigned long freed = 0; spin_lock(&erofs_sb_list_lock); - do + do { run_no = ++shrinker_run_no; - while (run_no == 0); + } while (run_no == 0); /* Iterate over all mounted superblocks and try to shrink them */ p = erofs_sb_list.next; @@ -308,9 +265,7 @@ unsigned long erofs_shrink_scan(struct shrinker *shrink, spin_unlock(&erofs_sb_list_lock); sbi->shrinker_run_no = run_no; -#ifdef CONFIG_EROFS_FS_ZIP - freed += erofs_shrink_workstation(sbi, nr - freed, false); -#endif + freed += erofs_shrink_workstation(sbi, nr - freed); spin_lock(&erofs_sb_list_lock); /* Get the next list element before we move this one */ @@ -330,3 +285,19 @@ unsigned long erofs_shrink_scan(struct shrinker *shrink, return freed; } +static struct shrinker erofs_shrinker_info = { + .scan_objects = erofs_shrink_scan, + .count_objects = erofs_shrink_count, + .seeks = DEFAULT_SEEKS, +}; + +int __init erofs_init_shrinker(void) +{ + return register_shrinker(&erofs_shrinker_info); +} + +void erofs_exit_shrinker(void) +{ + unregister_shrinker(&erofs_shrinker_info); +} +#endif /* !CONFIG_EROFS_FS_ZIP */ diff --git a/drivers/staging/erofs/xattr.c b/fs/erofs/xattr.c similarity index 70% rename from drivers/staging/erofs/xattr.c rename to fs/erofs/xattr.c index d48687ca2199..2b9bb3a7a5f2 100644 --- a/drivers/staging/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -1,14 +1,7 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* - * linux/drivers/staging/erofs/xattr.c - * * Copyright (C) 2017-2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. + * https://www.huawei.com/ */ #include #include "xattr.h" @@ -19,13 +12,13 @@ struct xattr_iter { void *kaddr; erofs_blk_t blkaddr; - unsigned ofs; + unsigned int ofs; }; static inline void xattr_iter_end(struct xattr_iter *it, bool atomic) { /* the only user of kunmap() is 'init_inode_xattrs' */ - if (unlikely(!atomic)) + if (!atomic) kunmap(it->page); else kunmap_atomic(it->kaddr); @@ -44,23 +37,30 @@ static inline void xattr_iter_end_final(struct xattr_iter *it) static int init_inode_xattrs(struct inode *inode) { - struct erofs_vnode *const vi = EROFS_V(inode); + struct erofs_inode *const vi = EROFS_I(inode); struct xattr_iter it; - unsigned i; + unsigned int i; struct erofs_xattr_ibody_header *ih; + struct super_block *sb; struct erofs_sb_info *sbi; bool atomic_map; int ret = 0; /* the most case is that xattrs of this inode are initialized. */ - if (test_bit(EROFS_V_EA_INITED_BIT, &vi->flags)) + if (test_bit(EROFS_I_EA_INITED_BIT, &vi->flags)) { + /* + * paired with smp_mb() at the end of the function to ensure + * fields will only be observed after the bit is set. + */ + smp_mb(); return 0; + } - if (wait_on_bit_lock(&vi->flags, EROFS_V_BL_XATTR_BIT, TASK_KILLABLE)) + if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_XATTR_BIT, TASK_KILLABLE)) return -ERESTARTSYS; /* someone has initialized xattrs for us? */ - if (test_bit(EROFS_V_EA_INITED_BIT, &vi->flags)) + if (test_bit(EROFS_I_EA_INITED_BIT, &vi->flags)) goto out_unlock; /* @@ -72,25 +72,29 @@ static int init_inode_xattrs(struct inode *inode) * undefined right now (maybe use later with some new sb feature). */ if (vi->xattr_isize == sizeof(struct erofs_xattr_ibody_header)) { - errln("xattr_isize %d of nid %llu is not supported yet", - vi->xattr_isize, vi->nid); - ret = -ENOTSUPP; + erofs_err(inode->i_sb, + "xattr_isize %d of nid %llu is not supported yet", + vi->xattr_isize, vi->nid); + ret = -EOPNOTSUPP; goto out_unlock; } else if (vi->xattr_isize < sizeof(struct erofs_xattr_ibody_header)) { - if (unlikely(vi->xattr_isize)) { + if (vi->xattr_isize) { + erofs_err(inode->i_sb, + "bogus xattr ibody @ nid %llu", vi->nid); DBG_BUGON(1); - ret = -EIO; + ret = -EFSCORRUPTED; goto out_unlock; /* xattr ondisk layout error */ } ret = -ENOATTR; goto out_unlock; } - sbi = EROFS_I_SB(inode); + sb = inode->i_sb; + sbi = EROFS_SB(sb); it.blkaddr = erofs_blknr(iloc(sbi, vi->nid) + vi->inode_isize); it.ofs = erofs_blkoff(iloc(sbi, vi->nid) + vi->inode_isize); - it.page = erofs_get_inline_page(inode, it.blkaddr); + it.page = erofs_get_meta_page(sb, it.blkaddr); if (IS_ERR(it.page)) { ret = PTR_ERR(it.page); goto out_unlock; @@ -115,13 +119,12 @@ static int init_inode_xattrs(struct inode *inode) it.ofs += sizeof(struct erofs_xattr_ibody_header); for (i = 0; i < vi->xattr_shared_count; ++i) { - if (unlikely(it.ofs >= EROFS_BLKSIZ)) { + if (it.ofs >= EROFS_BLKSIZ) { /* cannot be unaligned */ - BUG_ON(it.ofs != EROFS_BLKSIZ); + DBG_BUGON(it.ofs != EROFS_BLKSIZ); xattr_iter_end(&it, atomic_map); - it.page = erofs_get_meta_page(inode->i_sb, - ++it.blkaddr, S_ISDIR(inode->i_mode)); + it.page = erofs_get_meta_page(sb, ++it.blkaddr); if (IS_ERR(it.page)) { kfree(vi->xattr_shared_xattrs); vi->xattr_shared_xattrs = NULL; @@ -139,18 +142,29 @@ static int init_inode_xattrs(struct inode *inode) } xattr_iter_end(&it, atomic_map); - set_bit(EROFS_V_EA_INITED_BIT, &vi->flags); + /* paired with smp_mb() at the beginning of the function. */ + smp_mb(); + set_bit(EROFS_I_EA_INITED_BIT, &vi->flags); out_unlock: - clear_and_wake_up_bit(EROFS_V_BL_XATTR_BIT, &vi->flags); + clear_and_wake_up_bit(EROFS_I_BL_XATTR_BIT, &vi->flags); return ret; } +/* + * the general idea for these return values is + * if 0 is returned, go on processing the current xattr; + * 1 (> 0) is returned, skip this round to process the next xattr; + * -err (< 0) is returned, an error (maybe ENOXATTR) occurred + * and need to be handled + */ struct xattr_iter_handlers { - int (*entry)(struct xattr_iter *, struct erofs_xattr_entry *); - int (*name)(struct xattr_iter *, unsigned, char *, unsigned); - int (*alloc_buffer)(struct xattr_iter *, unsigned); - void (*value)(struct xattr_iter *, unsigned, char *, unsigned); + int (*entry)(struct xattr_iter *_it, struct erofs_xattr_entry *entry); + int (*name)(struct xattr_iter *_it, unsigned int processed, char *buf, + unsigned int len); + int (*alloc_buffer)(struct xattr_iter *_it, unsigned int value_sz); + void (*value)(struct xattr_iter *_it, unsigned int processed, char *buf, + unsigned int len); }; static inline int xattr_iter_fixup(struct xattr_iter *it) @@ -161,7 +175,8 @@ static inline int xattr_iter_fixup(struct xattr_iter *it) xattr_iter_end(it, true); it->blkaddr += erofs_blknr(it->ofs); - it->page = erofs_get_meta_page(it->sb, it->blkaddr, false); + + it->page = erofs_get_meta_page(it->sb, it->blkaddr); if (IS_ERR(it->page)) { int err = PTR_ERR(it->page); @@ -175,15 +190,15 @@ static inline int xattr_iter_fixup(struct xattr_iter *it) } static int inline_xattr_iter_begin(struct xattr_iter *it, - struct inode *inode) + struct inode *inode) { - struct erofs_vnode *const vi = EROFS_V(inode); + struct erofs_inode *const vi = EROFS_I(inode); struct erofs_sb_info *const sbi = EROFS_SB(inode->i_sb); - unsigned xattr_header_sz, inline_xattr_ofs; + unsigned int xattr_header_sz, inline_xattr_ofs; xattr_header_sz = inlinexattr_header_size(inode); - if (unlikely(xattr_header_sz >= vi->xattr_isize)) { - BUG_ON(xattr_header_sz > vi->xattr_isize); + if (xattr_header_sz >= vi->xattr_isize) { + DBG_BUGON(xattr_header_sz > vi->xattr_isize); return -ENOATTR; } @@ -192,7 +207,7 @@ static int inline_xattr_iter_begin(struct xattr_iter *it, it->blkaddr = erofs_blknr(iloc(sbi, vi->nid) + inline_xattr_ofs); it->ofs = erofs_blkoff(iloc(sbi, vi->nid) + inline_xattr_ofs); - it->page = erofs_get_inline_page(inode, it->blkaddr); + it->page = erofs_get_meta_page(inode->i_sb, it->blkaddr); if (IS_ERR(it->page)) return PTR_ERR(it->page); @@ -200,11 +215,16 @@ static int inline_xattr_iter_begin(struct xattr_iter *it, return vi->xattr_isize - xattr_header_sz; } +/* + * Regardless of success or failure, `xattr_foreach' will end up with + * `ofs' pointing to the next xattr item rather than an arbitrary position. + */ static int xattr_foreach(struct xattr_iter *it, - const struct xattr_iter_handlers *op, unsigned int *tlimit) + const struct xattr_iter_handlers *op, + unsigned int *tlimit) { struct erofs_xattr_entry entry; - unsigned value_sz, processed, slice; + unsigned int value_sz, processed, slice; int err; /* 0. fixup blkaddr, ofs, ipage */ @@ -218,10 +238,14 @@ static int xattr_foreach(struct xattr_iter *it, * therefore entry should be in the page */ entry = *(struct erofs_xattr_entry *)(it->kaddr + it->ofs); - if (tlimit != NULL) { - unsigned entry_sz = EROFS_XATTR_ENTRY_SIZE(&entry); + if (tlimit) { + unsigned int entry_sz = erofs_xattr_entry_size(&entry); - BUG_ON(*tlimit < entry_sz); + /* xattr on-disk corruption: xattr entry beyond xattr_isize */ + if (*tlimit < entry_sz) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } *tlimit -= entry_sz; } @@ -240,7 +264,7 @@ static int xattr_foreach(struct xattr_iter *it, while (processed < entry.e_name_len) { if (it->ofs >= EROFS_BLKSIZ) { - BUG_ON(it->ofs > EROFS_BLKSIZ); + DBG_BUGON(it->ofs > EROFS_BLKSIZ); err = xattr_iter_fixup(it); if (err) @@ -248,8 +272,8 @@ static int xattr_foreach(struct xattr_iter *it, it->ofs = 0; } - slice = min_t(unsigned, PAGE_SIZE - it->ofs, - entry.e_name_len - processed); + slice = min_t(unsigned int, PAGE_SIZE - it->ofs, + entry.e_name_len - processed); /* handle name */ err = op->name(it, processed, it->kaddr + it->ofs, slice); @@ -265,7 +289,7 @@ static int xattr_foreach(struct xattr_iter *it, /* 3. handle xattr value */ processed = 0; - if (op->alloc_buffer != NULL) { + if (op->alloc_buffer) { err = op->alloc_buffer(it, value_sz); if (err) { it->ofs += value_sz; @@ -275,7 +299,7 @@ static int xattr_foreach(struct xattr_iter *it, while (processed < value_sz) { if (it->ofs >= EROFS_BLKSIZ) { - BUG_ON(it->ofs > EROFS_BLKSIZ); + DBG_BUGON(it->ofs > EROFS_BLKSIZ); err = xattr_iter_fixup(it); if (err) @@ -283,17 +307,17 @@ static int xattr_foreach(struct xattr_iter *it, it->ofs = 0; } - slice = min_t(unsigned, PAGE_SIZE - it->ofs, - value_sz - processed); + slice = min_t(unsigned int, PAGE_SIZE - it->ofs, + value_sz - processed); op->value(it, processed, it->kaddr + it->ofs, slice); it->ofs += slice; processed += slice; } out: - /* we assume that ofs is aligned with 4 bytes */ + /* xattrs should be 4-byte aligned (on-disk constraint) */ it->ofs = EROFS_XATTR_ALIGN(it->ofs); - return err; + return err < 0 ? err : 0; } struct getxattr_iter { @@ -305,7 +329,7 @@ struct getxattr_iter { }; static int xattr_entrymatch(struct xattr_iter *_it, - struct erofs_xattr_entry *entry) + struct erofs_xattr_entry *entry) { struct getxattr_iter *it = container_of(_it, struct getxattr_iter, it); @@ -314,7 +338,7 @@ static int xattr_entrymatch(struct xattr_iter *_it, } static int xattr_namematch(struct xattr_iter *_it, - unsigned processed, char *buf, unsigned len) + unsigned int processed, char *buf, unsigned int len) { struct getxattr_iter *it = container_of(_it, struct getxattr_iter, it); @@ -322,17 +346,18 @@ static int xattr_namematch(struct xattr_iter *_it, } static int xattr_checkbuffer(struct xattr_iter *_it, - unsigned value_sz) + unsigned int value_sz) { struct getxattr_iter *it = container_of(_it, struct getxattr_iter, it); int err = it->buffer_size < value_sz ? -ERANGE : 0; it->buffer_size = value_sz; - return it->buffer == NULL ? 1 : err; + return !it->buffer ? 1 : err; } static void xattr_copyvalue(struct xattr_iter *_it, - unsigned processed, char *buf, unsigned len) + unsigned int processed, + char *buf, unsigned int len) { struct getxattr_iter *it = container_of(_it, struct getxattr_iter, it); @@ -349,7 +374,7 @@ static const struct xattr_iter_handlers find_xattr_handlers = { static int inline_getxattr(struct inode *inode, struct getxattr_iter *it) { int ret; - unsigned remaining; + unsigned int remaining; ret = inline_xattr_iter_begin(&it->it, inode); if (ret < 0) @@ -358,22 +383,20 @@ static int inline_getxattr(struct inode *inode, struct getxattr_iter *it) remaining = ret; while (remaining) { ret = xattr_foreach(&it->it, &find_xattr_handlers, &remaining); - if (ret >= 0) - break; - - if (ret != -ENOATTR) /* -ENOMEM, -EIO, etc. */ + if (ret != -ENOATTR) break; } xattr_iter_end_final(&it->it); - return ret < 0 ? ret : it->buffer_size; + return ret ? ret : it->buffer_size; } static int shared_getxattr(struct inode *inode, struct getxattr_iter *it) { - struct erofs_vnode *const vi = EROFS_V(inode); - struct erofs_sb_info *const sbi = EROFS_SB(inode->i_sb); - unsigned i; + struct erofs_inode *const vi = EROFS_I(inode); + struct super_block *const sb = inode->i_sb; + struct erofs_sb_info *const sbi = EROFS_SB(sb); + unsigned int i; int ret = -ENOATTR; for (i = 0; i < vi->xattr_shared_count; ++i) { @@ -386,8 +409,7 @@ static int shared_getxattr(struct inode *inode, struct getxattr_iter *it) if (i) xattr_iter_end(&it->it, true); - it->it.page = erofs_get_meta_page(inode->i_sb, - blkaddr, false); + it->it.page = erofs_get_meta_page(sb, blkaddr); if (IS_ERR(it->it.page)) return PTR_ERR(it->it.page); @@ -396,16 +418,13 @@ static int shared_getxattr(struct inode *inode, struct getxattr_iter *it) } ret = xattr_foreach(&it->it, &find_xattr_handlers, NULL); - if (ret >= 0) - break; - - if (ret != -ENOATTR) /* -ENOMEM, -EIO, etc. */ + if (ret != -ENOATTR) break; } if (vi->xattr_shared_count) xattr_iter_end_final(&it->it); - return ret < 0 ? ret : it->buffer_size; + return ret ? ret : it->buffer_size; } static bool erofs_xattr_user_list(struct dentry *dentry) @@ -419,13 +438,13 @@ static bool erofs_xattr_trusted_list(struct dentry *dentry) } int erofs_getxattr(struct inode *inode, int index, - const char *name, - void *buffer, size_t buffer_size) + const char *name, + void *buffer, size_t buffer_size) { int ret; struct getxattr_iter it; - if (unlikely(name == NULL)) + if (!name) return -EINVAL; ret = init_inode_xattrs(inode); @@ -450,8 +469,8 @@ int erofs_getxattr(struct inode *inode, int index, } static int erofs_xattr_generic_get(const struct xattr_handler *handler, - struct dentry *unused, struct inode *inode, - const char *name, void *buffer, size_t size) + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) { struct erofs_sb_info *const sbi = EROFS_I_SB(inode); @@ -461,8 +480,6 @@ static int erofs_xattr_generic_get(const struct xattr_handler *handler, return -EOPNOTSUPP; break; case EROFS_XATTR_INDEX_TRUSTED: - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; break; case EROFS_XATTR_INDEX_SECURITY: break; @@ -517,24 +534,23 @@ struct listxattr_iter { }; static int xattr_entrylist(struct xattr_iter *_it, - struct erofs_xattr_entry *entry) + struct erofs_xattr_entry *entry) { struct listxattr_iter *it = container_of(_it, struct listxattr_iter, it); - unsigned prefix_len; + unsigned int prefix_len; const char *prefix; const struct xattr_handler *h = erofs_xattr_handler(entry->e_name_index); - if (h == NULL || (h->list != NULL && !h->list(it->dentry))) + if (!h || (h->list && !h->list(it->dentry))) return 1; - /* Note that at least one of 'prefix' and 'name' should be non-NULL */ - prefix = h->prefix != NULL ? h->prefix : h->name; + prefix = xattr_prefix(h); prefix_len = strlen(prefix); - if (it->buffer == NULL) { + if (!it->buffer) { it->buffer_ofs += prefix_len + entry->e_name_len + 1; return 1; } @@ -549,7 +565,7 @@ static int xattr_entrylist(struct xattr_iter *_it, } static int xattr_namelist(struct xattr_iter *_it, - unsigned processed, char *buf, unsigned len) + unsigned int processed, char *buf, unsigned int len) { struct listxattr_iter *it = container_of(_it, struct listxattr_iter, it); @@ -560,7 +576,7 @@ static int xattr_namelist(struct xattr_iter *_it, } static int xattr_skipvalue(struct xattr_iter *_it, - unsigned value_sz) + unsigned int value_sz) { struct listxattr_iter *it = container_of(_it, struct listxattr_iter, it); @@ -579,7 +595,7 @@ static const struct xattr_iter_handlers list_xattr_handlers = { static int inline_listxattr(struct listxattr_iter *it) { int ret; - unsigned remaining; + unsigned int remaining; ret = inline_xattr_iter_begin(&it->it, d_inode(it->dentry)); if (ret < 0) @@ -588,19 +604,20 @@ static int inline_listxattr(struct listxattr_iter *it) remaining = ret; while (remaining) { ret = xattr_foreach(&it->it, &list_xattr_handlers, &remaining); - if (ret < 0) + if (ret) break; } xattr_iter_end_final(&it->it); - return ret < 0 ? ret : it->buffer_ofs; + return ret ? ret : it->buffer_ofs; } static int shared_listxattr(struct listxattr_iter *it) { struct inode *const inode = d_inode(it->dentry); - struct erofs_vnode *const vi = EROFS_V(inode); - struct erofs_sb_info *const sbi = EROFS_I_SB(inode); - unsigned i; + struct erofs_inode *const vi = EROFS_I(inode); + struct super_block *const sb = inode->i_sb; + struct erofs_sb_info *const sbi = EROFS_SB(sb); + unsigned int i; int ret = 0; for (i = 0; i < vi->xattr_shared_count; ++i) { @@ -612,8 +629,7 @@ static int shared_listxattr(struct listxattr_iter *it) if (i) xattr_iter_end(&it->it, true); - it->it.page = erofs_get_meta_page(inode->i_sb, - blkaddr, false); + it->it.page = erofs_get_meta_page(sb, blkaddr); if (IS_ERR(it->it.page)) return PTR_ERR(it->it.page); @@ -622,17 +638,17 @@ static int shared_listxattr(struct listxattr_iter *it) } ret = xattr_foreach(&it->it, &list_xattr_handlers, NULL); - if (ret < 0) + if (ret) break; } if (vi->xattr_shared_count) xattr_iter_end_final(&it->it); - return ret < 0 ? ret : it->buffer_ofs; + return ret ? ret : it->buffer_ofs; } ssize_t erofs_listxattr(struct dentry *dentry, - char *buffer, size_t buffer_size) + char *buffer, size_t buffer_size) { int ret; struct listxattr_iter it; @@ -656,3 +672,39 @@ ssize_t erofs_listxattr(struct dentry *dentry, return shared_listxattr(&it); } +#ifdef CONFIG_EROFS_FS_POSIX_ACL +struct posix_acl *erofs_get_acl(struct inode *inode, int type) +{ + struct posix_acl *acl; + int prefix, rc; + char *value = NULL; + + switch (type) { + case ACL_TYPE_ACCESS: + prefix = EROFS_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + prefix = EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT; + break; + default: + return ERR_PTR(-EINVAL); + } + + rc = erofs_getxattr(inode, prefix, "", NULL, 0); + if (rc > 0) { + value = kmalloc(rc, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + rc = erofs_getxattr(inode, prefix, "", value, rc); + } + + if (rc == -ENOATTR) + acl = NULL; + else if (rc < 0) + acl = ERR_PTR(rc); + else + acl = posix_acl_from_xattr(&init_user_ns, value, rc); + kfree(value); + return acl; +} +#endif diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h new file mode 100644 index 000000000000..366dcb400525 --- /dev/null +++ b/fs/erofs/xattr.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_XATTR_H +#define __EROFS_XATTR_H + +#include "internal.h" +#include +#include + +/* Attribute not found */ +#define ENOATTR ENODATA + +static inline unsigned int inlinexattr_header_size(struct inode *inode) +{ + return sizeof(struct erofs_xattr_ibody_header) + + sizeof(u32) * EROFS_I(inode)->xattr_shared_count; +} + +static inline erofs_blk_t xattrblock_addr(struct erofs_sb_info *sbi, + unsigned int xattr_id) +{ +#ifdef CONFIG_EROFS_FS_XATTR + return sbi->xattr_blkaddr + + xattr_id * sizeof(__u32) / EROFS_BLKSIZ; +#else + return 0; +#endif +} + +static inline unsigned int xattrblock_offset(struct erofs_sb_info *sbi, + unsigned int xattr_id) +{ + return (xattr_id * sizeof(__u32)) % EROFS_BLKSIZ; +} + +#ifdef CONFIG_EROFS_FS_XATTR +extern const struct xattr_handler erofs_xattr_user_handler; +extern const struct xattr_handler erofs_xattr_trusted_handler; +#ifdef CONFIG_EROFS_FS_SECURITY +extern const struct xattr_handler erofs_xattr_security_handler; +#endif + +static inline const struct xattr_handler *erofs_xattr_handler(unsigned int idx) +{ + static const struct xattr_handler *xattr_handler_map[] = { + [EROFS_XATTR_INDEX_USER] = &erofs_xattr_user_handler, +#ifdef CONFIG_EROFS_FS_POSIX_ACL + [EROFS_XATTR_INDEX_POSIX_ACL_ACCESS] = + &posix_acl_access_xattr_handler, + [EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT] = + &posix_acl_default_xattr_handler, +#endif + [EROFS_XATTR_INDEX_TRUSTED] = &erofs_xattr_trusted_handler, +#ifdef CONFIG_EROFS_FS_SECURITY + [EROFS_XATTR_INDEX_SECURITY] = &erofs_xattr_security_handler, +#endif + }; + + return idx && idx < ARRAY_SIZE(xattr_handler_map) ? + xattr_handler_map[idx] : NULL; +} + +extern const struct xattr_handler *erofs_xattr_handlers[]; + +int erofs_getxattr(struct inode *, int, const char *, void *, size_t); +ssize_t erofs_listxattr(struct dentry *, char *, size_t); +#else +static inline int erofs_getxattr(struct inode *inode, int index, + const char *name, void *buffer, + size_t buffer_size) +{ + return -EOPNOTSUPP; +} + +#define erofs_listxattr (NULL) +#define erofs_xattr_handlers (NULL) +#endif /* !CONFIG_EROFS_FS_XATTR */ + +#ifdef CONFIG_EROFS_FS_POSIX_ACL +struct posix_acl *erofs_get_acl(struct inode *inode, int type); +#else +#define erofs_get_acl (NULL) +#endif + +#endif + diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c new file mode 100644 index 000000000000..2d2cbfac500b --- /dev/null +++ b/fs/erofs/zdata.c @@ -0,0 +1,1479 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#include "zdata.h" +#include "compress.h" +#include + +#include + +/* + * since pclustersize is variable for big pcluster feature, introduce slab + * pools implementation for different pcluster sizes. + */ +struct z_erofs_pcluster_slab { + struct kmem_cache *slab; + unsigned int maxpages; + char name[48]; +}; + +#define _PCLP(n) { .maxpages = n } + +static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { + _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), + _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES) +}; + +static void z_erofs_destroy_pcluster_pool(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { + if (!pcluster_pool[i].slab) + continue; + kmem_cache_destroy(pcluster_pool[i].slab); + pcluster_pool[i].slab = NULL; + } +} + +static int z_erofs_create_pcluster_pool(void) +{ + struct z_erofs_pcluster_slab *pcs; + struct z_erofs_pcluster *a; + unsigned int size; + + for (pcs = pcluster_pool; + pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { + size = struct_size(a, compressed_pages, pcs->maxpages); + + sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages); + pcs->slab = kmem_cache_create(pcs->name, size, 0, + SLAB_RECLAIM_ACCOUNT, NULL); + if (pcs->slab) + continue; + + z_erofs_destroy_pcluster_pool(); + return -ENOMEM; + } + return 0; +} + +static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { + struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; + struct z_erofs_pcluster *pcl; + + if (nrpages > pcs->maxpages) + continue; + + pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); + if (!pcl) + return ERR_PTR(-ENOMEM); + pcl->pclusterpages = nrpages; + return pcl; + } + return ERR_PTR(-EINVAL); +} + +static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { + struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; + + if (pcl->pclusterpages > pcs->maxpages) + continue; + + kmem_cache_free(pcs->slab, pcl); + return; + } + DBG_BUGON(1); +} + +/* + * a compressed_pages[] placeholder in order to avoid + * being filled with file pages for in-place decompression. + */ +#define PAGE_UNALLOCATED ((void *)0x5F0E4B1D) + +/* how to allocate cached pages for a pcluster */ +enum z_erofs_cache_alloctype { + DONTALLOC, /* don't allocate any cached pages */ + DELAYEDALLOC, /* delayed allocation (at the time of submitting io) */ + /* + * try to use cached I/O if page allocation succeeds or fallback + * to in-place I/O instead to avoid any direct reclaim. + */ + TRYALLOC, +}; + +/* + * tagged pointer with 1-bit tag for all compressed pages + * tag 0 - the page is just found with an extra page reference + */ +typedef tagptr1_t compressed_page_t; + +#define tag_compressed_page_justfound(page) \ + tagptr_fold(compressed_page_t, page, 1) + +static struct workqueue_struct *z_erofs_workqueue __read_mostly; + +void z_erofs_exit_zip_subsystem(void) +{ + destroy_workqueue(z_erofs_workqueue); + z_erofs_destroy_pcluster_pool(); +} + +static inline int z_erofs_init_workqueue(void) +{ + const unsigned int onlinecpus = num_possible_cpus(); + + /* + * no need to spawn too many threads, limiting threads could minimum + * scheduling overhead, perhaps per-CPU threads should be better? + */ + z_erofs_workqueue = alloc_workqueue("erofs_unzipd", + WQ_UNBOUND | WQ_HIGHPRI, + onlinecpus + onlinecpus / 4); + return z_erofs_workqueue ? 0 : -ENOMEM; +} + +int __init z_erofs_init_zip_subsystem(void) +{ + int err = z_erofs_create_pcluster_pool(); + + if (err) + return err; + err = z_erofs_init_workqueue(); + if (err) + z_erofs_destroy_pcluster_pool(); + return err; +} + +enum z_erofs_collectmode { + COLLECT_SECONDARY, + COLLECT_PRIMARY, + /* + * The current collection was the tail of an exist chain, in addition + * that the previous processed chained collections are all decided to + * be hooked up to it. + * A new chain will be created for the remaining collections which are + * not processed yet, therefore different from COLLECT_PRIMARY_FOLLOWED, + * the next collection cannot reuse the whole page safely in + * the following scenario: + * ________________________________________________________________ + * | tail (partial) page | head (partial) page | + * | (belongs to the next cl) | (belongs to the current cl) | + * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________| + */ + COLLECT_PRIMARY_HOOKED, + /* + * a weak form of COLLECT_PRIMARY_FOLLOWED, the difference is that it + * could be dispatched into bypass queue later due to uptodated managed + * pages. All related online pages cannot be reused for inplace I/O (or + * pagevec) since it can be directly decoded without I/O submission. + */ + COLLECT_PRIMARY_FOLLOWED_NOINPLACE, + /* + * The current collection has been linked with the owned chain, and + * could also be linked with the remaining collections, which means + * if the processing page is the tail page of the collection, thus + * the current collection can safely use the whole page (since + * the previous collection is under control) for in-place I/O, as + * illustrated below: + * ________________________________________________________________ + * | tail (partial) page | head (partial) page | + * | (of the current cl) | (of the previous collection) | + * | PRIMARY_FOLLOWED or | | + * |_____PRIMARY_HOOKED___|____________PRIMARY_FOLLOWED____________| + * + * [ (*) the above page can be used as inplace I/O. ] + */ + COLLECT_PRIMARY_FOLLOWED, +}; + +struct z_erofs_collector { + struct z_erofs_pagevec_ctor vector; + + struct z_erofs_pcluster *pcl, *tailpcl; + struct z_erofs_collection *cl; + /* a pointer used to pick up inplace I/O pages */ + struct page **icpage_ptr; + z_erofs_next_pcluster_t owned_head; + + enum z_erofs_collectmode mode; +}; + +struct z_erofs_decompress_frontend { + struct inode *const inode; + + struct z_erofs_collector clt; + struct erofs_map_blocks map; + + bool readahead; + /* used for applying cache strategy on the fly */ + bool backmost; + erofs_off_t headoffset; +}; + +#define COLLECTOR_INIT() { \ + .owned_head = Z_EROFS_PCLUSTER_TAIL, \ + .mode = COLLECT_PRIMARY_FOLLOWED } + +#define DECOMPRESS_FRONTEND_INIT(__i) { \ + .inode = __i, .clt = COLLECTOR_INIT(), \ + .backmost = true, } + +static struct page *z_pagemap_global[Z_EROFS_VMAP_GLOBAL_PAGES]; +static DEFINE_MUTEX(z_pagemap_global_lock); + +static void preload_compressed_pages(struct z_erofs_collector *clt, + struct address_space *mc, + enum z_erofs_cache_alloctype type, + struct list_head *pagepool) +{ + struct z_erofs_pcluster *pcl = clt->pcl; + bool standalone = true; + gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | + __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; + struct page **pages; + pgoff_t index; + + if (clt->mode < COLLECT_PRIMARY_FOLLOWED) + return; + + pages = pcl->compressed_pages; + index = pcl->obj.index; + for (; index < pcl->obj.index + pcl->pclusterpages; ++index, ++pages) { + struct page *page; + compressed_page_t t; + struct page *newpage = NULL; + + /* the compressed page was loaded before */ + if (READ_ONCE(*pages)) + continue; + + page = find_get_page(mc, index); + + if (page) { + t = tag_compressed_page_justfound(page); + } else { + /* I/O is needed, no possible to decompress directly */ + standalone = false; + switch (type) { + case DELAYEDALLOC: + t = tagptr_init(compressed_page_t, + PAGE_UNALLOCATED); + break; + case TRYALLOC: + newpage = erofs_allocpage(pagepool, gfp); + if (!newpage) + continue; + set_page_private(newpage, + Z_EROFS_PREALLOCATED_PAGE); + t = tag_compressed_page_justfound(newpage); + break; + default: /* DONTALLOC */ + continue; + } + } + + if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t))) + continue; + + if (page) { + put_page(page); + } else if (newpage) { + set_page_private(newpage, 0); + list_add(&newpage->lru, pagepool); + } + } + + /* + * don't do inplace I/O if all compressed pages are available in + * managed cache since it can be moved to the bypass queue instead. + */ + if (standalone) + clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE; +} + +/* called by erofs_shrinker to get rid of all compressed_pages */ +int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, + struct erofs_workgroup *grp) +{ + struct z_erofs_pcluster *const pcl = + container_of(grp, struct z_erofs_pcluster, obj); + struct address_space *const mapping = MNGD_MAPPING(sbi); + int i; + + /* + * refcount of workgroup is now freezed as 1, + * therefore no need to worry about available decompression users. + */ + for (i = 0; i < pcl->pclusterpages; ++i) { + struct page *page = pcl->compressed_pages[i]; + + if (!page) + continue; + + /* block other users from reclaiming or migrating the page */ + if (!trylock_page(page)) + return -EBUSY; + + if (page->mapping != mapping) + continue; + + /* barrier is implied in the following 'unlock_page' */ + WRITE_ONCE(pcl->compressed_pages[i], NULL); + + set_page_private(page, 0); + ClearPagePrivate(page); + unlock_page(page); + put_page(page); + } + return 0; +} + +int erofs_try_to_free_cached_page(struct address_space *mapping, + struct page *page) +{ + struct z_erofs_pcluster *const pcl = (void *)page_private(page); + int ret = 0; /* 0 - busy */ + + if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) { + unsigned int i; + + for (i = 0; i < pcl->pclusterpages; ++i) { + if (pcl->compressed_pages[i] == page) { + WRITE_ONCE(pcl->compressed_pages[i], NULL); + ret = 1; + break; + } + } + erofs_workgroup_unfreeze(&pcl->obj, 1); + + if (ret) { + set_page_private(page, 0); + ClearPagePrivate(page); + put_page(page); + } + } + return ret; +} + +/* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */ +static bool z_erofs_try_inplace_io(struct z_erofs_collector *clt, + struct page *page) +{ + struct z_erofs_pcluster *const pcl = clt->pcl; + + while (clt->icpage_ptr > pcl->compressed_pages) + if (!cmpxchg(--clt->icpage_ptr, NULL, page)) + return true; + return false; +} + +/* callers must be with collection lock held */ +static int z_erofs_attach_page(struct z_erofs_collector *clt, + struct page *page, enum z_erofs_page_type type, + bool pvec_safereuse) +{ + int ret; + + /* give priority for inplaceio */ + if (clt->mode >= COLLECT_PRIMARY && + type == Z_EROFS_PAGE_TYPE_EXCLUSIVE && + z_erofs_try_inplace_io(clt, page)) + return 0; + + ret = z_erofs_pagevec_enqueue(&clt->vector, page, type, + pvec_safereuse); + clt->cl->vcnt += (unsigned int)ret; + return ret ? 0 : -EAGAIN; +} + +static void z_erofs_try_to_claim_pcluster(struct z_erofs_collector *clt) +{ + struct z_erofs_pcluster *pcl = clt->pcl; + z_erofs_next_pcluster_t *owned_head = &clt->owned_head; + + /* type 1, nil pcluster (this pcluster doesn't belong to any chain.) */ + if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL, + *owned_head) == Z_EROFS_PCLUSTER_NIL) { + *owned_head = &pcl->next; + /* so we can attach this pcluster to our submission chain. */ + clt->mode = COLLECT_PRIMARY_FOLLOWED; + return; + } + + /* + * type 2, link to the end of an existing open chain, be careful + * that its submission is controlled by the original attached chain. + */ + if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, + *owned_head) == Z_EROFS_PCLUSTER_TAIL) { + *owned_head = Z_EROFS_PCLUSTER_TAIL; + clt->mode = COLLECT_PRIMARY_HOOKED; + clt->tailpcl = NULL; + return; + } + /* type 3, it belongs to a chain, but it isn't the end of the chain */ + clt->mode = COLLECT_PRIMARY; +} + +static int z_erofs_lookup_collection(struct z_erofs_collector *clt, + struct inode *inode, + struct erofs_map_blocks *map) +{ + struct erofs_workgroup *grp; + struct z_erofs_pcluster *pcl; + struct z_erofs_collection *cl; + unsigned int length; + + grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT); + if (!grp) + return -ENOENT; + + pcl = container_of(grp, struct z_erofs_pcluster, obj); + if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) { + DBG_BUGON(1); + erofs_workgroup_put(grp); + return -EFSCORRUPTED; + } + + cl = z_erofs_primarycollection(pcl); + if (cl->pageofs != (map->m_la & ~PAGE_MASK)) { + DBG_BUGON(1); + erofs_workgroup_put(grp); + return -EFSCORRUPTED; + } + + length = READ_ONCE(pcl->length); + if (length & Z_EROFS_PCLUSTER_FULL_LENGTH) { + if ((map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) > length) { + DBG_BUGON(1); + erofs_workgroup_put(grp); + return -EFSCORRUPTED; + } + } else { + unsigned int llen = map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT; + + if (map->m_flags & EROFS_MAP_FULL_MAPPED) + llen |= Z_EROFS_PCLUSTER_FULL_LENGTH; + + while (llen > length && + length != cmpxchg_relaxed(&pcl->length, length, llen)) { + cpu_relax(); + length = READ_ONCE(pcl->length); + } + } + mutex_lock(&cl->lock); + /* used to check tail merging loop due to corrupted images */ + if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL) + clt->tailpcl = pcl; + clt->pcl = pcl; + z_erofs_try_to_claim_pcluster(clt); + clt->pcl = pcl; + clt->cl = cl; + return 0; +} + +static int z_erofs_register_collection(struct z_erofs_collector *clt, + struct inode *inode, + struct erofs_map_blocks *map) +{ + struct z_erofs_pcluster *pcl; + struct z_erofs_collection *cl; + int err; + + /* no available pcluster, let's allocate one */ + pcl = z_erofs_alloc_pcluster(map->m_plen >> PAGE_SHIFT); + if (IS_ERR(pcl)) + return PTR_ERR(pcl); + + atomic_set(&pcl->obj.refcount, 1); + pcl->obj.index = map->m_pa >> PAGE_SHIFT; + + pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) | + (map->m_flags & EROFS_MAP_FULL_MAPPED ? + Z_EROFS_PCLUSTER_FULL_LENGTH : 0); + + if (map->m_flags & EROFS_MAP_ZIPPED) + pcl->algorithmformat = Z_EROFS_COMPRESSION_LZ4; + else + pcl->algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; + + /* new pclusters should be claimed as type 1, primary and followed */ + pcl->next = clt->owned_head; + clt->mode = COLLECT_PRIMARY_FOLLOWED; + + cl = z_erofs_primarycollection(pcl); + cl->pageofs = map->m_la & ~PAGE_MASK; + + /* + * lock all primary followed works before visible to others + * and mutex_trylock *never* fails for a new pcluster. + */ + mutex_init(&cl->lock); + DBG_BUGON(!mutex_trylock(&cl->lock)); + + err = erofs_register_workgroup(inode->i_sb, &pcl->obj); + if (err) { + mutex_unlock(&cl->lock); + z_erofs_free_pcluster(pcl); + return -EAGAIN; + } + /* used to check tail merging loop due to corrupted images */ + if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL) + clt->tailpcl = pcl; + clt->owned_head = &pcl->next; + clt->pcl = pcl; + clt->cl = cl; + return 0; +} + +static int z_erofs_collector_begin(struct z_erofs_collector *clt, + struct inode *inode, + struct erofs_map_blocks *map) +{ + int ret; + + DBG_BUGON(clt->cl); + + /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous collection */ + DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL); + DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); + + if (!PAGE_ALIGNED(map->m_pa)) { + DBG_BUGON(1); + return -EINVAL; + } + +repeat: + ret = z_erofs_lookup_collection(clt, inode, map); + if (ret == -ENOENT) { + ret = z_erofs_register_collection(clt, inode, map); + + /* someone registered at the same time, give another try */ + if (ret == -EAGAIN) { + cond_resched(); + goto repeat; + } + } + + if (ret) + return ret; + + z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS, + clt->cl->pagevec, clt->cl->vcnt); + + /* since file-backed online pages are traversed in reverse order */ + clt->icpage_ptr = clt->pcl->compressed_pages + clt->pcl->pclusterpages; + return 0; +} + +/* + * keep in mind that no referenced pclusters will be freed + * only after a RCU grace period. + */ +static void z_erofs_rcu_callback(struct rcu_head *head) +{ + struct z_erofs_collection *const cl = + container_of(head, struct z_erofs_collection, rcu); + + z_erofs_free_pcluster(container_of(cl, struct z_erofs_pcluster, + primary_collection)); +} + +void erofs_workgroup_free_rcu(struct erofs_workgroup *grp) +{ + struct z_erofs_pcluster *const pcl = + container_of(grp, struct z_erofs_pcluster, obj); + struct z_erofs_collection *const cl = z_erofs_primarycollection(pcl); + + call_rcu(&cl->rcu, z_erofs_rcu_callback); +} + +static void z_erofs_collection_put(struct z_erofs_collection *cl) +{ + struct z_erofs_pcluster *const pcl = + container_of(cl, struct z_erofs_pcluster, primary_collection); + + erofs_workgroup_put(&pcl->obj); +} + +static bool z_erofs_collector_end(struct z_erofs_collector *clt) +{ + struct z_erofs_collection *cl = clt->cl; + + if (!cl) + return false; + + z_erofs_pagevec_ctor_exit(&clt->vector, false); + mutex_unlock(&cl->lock); + + /* + * if all pending pages are added, don't hold its reference + * any longer if the pcluster isn't hosted by ourselves. + */ + if (clt->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE) + z_erofs_collection_put(cl); + + clt->cl = NULL; + return true; +} + +static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe, + unsigned int cachestrategy, + erofs_off_t la) +{ + if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED) + return false; + + if (fe->backmost) + return true; + + return cachestrategy >= EROFS_ZIP_CACHE_READAROUND && + la < fe->headoffset; +} + +static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, + struct page *page, struct list_head *pagepool) +{ + struct inode *const inode = fe->inode; + struct erofs_sb_info *const sbi = EROFS_I_SB(inode); + struct erofs_map_blocks *const map = &fe->map; + struct z_erofs_collector *const clt = &fe->clt; + const loff_t offset = page_offset(page); + bool tight = true; + + enum z_erofs_cache_alloctype cache_strategy; + enum z_erofs_page_type page_type; + unsigned int cur, end, spiltted, index; + int err = 0; + + /* register locked file pages as online pages in pack */ + z_erofs_onlinepage_init(page); + + spiltted = 0; + end = PAGE_SIZE; +repeat: + cur = end - 1; + + /* lucky, within the range of the current map_blocks */ + if (offset + cur >= map->m_la && + offset + cur < map->m_la + map->m_llen) { + /* didn't get a valid collection previously (very rare) */ + if (!clt->cl) + goto restart_now; + goto hitted; + } + + /* go ahead the next map_blocks */ + erofs_dbg("%s: [out-of-range] pos %llu", __func__, offset + cur); + + if (z_erofs_collector_end(clt)) + fe->backmost = false; + + map->m_la = offset + cur; + map->m_llen = 0; + err = z_erofs_map_blocks_iter(inode, map, 0); + if (err) + goto err_out; + +restart_now: + if (!(map->m_flags & EROFS_MAP_MAPPED)) + goto hitted; + + err = z_erofs_collector_begin(clt, inode, map); + if (err) + goto err_out; + + /* preload all compressed pages (maybe downgrade role if necessary) */ + if (should_alloc_managed_pages(fe, sbi->cache_strategy, map->m_la)) + cache_strategy = TRYALLOC; + else + cache_strategy = DONTALLOC; + + preload_compressed_pages(clt, MNGD_MAPPING(sbi), + cache_strategy, pagepool); + +hitted: + /* + * Ensure the current partial page belongs to this submit chain rather + * than other concurrent submit chains or the noio(bypass) chain since + * those chains are handled asynchronously thus the page cannot be used + * for inplace I/O or pagevec (should be processed in strict order.) + */ + tight &= (clt->mode >= COLLECT_PRIMARY_HOOKED && + clt->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE); + + cur = end - min_t(unsigned int, offset + end - map->m_la, end); + if (!(map->m_flags & EROFS_MAP_MAPPED)) { + zero_user_segment(page, cur, end); + goto next_part; + } + + /* let's derive page type */ + page_type = cur ? Z_EROFS_VLE_PAGE_TYPE_HEAD : + (!spiltted ? Z_EROFS_PAGE_TYPE_EXCLUSIVE : + (tight ? Z_EROFS_PAGE_TYPE_EXCLUSIVE : + Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED)); + + if (cur) + tight &= (clt->mode >= COLLECT_PRIMARY_FOLLOWED); + +retry: + err = z_erofs_attach_page(clt, page, page_type, + clt->mode >= COLLECT_PRIMARY_FOLLOWED); + /* should allocate an additional short-lived page for pagevec */ + if (err == -EAGAIN) { + struct page *const newpage = + alloc_page(GFP_NOFS | __GFP_NOFAIL); + + set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE); + err = z_erofs_attach_page(clt, newpage, + Z_EROFS_PAGE_TYPE_EXCLUSIVE, true); + if (!err) + goto retry; + } + + if (err) + goto err_out; + + index = page->index - (map->m_la >> PAGE_SHIFT); + + z_erofs_onlinepage_fixup(page, index, true); + + /* bump up the number of spiltted parts of a page */ + ++spiltted; + /* also update nr_pages */ + clt->cl->nr_pages = max_t(pgoff_t, clt->cl->nr_pages, index + 1); +next_part: + /* can be used for verification */ + map->m_llen = offset + cur - map->m_la; + + end = cur; + if (end > 0) + goto repeat; + +out: + z_erofs_onlinepage_endio(page); + + erofs_dbg("%s, finish page: %pK spiltted: %u map->m_llen %llu", + __func__, page, spiltted, map->m_llen); + return err; + + /* if some error occurred while processing this page */ +err_out: + SetPageError(page); + goto out; +} + +static void z_erofs_decompressqueue_work(struct work_struct *work); +static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, + bool sync, int bios) +{ + struct erofs_sb_info *const sbi = EROFS_SB(io->sb); + + /* wake up the caller thread for sync decompression */ + if (sync) { + unsigned long flags; + + spin_lock_irqsave(&io->u.wait.lock, flags); + if (!atomic_add_return(bios, &io->pending_bios)) + wake_up_locked(&io->u.wait); + spin_unlock_irqrestore(&io->u.wait.lock, flags); + return; + } + + if (atomic_add_return(bios, &io->pending_bios)) + return; + /* Use workqueue and sync decompression for atomic contexts only */ + if (in_atomic() || irqs_disabled()) { + queue_work(z_erofs_workqueue, &io->u.work); + sbi->readahead_sync_decompress = true; + return; + } + z_erofs_decompressqueue_work(&io->u.work); +} + +static bool z_erofs_page_is_invalidated(struct page *page) +{ + return !page->mapping && !z_erofs_is_shortlived_page(page); +} + +static void z_erofs_decompressqueue_endio(struct bio *bio) +{ + tagptr1_t t = tagptr_init(tagptr1_t, bio->bi_private); + struct z_erofs_decompressqueue *q = tagptr_unfold_ptr(t); + blk_status_t err = bio->bi_status; + struct bio_vec *bvec; + unsigned int i; + + bio_for_each_segment_all(bvec, bio, i) { + struct page *page = bvec->bv_page; + + DBG_BUGON(PageUptodate(page)); + DBG_BUGON(z_erofs_page_is_invalidated(page)); + + if (err) + SetPageError(page); + + if (erofs_page_is_managed(EROFS_SB(q->sb), page)) { + if (!err) + SetPageUptodate(page); + unlock_page(page); + } + } + z_erofs_decompress_kickoff(q, tagptr_unfold_tags(t), -1); + bio_put(bio); +} + +static int z_erofs_decompress_pcluster(struct super_block *sb, + struct z_erofs_pcluster *pcl, + struct list_head *pagepool) +{ + struct erofs_sb_info *const sbi = EROFS_SB(sb); + struct z_erofs_pagevec_ctor ctor; + unsigned int i, inputsize, outputsize, llen, nr_pages; + struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES]; + struct page **pages, **compressed_pages, *page; + + enum z_erofs_page_type page_type; + bool overlapped, partial; + struct z_erofs_collection *cl; + int err; + + might_sleep(); + cl = z_erofs_primarycollection(pcl); + DBG_BUGON(!READ_ONCE(cl->nr_pages)); + + mutex_lock(&cl->lock); + nr_pages = cl->nr_pages; + + if (nr_pages <= Z_EROFS_VMAP_ONSTACK_PAGES) { + pages = pages_onstack; + } else if (nr_pages <= Z_EROFS_VMAP_GLOBAL_PAGES && + mutex_trylock(&z_pagemap_global_lock)) { + pages = z_pagemap_global; + } else { + gfp_t gfp_flags = GFP_KERNEL; + + if (nr_pages > Z_EROFS_VMAP_GLOBAL_PAGES) + gfp_flags |= __GFP_NOFAIL; + + pages = kvmalloc_array(nr_pages, sizeof(struct page *), + gfp_flags); + + /* fallback to global pagemap for the lowmem scenario */ + if (!pages) { + mutex_lock(&z_pagemap_global_lock); + pages = z_pagemap_global; + } + } + + for (i = 0; i < nr_pages; ++i) + pages[i] = NULL; + + err = 0; + z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS, + cl->pagevec, 0); + + for (i = 0; i < cl->vcnt; ++i) { + unsigned int pagenr; + + page = z_erofs_pagevec_dequeue(&ctor, &page_type); + + /* all pages in pagevec ought to be valid */ + DBG_BUGON(!page); + DBG_BUGON(z_erofs_page_is_invalidated(page)); + + if (z_erofs_put_shortlivedpage(pagepool, page)) + continue; + + if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD) + pagenr = 0; + else + pagenr = z_erofs_onlinepage_index(page); + + DBG_BUGON(pagenr >= nr_pages); + + /* + * currently EROFS doesn't support multiref(dedup), + * so here erroring out one multiref page. + */ + if (pages[pagenr]) { + DBG_BUGON(1); + SetPageError(pages[pagenr]); + z_erofs_onlinepage_endio(pages[pagenr]); + err = -EFSCORRUPTED; + } + pages[pagenr] = page; + } + z_erofs_pagevec_ctor_exit(&ctor, true); + + overlapped = false; + compressed_pages = pcl->compressed_pages; + + for (i = 0; i < pcl->pclusterpages; ++i) { + unsigned int pagenr; + + page = compressed_pages[i]; + + /* all compressed pages ought to be valid */ + DBG_BUGON(!page); + DBG_BUGON(z_erofs_page_is_invalidated(page)); + + if (!z_erofs_is_shortlived_page(page)) { + if (erofs_page_is_managed(sbi, page)) { + if (!PageUptodate(page)) + err = -EIO; + continue; + } + + /* + * only if non-head page can be selected + * for inplace decompression + */ + pagenr = z_erofs_onlinepage_index(page); + + DBG_BUGON(pagenr >= nr_pages); + if (pages[pagenr]) { + DBG_BUGON(1); + SetPageError(pages[pagenr]); + z_erofs_onlinepage_endio(pages[pagenr]); + err = -EFSCORRUPTED; + } + pages[pagenr] = page; + + overlapped = true; + } + + /* PG_error needs checking for all non-managed pages */ + if (PageError(page)) { + DBG_BUGON(PageUptodate(page)); + err = -EIO; + } + } + + if (err) + goto out; + + llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT; + if (nr_pages << PAGE_SHIFT >= cl->pageofs + llen) { + outputsize = llen; + partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH); + } else { + outputsize = (nr_pages << PAGE_SHIFT) - cl->pageofs; + partial = true; + } + + inputsize = pcl->pclusterpages * PAGE_SIZE; + err = z_erofs_decompress(&(struct z_erofs_decompress_req) { + .sb = sb, + .in = compressed_pages, + .out = pages, + .pageofs_out = cl->pageofs, + .inputsize = inputsize, + .outputsize = outputsize, + .alg = pcl->algorithmformat, + .inplace_io = overlapped, + .partial_decoding = partial + }, pagepool); + +out: + /* must handle all compressed pages before ending pages */ + for (i = 0; i < pcl->pclusterpages; ++i) { + page = compressed_pages[i]; + + if (erofs_page_is_managed(sbi, page)) + continue; + + /* recycle all individual short-lived pages */ + (void)z_erofs_put_shortlivedpage(pagepool, page); + + WRITE_ONCE(compressed_pages[i], NULL); + } + + for (i = 0; i < nr_pages; ++i) { + page = pages[i]; + if (!page) + continue; + + DBG_BUGON(z_erofs_page_is_invalidated(page)); + + /* recycle all individual short-lived pages */ + if (z_erofs_put_shortlivedpage(pagepool, page)) + continue; + + if (err < 0) + SetPageError(page); + + z_erofs_onlinepage_endio(page); + } + + if (pages == z_pagemap_global) + mutex_unlock(&z_pagemap_global_lock); + else if (pages != pages_onstack) + kvfree(pages); + + cl->nr_pages = 0; + cl->vcnt = 0; + + /* all cl locks MUST be taken before the following line */ + WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL); + + /* all cl locks SHOULD be released right now */ + mutex_unlock(&cl->lock); + + z_erofs_collection_put(cl); + return err; +} + +static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, + struct list_head *pagepool) +{ + z_erofs_next_pcluster_t owned = io->head; + + while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) { + struct z_erofs_pcluster *pcl; + + /* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */ + DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL); + + /* no possible that 'owned' equals NULL */ + DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL); + + pcl = container_of(owned, struct z_erofs_pcluster, next); + owned = READ_ONCE(pcl->next); + + z_erofs_decompress_pcluster(io->sb, pcl, pagepool); + } +} + +static void z_erofs_decompressqueue_work(struct work_struct *work) +{ + struct z_erofs_decompressqueue *bgq = + container_of(work, struct z_erofs_decompressqueue, u.work); + LIST_HEAD(pagepool); + + DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL_CLOSED); + z_erofs_decompress_queue(bgq, &pagepool); + + put_pages_list(&pagepool); + kvfree(bgq); +} + +static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, + unsigned int nr, + struct list_head *pagepool, + struct address_space *mc, + gfp_t gfp) +{ + const pgoff_t index = pcl->obj.index; + bool tocache = false; + + struct address_space *mapping; + struct page *oldpage, *page; + + compressed_page_t t; + int justfound; + +repeat: + page = READ_ONCE(pcl->compressed_pages[nr]); + oldpage = page; + + if (!page) + goto out_allocpage; + + /* + * the cached page has not been allocated and + * an placeholder is out there, prepare it now. + */ + if (page == PAGE_UNALLOCATED) { + tocache = true; + goto out_allocpage; + } + + /* process the target tagged pointer */ + t = tagptr_init(compressed_page_t, page); + justfound = tagptr_unfold_tags(t); + page = tagptr_unfold_ptr(t); + + /* + * preallocated cached pages, which is used to avoid direct reclaim + * otherwise, it will go inplace I/O path instead. + */ + if (page->private == Z_EROFS_PREALLOCATED_PAGE) { + WRITE_ONCE(pcl->compressed_pages[nr], page); + set_page_private(page, 0); + tocache = true; + goto out_tocache; + } + mapping = READ_ONCE(page->mapping); + + /* + * file-backed online pages in plcuster are all locked steady, + * therefore it is impossible for `mapping' to be NULL. + */ + if (mapping && mapping != mc) + /* ought to be unmanaged pages */ + goto out; + + /* directly return for shortlived page as well */ + if (z_erofs_is_shortlived_page(page)) + goto out; + + lock_page(page); + + /* only true if page reclaim goes wrong, should never happen */ + DBG_BUGON(justfound && PagePrivate(page)); + + /* the page is still in manage cache */ + if (page->mapping == mc) { + WRITE_ONCE(pcl->compressed_pages[nr], page); + + ClearPageError(page); + if (!PagePrivate(page)) { + /* + * impossible to be !PagePrivate(page) for + * the current restriction as well if + * the page is already in compressed_pages[]. + */ + DBG_BUGON(!justfound); + + justfound = 0; + set_page_private(page, (unsigned long)pcl); + SetPagePrivate(page); + } + + /* no need to submit io if it is already up-to-date */ + if (PageUptodate(page)) { + unlock_page(page); + page = NULL; + } + goto out; + } + + /* + * the managed page has been truncated, it's unsafe to + * reuse this one, let's allocate a new cache-managed page. + */ + DBG_BUGON(page->mapping); + DBG_BUGON(!justfound); + + tocache = true; + unlock_page(page); + put_page(page); +out_allocpage: + page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL); + if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) { + list_add(&page->lru, pagepool); + cond_resched(); + goto repeat; + } +out_tocache: + if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { + /* turn into temporary page if fails (1 ref) */ + set_page_private(page, Z_EROFS_SHORTLIVED_PAGE); + goto out; + } + set_page_private(page, (unsigned long)pcl); + SetPagePrivate(page); + +out: /* the only exit (for tracing and debugging) */ + return page; +} + +static struct z_erofs_decompressqueue * +jobqueue_init(struct super_block *sb, + struct z_erofs_decompressqueue *fgq, bool *fg) +{ + struct z_erofs_decompressqueue *q; + + if (fg && !*fg) { + q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN); + if (!q) { + *fg = true; + goto fg_out; + } + INIT_WORK(&q->u.work, z_erofs_decompressqueue_work); + } else { +fg_out: + q = fgq; + init_waitqueue_head(&fgq->u.wait); + atomic_set(&fgq->pending_bios, 0); + } + q->sb = sb; + q->head = Z_EROFS_PCLUSTER_TAIL_CLOSED; + return q; +} + +/* define decompression jobqueue types */ +enum { + JQ_BYPASS, + JQ_SUBMIT, + NR_JOBQUEUES, +}; + +static void *jobqueueset_init(struct super_block *sb, + struct z_erofs_decompressqueue *q[], + struct z_erofs_decompressqueue *fgq, bool *fg) +{ + /* + * if managed cache is enabled, bypass jobqueue is needed, + * no need to read from device for all pclusters in this queue. + */ + q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL); + q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, fg); + + return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], *fg)); +} + +static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, + z_erofs_next_pcluster_t qtail[], + z_erofs_next_pcluster_t owned_head) +{ + z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT]; + z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS]; + + DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); + if (owned_head == Z_EROFS_PCLUSTER_TAIL) + owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED; + + WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED); + + WRITE_ONCE(*submit_qtail, owned_head); + WRITE_ONCE(*bypass_qtail, &pcl->next); + + qtail[JQ_BYPASS] = &pcl->next; +} + +static void z_erofs_submit_queue(struct super_block *sb, + struct z_erofs_decompress_frontend *f, + struct list_head *pagepool, + struct z_erofs_decompressqueue *fgq, + bool *force_fg) +{ + struct erofs_sb_info *const sbi = EROFS_SB(sb); + z_erofs_next_pcluster_t qtail[NR_JOBQUEUES]; + struct z_erofs_decompressqueue *q[NR_JOBQUEUES]; + void *bi_private; + z_erofs_next_pcluster_t owned_head = f->clt.owned_head; + /* since bio will be NULL, no need to initialize last_index */ + pgoff_t last_index; + unsigned int nr_bios = 0; + struct bio *bio = NULL; + + bi_private = jobqueueset_init(sb, q, fgq, force_fg); + qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; + qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; + + /* by default, all need io submission */ + q[JQ_SUBMIT]->head = owned_head; + + do { + struct z_erofs_pcluster *pcl; + pgoff_t cur, end; + unsigned int i = 0; + bool bypass = true; + + /* no possible 'owned_head' equals the following */ + DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); + DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL); + + pcl = container_of(owned_head, struct z_erofs_pcluster, next); + + cur = pcl->obj.index; + end = cur + pcl->pclusterpages; + + /* close the main owned chain at first */ + owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, + Z_EROFS_PCLUSTER_TAIL_CLOSED); + + do { + struct page *page; + + page = pickup_page_for_submission(pcl, i++, pagepool, + MNGD_MAPPING(sbi), + GFP_NOFS); + if (!page) + continue; + + if (bio && cur != last_index + 1) { +submit_bio_retry: + submit_bio(bio); + bio = NULL; + } + + if (!bio) { + bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); + + bio->bi_end_io = z_erofs_decompressqueue_endio; + bio_set_dev(bio, sb->s_bdev); + bio->bi_iter.bi_sector = (sector_t)cur << + LOG_SECTORS_PER_BLOCK; + bio->bi_private = bi_private; + bio->bi_opf = REQ_OP_READ; + if (f->readahead) + bio->bi_opf |= REQ_RAHEAD; + ++nr_bios; + } + + if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) + goto submit_bio_retry; + + last_index = cur; + bypass = false; + } while (++cur < end); + + if (!bypass) + qtail[JQ_SUBMIT] = &pcl->next; + else + move_to_bypass_jobqueue(pcl, qtail, owned_head); + } while (owned_head != Z_EROFS_PCLUSTER_TAIL); + + if (bio) + submit_bio(bio); + + /* + * although background is preferred, no one is pending for submission. + * don't issue workqueue for decompression but drop it directly instead. + */ + if (!*force_fg && !nr_bios) { + kvfree(q[JQ_SUBMIT]); + return; + } + z_erofs_decompress_kickoff(q[JQ_SUBMIT], *force_fg, nr_bios); +} + +static void z_erofs_runqueue(struct super_block *sb, + struct z_erofs_decompress_frontend *f, + struct list_head *pagepool, bool force_fg) +{ + struct z_erofs_decompressqueue io[NR_JOBQUEUES]; + + if (f->clt.owned_head == Z_EROFS_PCLUSTER_TAIL) + return; + z_erofs_submit_queue(sb, f, pagepool, io, &force_fg); + + /* handle bypass queue (no i/o pclusters) immediately */ + z_erofs_decompress_queue(&io[JQ_BYPASS], pagepool); + + if (!force_fg) + return; + + /* wait until all bios are completed */ + io_wait_event(io[JQ_SUBMIT].u.wait, + !atomic_read(&io[JQ_SUBMIT].pending_bios)); + + /* handle synchronous decompress queue in the caller context */ + z_erofs_decompress_queue(&io[JQ_SUBMIT], pagepool); +} + +static int z_erofs_readpage(struct file *file, struct page *page) +{ + struct inode *const inode = page->mapping->host; + struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); + int err; + LIST_HEAD(pagepool); + + trace_erofs_readpage(page, false); + + f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT; + + err = z_erofs_do_read_page(&f, page, &pagepool); + (void)z_erofs_collector_end(&f.clt); + + /* if some compressed cluster ready, need submit them anyway */ + z_erofs_runqueue(inode->i_sb, &f, &pagepool, true); + + if (err) + erofs_err(inode->i_sb, "failed to read, err [%d]", err); + + if (f.map.mpage) + put_page(f.map.mpage); + + /* clean up the remaining free pages */ + put_pages_list(&pagepool); + return err; +} + +static int z_erofs_readpages(struct file *filp, struct address_space *mapping, + struct list_head *pages, unsigned int nr_pages) +{ + struct inode *const inode = mapping->host; + struct erofs_sb_info *const sbi = EROFS_I_SB(inode); + + bool sync = (sbi->readahead_sync_decompress && + nr_pages <= sbi->max_sync_decompress_pages); + struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); + gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL); + struct page *head = NULL; + LIST_HEAD(pagepool); + + trace_erofs_readpages(mapping->host, lru_to_page(pages), + nr_pages, false); + + f.headoffset = (erofs_off_t)lru_to_page(pages)->index << PAGE_SHIFT; + + for (; nr_pages; --nr_pages) { + struct page *page = lru_to_page(pages); + + prefetchw(&page->flags); + list_del(&page->lru); + + /* + * A pure asynchronous readahead is indicated if + * a PG_readahead marked page is hitted at first. + * Let's also do asynchronous decompression for this case. + */ + sync &= !(PageReadahead(page) && !head); + + if (add_to_page_cache_lru(page, mapping, page->index, gfp)) { + list_add(&page->lru, &pagepool); + continue; + } + + set_page_private(page, (unsigned long)head); + head = page; + } + + while (head) { + struct page *page = head; + int err; + + /* traversal in reverse order */ + head = (void *)page_private(page); + + err = z_erofs_do_read_page(&f, page, &pagepool); + if (err) + erofs_err(inode->i_sb, + "readahead error at page %lu @ nid %llu", + page->index, EROFS_I(inode)->nid); + put_page(page); + } + + (void)z_erofs_collector_end(&f.clt); + + z_erofs_runqueue(inode->i_sb, &f, &pagepool, sync); + + if (f.map.mpage) + put_page(f.map.mpage); + + /* clean up the remaining free pages */ + put_pages_list(&pagepool); + return 0; +} + +const struct address_space_operations z_erofs_aops = { + .readpage = z_erofs_readpage, + .readpages = z_erofs_readpages, +}; diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h new file mode 100644 index 000000000000..3a008f1b9f78 --- /dev/null +++ b/fs/erofs/zdata.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_FS_ZDATA_H +#define __EROFS_FS_ZDATA_H + +#include "internal.h" +#include "zpvec.h" + +#define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE) +#define Z_EROFS_NR_INLINE_PAGEVECS 3 + +/* + * Structure fields follow one of the following exclusion rules. + * + * I: Modifiable by initialization/destruction paths and read-only + * for everyone else; + * + * L: Field should be protected by pageset lock; + * + * A: Field should be accessed / updated in atomic for parallelized code. + */ +struct z_erofs_collection { + struct mutex lock; + + /* I: page offset of start position of decompression */ + unsigned short pageofs; + + /* L: maximum relative page index in pagevec[] */ + unsigned short nr_pages; + + /* L: total number of pages in pagevec[] */ + unsigned int vcnt; + + union { + /* L: inline a certain number of pagevecs for bootstrap */ + erofs_vtptr_t pagevec[Z_EROFS_NR_INLINE_PAGEVECS]; + + /* I: can be used to free the pcluster by RCU. */ + struct rcu_head rcu; + }; +}; + +#define Z_EROFS_PCLUSTER_FULL_LENGTH 0x00000001 +#define Z_EROFS_PCLUSTER_LENGTH_BIT 1 + +/* + * let's leave a type here in case of introducing + * another tagged pointer later. + */ +typedef void *z_erofs_next_pcluster_t; + +struct z_erofs_pcluster { + struct erofs_workgroup obj; + struct z_erofs_collection primary_collection; + + /* A: point to next chained pcluster or TAILs */ + z_erofs_next_pcluster_t next; + + /* A: lower limit of decompressed length and if full length or not */ + unsigned int length; + + /* I: physical cluster size in pages */ + unsigned short pclusterpages; + + /* I: compression algorithm format */ + unsigned char algorithmformat; + + /* A: compressed pages (can be cached or inplaced pages) */ + struct page *compressed_pages[]; +}; + +#define z_erofs_primarycollection(pcluster) (&(pcluster)->primary_collection) + +/* let's avoid the valid 32-bit kernel addresses */ + +/* the chained workgroup has't submitted io (still open) */ +#define Z_EROFS_PCLUSTER_TAIL ((void *)0x5F0ECAFE) +/* the chained workgroup has already submitted io */ +#define Z_EROFS_PCLUSTER_TAIL_CLOSED ((void *)0x5F0EDEAD) + +#define Z_EROFS_PCLUSTER_NIL (NULL) + +struct z_erofs_decompressqueue { + struct super_block *sb; + atomic_t pending_bios; + z_erofs_next_pcluster_t head; + + union { + wait_queue_head_t wait; + struct work_struct work; + } u; +}; + +#define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping) +static inline bool erofs_page_is_managed(const struct erofs_sb_info *sbi, + struct page *page) +{ + return page->mapping == MNGD_MAPPING(sbi); +} + +#define Z_EROFS_ONLINEPAGE_COUNT_BITS 2 +#define Z_EROFS_ONLINEPAGE_COUNT_MASK ((1 << Z_EROFS_ONLINEPAGE_COUNT_BITS) - 1) +#define Z_EROFS_ONLINEPAGE_INDEX_SHIFT (Z_EROFS_ONLINEPAGE_COUNT_BITS) + +/* + * waiters (aka. ongoing_packs): # to unlock the page + * sub-index: 0 - for partial page, >= 1 full page sub-index + */ +typedef atomic_t z_erofs_onlinepage_t; + +/* type punning */ +union z_erofs_onlinepage_converter { + z_erofs_onlinepage_t *o; + unsigned long *v; +}; + +static inline unsigned int z_erofs_onlinepage_index(struct page *page) +{ + union z_erofs_onlinepage_converter u; + + DBG_BUGON(!PagePrivate(page)); + u.v = &page_private(page); + + return atomic_read(u.o) >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT; +} + +static inline void z_erofs_onlinepage_init(struct page *page) +{ + union { + z_erofs_onlinepage_t o; + unsigned long v; + /* keep from being unlocked in advance */ + } u = { .o = ATOMIC_INIT(1) }; + + set_page_private(page, u.v); + smp_wmb(); + SetPagePrivate(page); +} + +static inline void z_erofs_onlinepage_fixup(struct page *page, + uintptr_t index, bool down) +{ + union z_erofs_onlinepage_converter u = { .v = &page_private(page) }; + int orig, orig_index, val; + +repeat: + orig = atomic_read(u.o); + orig_index = orig >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT; + if (orig_index) { + if (!index) + return; + + DBG_BUGON(orig_index != index); + } + + val = (index << Z_EROFS_ONLINEPAGE_INDEX_SHIFT) | + ((orig & Z_EROFS_ONLINEPAGE_COUNT_MASK) + (unsigned int)down); + if (atomic_cmpxchg(u.o, orig, val) != orig) + goto repeat; +} + +static inline void z_erofs_onlinepage_endio(struct page *page) +{ + union z_erofs_onlinepage_converter u; + unsigned int v; + + DBG_BUGON(!PagePrivate(page)); + u.v = &page_private(page); + + v = atomic_dec_return(u.o); + if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) { + set_page_private(page, 0); + ClearPagePrivate(page); + if (!PageError(page)) + SetPageUptodate(page); + unlock_page(page); + } + erofs_dbg("%s, page %p value %x", __func__, page, atomic_read(u.o)); +} + +#define Z_EROFS_VMAP_ONSTACK_PAGES \ + min_t(unsigned int, THREAD_SIZE / 8 / sizeof(struct page *), 96U) +#define Z_EROFS_VMAP_GLOBAL_PAGES 2048 + +#endif + diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c new file mode 100644 index 000000000000..a7da7ed4bda7 --- /dev/null +++ b/fs/erofs/zmap.c @@ -0,0 +1,598 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#include "internal.h" +#include +#include + +int z_erofs_fill_inode(struct inode *inode) +{ + struct erofs_inode *const vi = EROFS_I(inode); + struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); + + if (!erofs_sb_has_big_pcluster(sbi) && + vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { + vi->z_advise = 0; + vi->z_algorithmtype[0] = 0; + vi->z_algorithmtype[1] = 0; + vi->z_logical_clusterbits = LOG_BLOCK_SIZE; + set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); + } + inode->i_mapping->a_ops = &z_erofs_aops; + return 0; +} + +static int z_erofs_fill_inode_lazy(struct inode *inode) +{ + struct erofs_inode *const vi = EROFS_I(inode); + struct super_block *const sb = inode->i_sb; + int err; + erofs_off_t pos; + struct page *page; + void *kaddr; + struct z_erofs_map_header *h; + + if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { + /* + * paired with smp_mb() at the end of the function to ensure + * fields will only be observed after the bit is set. + */ + smp_mb(); + return 0; + } + + if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) + return -ERESTARTSYS; + + err = 0; + if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) + goto out_unlock; + + DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && + vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); + + pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize + + vi->xattr_isize, 8); + page = erofs_get_meta_page(sb, erofs_blknr(pos)); + if (IS_ERR(page)) { + err = PTR_ERR(page); + goto out_unlock; + } + + kaddr = kmap_atomic(page); + + h = kaddr + erofs_blkoff(pos); + vi->z_advise = le16_to_cpu(h->h_advise); + vi->z_algorithmtype[0] = h->h_algorithmtype & 15; + vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; + + if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown compression format %u for nid %llu, please upgrade kernel", + vi->z_algorithmtype[0], vi->nid); + err = -EOPNOTSUPP; + goto unmap_done; + } + + vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7); + if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && + vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { + erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", + vi->nid); + err = -EFSCORRUPTED; + goto unmap_done; + } + if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION && + !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ + !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { + erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", + vi->nid); + err = -EFSCORRUPTED; + goto unmap_done; + } + /* paired with smp_mb() at the beginning of the function */ + smp_mb(); + set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); +unmap_done: + kunmap_atomic(kaddr); + unlock_page(page); + put_page(page); +out_unlock: + clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); + return err; +} + +struct z_erofs_maprecorder { + struct inode *inode; + struct erofs_map_blocks *map; + void *kaddr; + + unsigned long lcn; + /* compression extent information gathered */ + u8 type; + u16 clusterofs; + u16 delta[2]; + erofs_blk_t pblk, compressedlcs; +}; + +static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, + erofs_blk_t eblk) +{ + struct super_block *const sb = m->inode->i_sb; + struct erofs_map_blocks *const map = m->map; + struct page *mpage = map->mpage; + + if (mpage) { + if (mpage->index == eblk) { + if (!m->kaddr) + m->kaddr = kmap_atomic(mpage); + return 0; + } + + if (m->kaddr) { + kunmap_atomic(m->kaddr); + m->kaddr = NULL; + } + put_page(mpage); + } + + mpage = erofs_get_meta_page(sb, eblk); + if (IS_ERR(mpage)) { + map->mpage = NULL; + return PTR_ERR(mpage); + } + m->kaddr = kmap_atomic(mpage); + unlock_page(mpage); + map->mpage = mpage; + return 0; +} + +static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, + unsigned long lcn) +{ + struct inode *const inode = m->inode; + struct erofs_inode *const vi = EROFS_I(inode); + const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid); + const erofs_off_t pos = + Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize + + vi->xattr_isize) + + lcn * sizeof(struct z_erofs_vle_decompressed_index); + struct z_erofs_vle_decompressed_index *di; + unsigned int advise, type; + int err; + + err = z_erofs_reload_indexes(m, erofs_blknr(pos)); + if (err) + return err; + + m->lcn = lcn; + di = m->kaddr + erofs_blkoff(pos); + + advise = le16_to_cpu(di->di_advise); + type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) & + ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1); + switch (type) { + case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: + m->clusterofs = 1 << vi->z_logical_clusterbits; + m->delta[0] = le16_to_cpu(di->di_u.delta[0]); + if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) { + if (!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + m->compressedlcs = m->delta[0] & + ~Z_EROFS_VLE_DI_D0_CBLKCNT; + m->delta[0] = 1; + } + m->delta[1] = le16_to_cpu(di->di_u.delta[1]); + break; + case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: + case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: + m->clusterofs = le16_to_cpu(di->di_clusterofs); + m->pblk = le32_to_cpu(di->di_u.blkaddr); + break; + default: + DBG_BUGON(1); + return -EOPNOTSUPP; + } + m->type = type; + return 0; +} + +static unsigned int decode_compactedbits(unsigned int lobits, + unsigned int lomask, + u8 *in, unsigned int pos, u8 *type) +{ + const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); + const unsigned int lo = v & lomask; + + *type = (v >> lobits) & 3; + return lo; +} + +static int unpack_compacted_index(struct z_erofs_maprecorder *m, + unsigned int amortizedshift, + unsigned int eofs) +{ + struct erofs_inode *const vi = EROFS_I(m->inode); + const unsigned int lclusterbits = vi->z_logical_clusterbits; + const unsigned int lomask = (1 << lclusterbits) - 1; + unsigned int vcnt, base, lo, encodebits, nblk; + int i; + u8 *in, type; + bool big_pcluster; + + if (1 << amortizedshift == 4) + vcnt = 2; + else if (1 << amortizedshift == 2 && lclusterbits == 12) + vcnt = 16; + else + return -EOPNOTSUPP; + + big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; + encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; + base = round_down(eofs, vcnt << amortizedshift); + in = m->kaddr + base; + + i = (eofs - base) >> amortizedshift; + + lo = decode_compactedbits(lclusterbits, lomask, + in, encodebits * i, &type); + m->type = type; + if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1 << lclusterbits; + if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { + if (!big_pcluster) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + m->compressedlcs = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; + m->delta[0] = 1; + return 0; + } else if (i + 1 != (int)vcnt) { + m->delta[0] = lo; + return 0; + } + /* + * since the last lcluster in the pack is special, + * of which lo saves delta[1] rather than delta[0]. + * Hence, get delta[0] by the previous lcluster indirectly. + */ + lo = decode_compactedbits(lclusterbits, lomask, + in, encodebits * (i - 1), &type); + if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) + lo = 0; + else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) + lo = 1; + m->delta[0] = lo + 1; + return 0; + } + m->clusterofs = lo; + m->delta[0] = 0; + /* figout out blkaddr (pblk) for HEAD lclusters */ + if (!big_pcluster) { + nblk = 1; + while (i > 0) { + --i; + lo = decode_compactedbits(lclusterbits, lomask, + in, encodebits * i, &type); + if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) + i -= lo; + + if (i >= 0) + ++nblk; + } + } else { + nblk = 0; + while (i > 0) { + --i; + lo = decode_compactedbits(lclusterbits, lomask, + in, encodebits * i, &type); + if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { + if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { + --i; + nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; + continue; + } + /* bigpcluster shouldn't have plain d0 == 1 */ + if (lo <= 1) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + i -= lo - 2; + continue; + } + ++nblk; + } + } + in += (vcnt << amortizedshift) - sizeof(__le32); + m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; + return 0; +} + +static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, + unsigned long lcn) +{ + struct inode *const inode = m->inode; + struct erofs_inode *const vi = EROFS_I(inode); + const unsigned int lclusterbits = vi->z_logical_clusterbits; + const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) + + vi->inode_isize + vi->xattr_isize, 8) + + sizeof(struct z_erofs_map_header); + const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); + unsigned int compacted_4b_initial, compacted_2b; + unsigned int amortizedshift; + erofs_off_t pos; + int err; + + if (lclusterbits != 12) + return -EOPNOTSUPP; + + if (lcn >= totalidx) + return -EINVAL; + + m->lcn = lcn; + /* used to align to 32-byte (compacted_2b) alignment */ + compacted_4b_initial = (32 - ebase % 32) / 4; + if (compacted_4b_initial == 32 / 4) + compacted_4b_initial = 0; + + if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) + compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); + else + compacted_2b = 0; + + pos = ebase; + if (lcn < compacted_4b_initial) { + amortizedshift = 2; + goto out; + } + pos += compacted_4b_initial * 4; + lcn -= compacted_4b_initial; + + if (lcn < compacted_2b) { + amortizedshift = 1; + goto out; + } + pos += compacted_2b * 2; + lcn -= compacted_2b; + amortizedshift = 2; +out: + pos += lcn * (1 << amortizedshift); + err = z_erofs_reload_indexes(m, erofs_blknr(pos)); + if (err) + return err; + return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos)); +} + +static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, + unsigned int lcn) +{ + const unsigned int datamode = EROFS_I(m->inode)->datalayout; + + if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) + return legacy_load_cluster_from_disk(m, lcn); + + if (datamode == EROFS_INODE_FLAT_COMPRESSION) + return compacted_load_cluster_from_disk(m, lcn); + + return -EINVAL; +} + +static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, + unsigned int lookback_distance) +{ + struct erofs_inode *const vi = EROFS_I(m->inode); + struct erofs_map_blocks *const map = m->map; + const unsigned int lclusterbits = vi->z_logical_clusterbits; + unsigned long lcn = m->lcn; + int err; + + if (lcn < lookback_distance) { + erofs_err(m->inode->i_sb, + "bogus lookback distance @ nid %llu", vi->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; + } + + /* load extent head logical cluster if needed */ + lcn -= lookback_distance; + err = z_erofs_load_cluster_from_disk(m, lcn); + if (err) + return err; + + switch (m->type) { + case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: + if (!m->delta[0]) { + erofs_err(m->inode->i_sb, + "invalid lookback distance 0 @ nid %llu", + vi->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; + } + return z_erofs_extent_lookback(m, m->delta[0]); + case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: + map->m_flags &= ~EROFS_MAP_ZIPPED; + /* fallthrough */ + case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: + map->m_la = (lcn << lclusterbits) | m->clusterofs; + break; + default: + erofs_err(m->inode->i_sb, + "unknown type %u @ lcn %lu of nid %llu", + m->type, lcn, vi->nid); + DBG_BUGON(1); + return -EOPNOTSUPP; + } + return 0; +} + +static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, + unsigned int initial_lcn) +{ + struct erofs_inode *const vi = EROFS_I(m->inode); + struct erofs_map_blocks *const map = m->map; + const unsigned int lclusterbits = vi->z_logical_clusterbits; + unsigned long lcn; + int err; + + DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && + m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD); + if (!(map->m_flags & EROFS_MAP_ZIPPED) || + !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { + map->m_plen = 1 << lclusterbits; + return 0; + } + + lcn = m->lcn + 1; + if (m->compressedlcs) + goto out; + + err = z_erofs_load_cluster_from_disk(m, lcn); + if (err) + return err; + + /* + * If the 1st NONHEAD lcluster has already been handled initially w/o + * valid compressedlcs, which means at least it mustn't be CBLKCNT, or + * an internal implemenatation error is detected. + * + * The following code can also handle it properly anyway, but let's + * BUG_ON in the debugging mode only for developers to notice that. + */ + DBG_BUGON(lcn == initial_lcn && + m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD); + + switch (m->type) { + case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: + case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: + /* + * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type + * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. + */ + m->compressedlcs = 1; + break; + case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: + if (m->delta[0] != 1) + goto err_bonus_cblkcnt; + if (m->compressedlcs) + break; + /* fallthrough */ + default: + erofs_err(m->inode->i_sb, + "cannot found CBLKCNT @ lcn %lu of nid %llu", + lcn, vi->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; + } +out: + map->m_plen = m->compressedlcs << lclusterbits; + return 0; +err_bonus_cblkcnt: + erofs_err(m->inode->i_sb, + "bogus CBLKCNT @ lcn %lu of nid %llu", + lcn, vi->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; +} + +int z_erofs_map_blocks_iter(struct inode *inode, + struct erofs_map_blocks *map, + int flags) +{ + struct erofs_inode *const vi = EROFS_I(inode); + struct z_erofs_maprecorder m = { + .inode = inode, + .map = map, + }; + int err = 0; + unsigned int lclusterbits, endoff; + unsigned long initial_lcn; + unsigned long long ofs, end; + + trace_z_erofs_map_blocks_iter_enter(inode, map, flags); + + /* when trying to read beyond EOF, leave it unmapped */ + if (map->m_la >= inode->i_size) { + map->m_llen = map->m_la + 1 - inode->i_size; + map->m_la = inode->i_size; + map->m_flags = 0; + goto out; + } + + err = z_erofs_fill_inode_lazy(inode); + if (err) + goto out; + + lclusterbits = vi->z_logical_clusterbits; + ofs = map->m_la; + initial_lcn = ofs >> lclusterbits; + endoff = ofs & ((1 << lclusterbits) - 1); + + err = z_erofs_load_cluster_from_disk(&m, initial_lcn); + if (err) + goto unmap_out; + + map->m_flags = EROFS_MAP_ZIPPED; /* by default, compressed */ + end = (m.lcn + 1ULL) << lclusterbits; + + switch (m.type) { + case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: + if (endoff >= m.clusterofs) + map->m_flags &= ~EROFS_MAP_ZIPPED; + /* fallthrough */ + case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: + if (endoff >= m.clusterofs) { + map->m_la = (m.lcn << lclusterbits) | m.clusterofs; + break; + } + /* m.lcn should be >= 1 if endoff < m.clusterofs */ + if (!m.lcn) { + erofs_err(inode->i_sb, + "invalid logical cluster 0 at nid %llu", + vi->nid); + err = -EFSCORRUPTED; + goto unmap_out; + } + end = (m.lcn << lclusterbits) | m.clusterofs; + map->m_flags |= EROFS_MAP_FULL_MAPPED; + m.delta[0] = 1; + /* fallthrough */ + case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: + /* get the corresponding first chunk */ + err = z_erofs_extent_lookback(&m, m.delta[0]); + if (err) + goto unmap_out; + break; + default: + erofs_err(inode->i_sb, + "unknown type %u @ offset %llu of nid %llu", + m.type, ofs, vi->nid); + err = -EOPNOTSUPP; + goto unmap_out; + } + + map->m_llen = end - map->m_la; + map->m_pa = blknr_to_addr(m.pblk); + map->m_flags |= EROFS_MAP_MAPPED; + + err = z_erofs_get_extent_compressedlen(&m, initial_lcn); + if (err) + goto out; +unmap_out: + if (m.kaddr) + kunmap_atomic(m.kaddr); + +out: + erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", + __func__, map->m_la, map->m_pa, + map->m_llen, map->m_plen, map->m_flags); + + trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); + + /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ + DBG_BUGON(err < 0 && err != -ENOMEM); + return err; +} diff --git a/drivers/staging/erofs/unzip_pagevec.h b/fs/erofs/zpvec.h similarity index 69% rename from drivers/staging/erofs/unzip_pagevec.h rename to fs/erofs/zpvec.h index efbf541e11bb..b05464f4a808 100644 --- a/drivers/staging/erofs/unzip_pagevec.h +++ b/fs/erofs/zpvec.h @@ -1,21 +1,14 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * linux/drivers/staging/erofs/unzip_pagevec.h - * +/* SPDX-License-Identifier: GPL-2.0-only */ +/* * Copyright (C) 2018 HUAWEI, Inc. - * http://www.huawei.com/ - * Created by Gao Xiang - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. + * https://www.huawei.com/ */ -#ifndef __EROFS_UNZIP_PAGEVEC_H -#define __EROFS_UNZIP_PAGEVEC_H +#ifndef __EROFS_FS_ZPVEC_H +#define __EROFS_FS_ZPVEC_H -#include +#include "tagptr.h" -/* page type in pagevec for unzip subsystem */ +/* page type in pagevec for decompress subsystem */ enum z_erofs_page_type { /* including Z_EROFS_VLE_PAGE_TAIL_EXCLUSIVE */ Z_EROFS_PAGE_TYPE_EXCLUSIVE, @@ -43,7 +36,7 @@ struct z_erofs_pagevec_ctor { static inline void z_erofs_pagevec_ctor_exit(struct z_erofs_pagevec_ctor *ctor, bool atomic) { - if (ctor->curr == NULL) + if (!ctor->curr) return; if (atomic) @@ -54,25 +47,22 @@ static inline void z_erofs_pagevec_ctor_exit(struct z_erofs_pagevec_ctor *ctor, static inline struct page * z_erofs_pagevec_ctor_next_page(struct z_erofs_pagevec_ctor *ctor, - unsigned nr) + unsigned int nr) { - unsigned index; + unsigned int index; /* keep away from occupied pages */ - if (ctor->next != NULL) + if (ctor->next) return ctor->next; for (index = 0; index < nr; ++index) { const erofs_vtptr_t t = ctor->pages[index]; - const unsigned tags = tagptr_unfold_tags(t); + const unsigned int tags = tagptr_unfold_tags(t); if (tags == Z_EROFS_PAGE_TYPE_EXCLUSIVE) return tagptr_unfold_ptr(t); } - - if (unlikely(nr >= ctor->nr)) - BUG(); - + DBG_BUGON(nr >= ctor->nr); return NULL; } @@ -94,8 +84,9 @@ z_erofs_pagevec_ctor_pagedown(struct z_erofs_pagevec_ctor *ctor, } static inline void z_erofs_pagevec_ctor_init(struct z_erofs_pagevec_ctor *ctor, - unsigned nr, - erofs_vtptr_t *pages, unsigned i) + unsigned int nr, + erofs_vtptr_t *pages, + unsigned int i) { ctor->nr = nr; ctor->curr = ctor->next = NULL; @@ -109,16 +100,14 @@ static inline void z_erofs_pagevec_ctor_init(struct z_erofs_pagevec_ctor *ctor, z_erofs_pagevec_ctor_pagedown(ctor, false); } } - ctor->next = z_erofs_pagevec_ctor_next_page(ctor, i); ctor->index = i; } -static inline bool -z_erofs_pagevec_ctor_enqueue(struct z_erofs_pagevec_ctor *ctor, - struct page *page, - enum z_erofs_page_type type, - bool pvec_safereuse) +static inline bool z_erofs_pagevec_enqueue(struct z_erofs_pagevec_ctor *ctor, + struct page *page, + enum z_erofs_page_type type, + bool pvec_safereuse) { if (!ctor->next) { /* some pages cannot be reused as pvec safely without I/O */ @@ -130,7 +119,7 @@ z_erofs_pagevec_ctor_enqueue(struct z_erofs_pagevec_ctor *ctor, return false; } - if (unlikely(ctor->index >= ctor->nr)) + if (ctor->index >= ctor->nr) z_erofs_pagevec_ctor_pagedown(ctor, false); /* exclusive page type must be 0 */ @@ -141,19 +130,17 @@ z_erofs_pagevec_ctor_enqueue(struct z_erofs_pagevec_ctor *ctor, if (type == (uintptr_t)ctor->next) { ctor->next = page; } - - ctor->pages[ctor->index++] = - tagptr_fold(erofs_vtptr_t, page, type); + ctor->pages[ctor->index++] = tagptr_fold(erofs_vtptr_t, page, type); return true; } static inline struct page * -z_erofs_pagevec_ctor_dequeue(struct z_erofs_pagevec_ctor *ctor, - enum z_erofs_page_type *type) +z_erofs_pagevec_dequeue(struct z_erofs_pagevec_ctor *ctor, + enum z_erofs_page_type *type) { erofs_vtptr_t t; - if (unlikely(ctor->index >= ctor->nr)) { + if (ctor->index >= ctor->nr) { DBG_BUGON(!ctor->next); z_erofs_pagevec_ctor_pagedown(ctor, true); } @@ -166,11 +153,7 @@ z_erofs_pagevec_ctor_dequeue(struct z_erofs_pagevec_ctor *ctor, if (*type == (uintptr_t)ctor->next) ctor->next = tagptr_unfold_ptr(t); - ctor->pages[ctor->index++] = - tagptr_fold(erofs_vtptr_t, NULL, 0); - + ctor->pages[ctor->index++] = tagptr_fold(erofs_vtptr_t, NULL, 0); return tagptr_unfold_ptr(t); } - #endif - diff --git a/fs/exec.c b/fs/exec.c index ca5a34ce7ff8..51f282d0dbd5 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1831,6 +1831,9 @@ static int __do_execve_file(int fd, struct filename *filename, goto out_unmark; bprm->argc = count(argv, MAX_ARG_STRINGS); + if (bprm->argc == 0) + pr_warn_once("process '%s' launched '%s' with NULL argv: empty string added\n", + current->comm, bprm->filename); if ((retval = bprm->argc) < 0) goto out; @@ -1855,6 +1858,20 @@ static int __do_execve_file(int fd, struct filename *filename, if (retval < 0) goto out; + /* + * When argv is empty, add an empty string ("") as argv[0] to + * ensure confused userspace programs that start processing + * from argv[1] won't end up walking envp. See also + * bprm_stack_limits(). + */ + if (bprm->argc == 0) { + const char *argv[] = { "", NULL }; + retval = copy_strings_kernel(1, argv, bprm); + if (retval < 0) + goto out; + bprm->argc = 1; + } + retval = exec_binprm(bprm); if (retval < 0) goto out; diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c index 9b0bf43e7cf7..0d5d1fe0c54c 100644 --- a/fs/ext4/dir.c +++ b/fs/ext4/dir.c @@ -30,8 +30,6 @@ #include "ext4.h" #include "xattr.h" -#define DOTDOT_OFFSET 12 - static int ext4_dx_readdir(struct file *, struct dir_context *); /** @@ -54,15 +52,14 @@ static int is_dx_dir(struct inode *inode) return 0; } -static bool is_fake_entry(struct inode *dir, ext4_lblk_t lblk, - unsigned int offset, unsigned int blocksize) +static bool is_fake_dir_entry(struct ext4_dir_entry_2 *de) { - /* Entries in the first block before this value refer to . or .. */ - if (lblk == 0 && offset <= DOTDOT_OFFSET) + /* Check if . or .. , or skip if namelen is 0 */ + if ((de->name_len > 0) && (de->name_len <= 2) && (de->name[0] == '.') && + (de->name[1] == '.' || de->name[1] == '\0')) return true; - /* Check if this is likely the csum entry */ - if (ext4_has_metadata_csum(dir->i_sb) && offset % blocksize == - blocksize - sizeof(struct ext4_dir_entry_tail)) + /* Check if this is a csum entry */ + if (de->file_type == EXT4_FT_DIR_CSUM) return true; return false; } @@ -79,16 +76,14 @@ int __ext4_check_dir_entry(const char *function, unsigned int line, struct inode *dir, struct file *filp, struct ext4_dir_entry_2 *de, struct buffer_head *bh, char *buf, int size, - ext4_lblk_t lblk, unsigned int offset) { const char *error_msg = NULL; const int rlen = ext4_rec_len_from_disk(de->rec_len, dir->i_sb->s_blocksize); const int next_offset = ((char *) de - buf) + rlen; - unsigned int blocksize = dir->i_sb->s_blocksize; - bool fake = is_fake_entry(dir, lblk, offset, blocksize); - bool next_fake = is_fake_entry(dir, lblk, next_offset, blocksize); + bool fake = is_fake_dir_entry(de); + bool has_csum = ext4_has_metadata_csum(dir->i_sb); if (unlikely(rlen < ext4_dir_rec_len(1, fake ? NULL : dir))) error_msg = "rec_len is smaller than minimal"; @@ -100,7 +95,7 @@ int __ext4_check_dir_entry(const char *function, unsigned int line, else if (unlikely(((char *) de - buf) + rlen > size)) error_msg = "directory entry overrun"; else if (unlikely(next_offset > size - ext4_dir_rec_len(1, - next_fake ? NULL : dir) && + has_csum ? NULL : dir) && next_offset != size)) error_msg = "directory entry too close to block end"; else if (unlikely(le32_to_cpu(de->inode) > @@ -112,15 +107,15 @@ int __ext4_check_dir_entry(const char *function, unsigned int line, if (filp) ext4_error_file(filp, function, line, bh->b_blocknr, "bad entry in directory: %s - offset=%u, " - "inode=%u, rec_len=%d, lblk=%d, size=%d fake=%d", + "inode=%u, rec_len=%d, size=%d fake=%d", error_msg, offset, le32_to_cpu(de->inode), - rlen, lblk, size, fake); + rlen, size, fake); else ext4_error_inode(dir, function, line, bh->b_blocknr, "bad entry in directory: %s - offset=%u, " - "inode=%u, rec_len=%d, lblk=%d, size=%d fake=%d", + "inode=%u, rec_len=%d, size=%d fake=%d", error_msg, offset, le32_to_cpu(de->inode), - rlen, lblk, size, fake); + rlen, size, fake); return 1; } @@ -262,7 +257,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); if (ext4_check_dir_entry(inode, file, de, bh, bh->b_data, bh->b_size, - map.m_lblk, offset)) { + offset)) { /* * On error, skip to the next block */ @@ -666,7 +661,7 @@ int ext4_check_all_de(struct inode *dir, struct buffer_head *bh, void *buf, top = buf + buf_size; while ((char *) de < top) { if (ext4_check_dir_entry(dir, NULL, de, bh, - buf, buf_size, 0, offset)) + buf, buf_size, offset)) return -EFSCORRUPTED; rlen = ext4_rec_len_from_disk(de->rec_len, buf_size); de = (struct ext4_dir_entry_2 *)((char *)de + rlen); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 838a07663848..4bfdcd31e469 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2508,17 +2508,16 @@ extern int __ext4_check_dir_entry(const char *, unsigned int, struct inode *, struct file *, struct ext4_dir_entry_2 *, struct buffer_head *, char *, int, - ext4_lblk_t, unsigned int); -#define ext4_check_dir_entry(dir, filp, de, bh, buf, size, lblk, offset) \ + unsigned int); +#define ext4_check_dir_entry(dir, filp, de, bh, buf, size, offset) \ unlikely(__ext4_check_dir_entry(__func__, __LINE__, (dir), (filp), \ - (de), (bh), (buf), (size), (lblk), (offset))) + (de), (bh), (buf), (size), (offset))) extern int ext4_htree_store_dirent(struct file *dir_file, __u32 hash, __u32 minor_hash, struct ext4_dir_entry_2 *dirent, struct fscrypt_str *ent_name); extern void ext4_htree_free_dir_info(struct dir_private_info *p); extern int ext4_find_dest_de(struct inode *dir, struct inode *inode, - ext4_lblk_t lblk, struct buffer_head *bh, void *buf, int buf_size, struct ext4_filename *fname, @@ -2707,12 +2706,11 @@ extern int ext4_search_dir(struct buffer_head *bh, int buf_size, struct inode *dir, struct ext4_filename *fname, - ext4_lblk_t lblk, unsigned int offset, + unsigned int offset, struct ext4_dir_entry_2 **res_dir); extern int ext4_generic_delete_entry(handle_t *handle, struct inode *dir, struct ext4_dir_entry_2 *de_del, - ext4_lblk_t lblk, struct buffer_head *bh, void *entry_buf, int buf_size, diff --git a/fs/ext4/hash.c b/fs/ext4/hash.c index 326b4d2b7678..93dac560ad48 100644 --- a/fs/ext4/hash.c +++ b/fs/ext4/hash.c @@ -294,7 +294,8 @@ int ext4fs_dirhash(const struct inode *dir, const char *name, int len, unsigned char *buff; struct qstr qstr = {.name = name, .len = len }; - if (len && needs_casefold(dir) && um) { + if (len && IS_CASEFOLDED(dir) && um && + (!IS_ENCRYPTED(dir) || fscrypt_has_encryption_key(dir))) { buff = kzalloc(sizeof(char) * PATH_MAX, GFP_KERNEL); if (!buff) return -ENOMEM; diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index a29e3fc3897e..a044f587398e 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -455,10 +455,7 @@ static int find_group_orlov(struct super_block *sb, struct inode *parent, int ret = -1; if (qstr) { - if (ext4_hash_in_dirent(parent)) - hinfo.hash_version = DX_HASH_SIPHASH; - else - hinfo.hash_version = DX_HASH_HALF_MD4; + hinfo.hash_version = DX_HASH_HALF_MD4; hinfo.seed = sbi->s_hash_seed; ext4fs_dirhash(parent, qstr->name, qstr->len, &hinfo); grp = hinfo.hash; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 57b17a71d087..aff46b12fcc9 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -1016,7 +1016,7 @@ void ext4_show_inline_dir(struct inode *dir, struct buffer_head *bh, offset, de_len, de->name_len, de->name, de->name_len, le32_to_cpu(de->inode)); if (ext4_check_dir_entry(dir, NULL, de, bh, - inline_start, inline_size, 0, offset)) + inline_start, inline_size, offset)) BUG(); offset += de_len; @@ -1042,7 +1042,7 @@ static int ext4_add_dirent_to_inline(handle_t *handle, int err; struct ext4_dir_entry_2 *de; - err = ext4_find_dest_de(dir, inode, 0, iloc->bh, inline_start, + err = ext4_find_dest_de(dir, inode, iloc->bh, inline_start, inline_size, fname, &de); if (err) return err; @@ -1429,7 +1429,7 @@ int htree_inlinedir_to_tree(struct file *dir_file, pos += ext4_rec_len_from_disk(de->rec_len, inline_size); if (ext4_check_dir_entry(inode, dir_file, de, iloc.bh, dir_buf, - inline_size, block, pos)) { + inline_size, pos)) { ret = count; goto out; } @@ -1586,7 +1586,7 @@ int ext4_read_inline_dir(struct file *file, de = (struct ext4_dir_entry_2 *) (dir_buf + ctx->pos - extra_offset); if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, - extra_size, 0, ctx->pos)) + extra_size, ctx->pos)) goto out; if (le32_to_cpu(de->inode)) { if (!dir_emit(ctx, de->name, de->name_len, @@ -1678,7 +1678,7 @@ struct buffer_head *ext4_find_inline_entry(struct inode *dir, EXT4_INLINE_DOTDOT_SIZE; inline_size = EXT4_MIN_INLINE_DATA_SIZE - EXT4_INLINE_DOTDOT_SIZE; ret = ext4_search_dir(iloc.bh, inline_start, inline_size, - dir, fname, 0, 0, res_dir); + dir, fname, 0, res_dir); if (ret == 1) goto out_find; if (ret < 0) @@ -1691,7 +1691,7 @@ struct buffer_head *ext4_find_inline_entry(struct inode *dir, inline_size = ext4_get_inline_size(dir) - EXT4_MIN_INLINE_DATA_SIZE; ret = ext4_search_dir(iloc.bh, inline_start, inline_size, - dir, fname, 0, 0, res_dir); + dir, fname, 0, res_dir); if (ret == 1) goto out_find; @@ -1740,7 +1740,7 @@ int ext4_delete_inline_entry(handle_t *handle, if (err) goto out; - err = ext4_generic_delete_entry(handle, dir, de_del, 0, bh, + err = ext4_generic_delete_entry(handle, dir, de_del, bh, inline_start, inline_size, 0); if (err) goto out; @@ -1824,7 +1824,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data) &inline_pos, &inline_size); if (ext4_check_dir_entry(dir, NULL, de, iloc.bh, inline_pos, - inline_size, 0, offset)) { + inline_size, offset)) { ext4_warning(dir->i_sb, "bad inline directory (dir #%lu) - " "inode %u, rec_len %u, name_len %d" @@ -2040,6 +2040,18 @@ int ext4_convert_inline_data(struct inode *inode) if (!ext4_has_inline_data(inode)) { ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); return 0; + } else if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { + /* + * Inode has inline data but EXT4_STATE_MAY_INLINE_DATA is + * cleared. This means we are in the middle of moving of + * inline data to delay allocated block. Just force writeout + * here to finish conversion. + */ + error = filemap_flush(inode->i_mapping); + if (error) + return error; + if (!ext4_has_inline_data(inode)) + return 0; } needed_blocks = ext4_writepage_trans_blocks(inode); diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 93ea26a62fcc..ea9b159a50f9 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -273,9 +273,9 @@ static struct dx_frame *dx_probe(struct ext4_filename *fname, struct dx_hash_info *hinfo, struct dx_frame *frame); static void dx_release(struct dx_frame *frames); -static int dx_make_map(struct inode *dir, struct ext4_dir_entry_2 *de, - unsigned blocksize, struct dx_hash_info *hinfo, - struct dx_map_entry map[]); +static int dx_make_map(struct inode *dir, struct buffer_head *bh, + struct dx_hash_info *hinfo, + struct dx_map_entry *map_tail); static void dx_sort_map(struct dx_map_entry *map, unsigned count); static struct ext4_dir_entry_2 *dx_move_dirents(struct inode *dir, char *from, char *to, struct dx_map_entry *offsets, @@ -290,7 +290,7 @@ static int ext4_htree_next_block(struct inode *dir, __u32 hash, __u32 *start_hash); static struct buffer_head * ext4_dx_find_entry(struct inode *dir, struct ext4_filename *fname, - struct ext4_dir_entry_2 **res_dir, ext4_lblk_t *lblk); + struct ext4_dir_entry_2 **res_dir); static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, struct inode *dir, struct inode *inode); @@ -756,12 +756,14 @@ static struct dx_frame * dx_probe(struct ext4_filename *fname, struct inode *dir, struct dx_hash_info *hinfo, struct dx_frame *frame_in) { - unsigned count, indirect; + unsigned count, indirect, level, i; struct dx_entry *at, *entries, *p, *q, *m; struct dx_root *root; struct dx_frame *frame = frame_in; struct dx_frame *ret_err = ERR_PTR(ERR_BAD_DX_DIR); u32 hash; + ext4_lblk_t block; + ext4_lblk_t blocks[EXT4_HTREE_LEVEL]; memset(frame_in, 0, EXT4_HTREE_LEVEL * sizeof(frame_in[0])); frame->bh = ext4_read_dirblock(dir, 0, INDEX); @@ -833,6 +835,8 @@ dx_probe(struct ext4_filename *fname, struct inode *dir, } dxtrace(printk("Look up %x", hash)); + level = 0; + blocks[0] = 0; while (1) { count = dx_get_count(entries); if (!count || count > dx_get_limit(entries)) { @@ -874,15 +878,27 @@ dx_probe(struct ext4_filename *fname, struct inode *dir, dx_get_block(at))); frame->entries = entries; frame->at = at; - if (!indirect--) + + block = dx_get_block(at); + for (i = 0; i <= level; i++) { + if (blocks[i] == block) { + ext4_warning_inode(dir, + "dx entry: tree cycle block %u points back to block %u", + blocks[level], block); + goto fail; + } + } + if (++level > indirect) return frame; + blocks[level] = block; frame++; - frame->bh = ext4_read_dirblock(dir, dx_get_block(at), INDEX); + frame->bh = ext4_read_dirblock(dir, block, INDEX); if (IS_ERR(frame->bh)) { ret_err = (struct dx_frame *) frame->bh; frame->bh = NULL; goto fail; } + entries = ((struct dx_node *) frame->bh->b_data)->entries; if (dx_get_limit(entries) != dx_node_limit(dir)) { @@ -1045,7 +1061,7 @@ static int htree_dirblock_to_tree(struct file *dir_file, #endif for (; de < top; de = ext4_next_entry(de, dir->i_sb->s_blocksize)) { if (ext4_check_dir_entry(dir, NULL, de, bh, - bh->b_data, bh->b_size, block, + bh->b_data, bh->b_size, (block<i_sb)) + ((char *)de - bh->b_data))) { /* silently ignore the rest of the block */ @@ -1229,12 +1245,11 @@ int ext4_htree_fill_tree(struct file *dir_file, __u32 start_hash, static inline int search_dirblock(struct buffer_head *bh, struct inode *dir, struct ext4_filename *fname, - ext4_lblk_t lblk, unsigned int offset, struct ext4_dir_entry_2 **res_dir) { return ext4_search_dir(bh, bh->b_data, dir->i_sb->s_blocksize, dir, - fname, lblk, offset, res_dir); + fname, offset, res_dir); } /* @@ -1245,15 +1260,23 @@ static inline int search_dirblock(struct buffer_head *bh, * Create map of hash values, offsets, and sizes, stored at end of block. * Returns number of entries mapped. */ -static int dx_make_map(struct inode *dir, struct ext4_dir_entry_2 *de, - unsigned blocksize, struct dx_hash_info *hinfo, +static int dx_make_map(struct inode *dir, struct buffer_head *bh, + struct dx_hash_info *hinfo, struct dx_map_entry *map_tail) { int count = 0; - char *base = (char *) de; + struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *)bh->b_data; + unsigned int buflen = bh->b_size; + char *base = bh->b_data; struct dx_hash_info h = *hinfo; - while ((char *) de < base + blocksize) { + if (ext4_has_metadata_csum(dir->i_sb)) + buflen -= sizeof(struct ext4_dir_entry_tail); + + while ((char *) de < base + buflen) { + if (ext4_check_dir_entry(dir, NULL, de, bh, base, buflen, + ((char *)de) - base)) + return -EFSCORRUPTED; if (de->name_len && de->inode) { if (ext4_hash_in_dirent(dir)) h.hash = EXT4_DIRENT_HASH(de); @@ -1266,8 +1289,7 @@ static int dx_make_map(struct inode *dir, struct ext4_dir_entry_2 *de, count++; cond_resched(); } - /* XXX: do we need to check rec_len == 0 case? -Chris */ - de = ext4_next_entry(de, blocksize); + de = ext4_next_entry(de, dir->i_sb->s_blocksize); } return count; } @@ -1373,7 +1395,8 @@ int ext4_fname_setup_ci_filename(struct inode *dir, const struct qstr *iname, struct dx_hash_info *hinfo = &name->hinfo; int len; - if (!needs_casefold(dir)) { + if (!IS_CASEFOLDED(dir) || !dir->i_sb->s_encoding || + (IS_ENCRYPTED(dir) && !fscrypt_has_encryption_key(dir))) { cf_name->name = NULL; return 0; } @@ -1424,7 +1447,8 @@ static bool ext4_match(struct inode *parent, #endif #ifdef CONFIG_UNICODE - if (needs_casefold(parent)) { + if (parent->i_sb->s_encoding && IS_CASEFOLDED(parent) && + (!IS_ENCRYPTED(parent) || fscrypt_has_encryption_key(parent))) { if (fname->cf_name.name) { struct qstr cf = {.name = fname->cf_name.name, .len = fname->cf_name.len}; @@ -1452,8 +1476,7 @@ static bool ext4_match(struct inode *parent, */ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, struct inode *dir, struct ext4_filename *fname, - ext4_lblk_t lblk, unsigned int offset, - struct ext4_dir_entry_2 **res_dir) + unsigned int offset, struct ext4_dir_entry_2 **res_dir) { struct ext4_dir_entry_2 * de; char * dlimit; @@ -1469,7 +1492,7 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, /* found a match - just to be sure, do * a full check */ if (ext4_check_dir_entry(dir, NULL, de, bh, search_buf, - buf_size, lblk, offset)) + buf_size, offset)) return -1; *res_dir = de; return 1; @@ -1515,7 +1538,7 @@ static int is_dx_internal_node(struct inode *dir, ext4_lblk_t block, static struct buffer_head *__ext4_find_entry(struct inode *dir, struct ext4_filename *fname, struct ext4_dir_entry_2 **res_dir, - int *inlined, ext4_lblk_t *lblk) + int *inlined) { struct super_block *sb; struct buffer_head *bh_use[NAMEI_RA_SIZE]; @@ -1539,8 +1562,6 @@ static struct buffer_head *__ext4_find_entry(struct inode *dir, int has_inline_data = 1; ret = ext4_find_inline_entry(dir, fname, res_dir, &has_inline_data); - if (lblk) - *lblk = 0; if (has_inline_data) { if (inlined) *inlined = 1; @@ -1559,7 +1580,7 @@ static struct buffer_head *__ext4_find_entry(struct inode *dir, goto restart; } if (is_dx(dir)) { - ret = ext4_dx_find_entry(dir, fname, res_dir, lblk); + ret = ext4_dx_find_entry(dir, fname, res_dir); /* * On success, or if the error was file not found, * return. Otherwise, fall back to doing a search the @@ -1624,11 +1645,9 @@ static struct buffer_head *__ext4_find_entry(struct inode *dir, goto cleanup_and_exit; } set_buffer_verified(bh); - i = search_dirblock(bh, dir, fname, block, + i = search_dirblock(bh, dir, fname, block << EXT4_BLOCK_SIZE_BITS(sb), res_dir); if (i == 1) { - if (lblk) - *lblk = block; EXT4_I(dir)->i_dir_start_lookup = block; ret = bh; goto cleanup_and_exit; @@ -1663,7 +1682,7 @@ static struct buffer_head *__ext4_find_entry(struct inode *dir, static struct buffer_head *ext4_find_entry(struct inode *dir, const struct qstr *d_name, struct ext4_dir_entry_2 **res_dir, - int *inlined, ext4_lblk_t *lblk) + int *inlined) { int err; struct ext4_filename fname; @@ -1675,7 +1694,7 @@ static struct buffer_head *ext4_find_entry(struct inode *dir, if (err) return ERR_PTR(err); - bh = __ext4_find_entry(dir, &fname, res_dir, inlined, lblk); + bh = __ext4_find_entry(dir, &fname, res_dir, inlined); ext4_fname_free_filename(&fname); return bh; @@ -1696,7 +1715,7 @@ static struct buffer_head *ext4_lookup_entry(struct inode *dir, if (err) return ERR_PTR(err); - bh = __ext4_find_entry(dir, &fname, res_dir, NULL, NULL); + bh = __ext4_find_entry(dir, &fname, res_dir, NULL); ext4_fname_free_filename(&fname); return bh; @@ -1704,7 +1723,7 @@ static struct buffer_head *ext4_lookup_entry(struct inode *dir, static struct buffer_head * ext4_dx_find_entry(struct inode *dir, struct ext4_filename *fname, - struct ext4_dir_entry_2 **res_dir, ext4_lblk_t *lblk) + struct ext4_dir_entry_2 **res_dir) { struct super_block * sb = dir->i_sb; struct dx_frame frames[EXT4_HTREE_LEVEL], *frame; @@ -1720,13 +1739,11 @@ static struct buffer_head * ext4_dx_find_entry(struct inode *dir, return (struct buffer_head *) frame; do { block = dx_get_block(frame->at); - if (lblk) - *lblk = block; bh = ext4_read_dirblock(dir, block, DIRENT_HTREE); if (IS_ERR(bh)) goto errout; - retval = search_dirblock(bh, dir, fname, block, + retval = search_dirblock(bh, dir, fname, block << EXT4_BLOCK_SIZE_BITS(sb), res_dir); if (retval == 1) @@ -1826,7 +1843,7 @@ struct dentry *ext4_get_parent(struct dentry *child) struct ext4_dir_entry_2 * de; struct buffer_head *bh; - bh = ext4_find_entry(d_inode(child), &dotdot, &de, NULL, NULL); + bh = ext4_find_entry(d_inode(child), &dotdot, &de, NULL); if (IS_ERR(bh)) return (struct dentry *) bh; if (!bh) @@ -1901,12 +1918,13 @@ static struct ext4_dir_entry_2 *dx_pack_dirents(struct inode *dir, char *base, * Returns pointer to de in block into which the new entry will be inserted. */ static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir, - struct buffer_head **bh, struct dx_frame *frame, - struct dx_hash_info *hinfo, ext4_lblk_t *newblock) + struct buffer_head **bh,struct dx_frame *frame, + struct dx_hash_info *hinfo) { unsigned blocksize = dir->i_sb->s_blocksize; unsigned count, continued; struct buffer_head *bh2; + ext4_lblk_t newblock; u32 hash2; struct dx_map_entry *map; char *data1 = (*bh)->b_data, *data2; @@ -1919,7 +1937,7 @@ static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir, if (ext4_has_metadata_csum(dir->i_sb)) csum_size = sizeof(struct ext4_dir_entry_tail); - bh2 = ext4_append(handle, dir, newblock); + bh2 = ext4_append(handle, dir, &newblock); if (IS_ERR(bh2)) { brelse(*bh); *bh = NULL; @@ -1940,8 +1958,11 @@ static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir, /* create map in the end of data2 block */ map = (struct dx_map_entry *) (data2 + blocksize); - count = dx_make_map(dir, (struct ext4_dir_entry_2 *) data1, - blocksize, hinfo, map); + count = dx_make_map(dir, *bh, hinfo, map); + if (count < 0) { + err = count; + goto journal_error; + } map -= count; dx_sort_map(map, count); /* Ensure that neither split block is over half full */ @@ -2000,7 +2021,7 @@ static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir, swap(*bh, bh2); de = de2; } - dx_insert_block(frame, hash2 + continued, *newblock); + dx_insert_block(frame, hash2 + continued, newblock); err = ext4_handle_dirty_dirent_node(handle, dir, bh2); if (err) goto journal_error; @@ -2020,7 +2041,6 @@ static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir, } int ext4_find_dest_de(struct inode *dir, struct inode *inode, - ext4_lblk_t lblk, struct buffer_head *bh, void *buf, int buf_size, struct ext4_filename *fname, @@ -2036,7 +2056,7 @@ int ext4_find_dest_de(struct inode *dir, struct inode *inode, top = buf + buf_size - reclen; while ((char *) de <= top) { if (ext4_check_dir_entry(dir, NULL, de, bh, - buf, buf_size, lblk, offset)) + buf, buf_size, offset)) return -EFSCORRUPTED; if (ext4_match(dir, fname, de)) return -EEXIST; @@ -2097,7 +2117,6 @@ void ext4_insert_dentry(struct inode *dir, static int add_dirent_to_buf(handle_t *handle, struct ext4_filename *fname, struct inode *dir, struct inode *inode, struct ext4_dir_entry_2 *de, - ext4_lblk_t blk, struct buffer_head *bh) { unsigned int blocksize = dir->i_sb->s_blocksize; @@ -2108,7 +2127,7 @@ static int add_dirent_to_buf(handle_t *handle, struct ext4_filename *fname, csum_size = sizeof(struct ext4_dir_entry_tail); if (!de) { - err = ext4_find_dest_de(dir, inode, blk, bh, bh->b_data, + err = ext4_find_dest_de(dir, inode, bh, bh->b_data, blocksize - csum_size, fname, &de); if (err) return err; @@ -2256,13 +2275,13 @@ static int make_indexed_dir(handle_t *handle, struct ext4_filename *fname, if (retval) goto out_frames; - de = do_split(handle, dir, &bh2, frame, &fname->hinfo, &block); + de = do_split(handle,dir, &bh2, frame, &fname->hinfo); if (IS_ERR(de)) { retval = PTR_ERR(de); goto out_frames; } - retval = add_dirent_to_buf(handle, fname, dir, inode, de, block, bh2); + retval = add_dirent_to_buf(handle, fname, dir, inode, de, bh2); out_frames: /* * Even if the block split failed, we have to properly write @@ -2361,7 +2380,7 @@ static int ext4_add_entry(handle_t *handle, struct dentry *dentry, goto out; } retval = add_dirent_to_buf(handle, &fname, dir, inode, - NULL, block, bh); + NULL, bh); if (retval != -ENOSPC) goto out; @@ -2390,7 +2409,7 @@ static int ext4_add_entry(handle_t *handle, struct dentry *dentry, initialize_dirent_tail(t, blocksize); } - retval = add_dirent_to_buf(handle, &fname, dir, inode, de, block, bh); + retval = add_dirent_to_buf(handle, &fname, dir, inode, de, bh); out: ext4_fname_free_filename(&fname); brelse(bh); @@ -2412,7 +2431,6 @@ static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, struct ext4_dir_entry_2 *de; int restart; int err; - ext4_lblk_t lblk; again: restart = 0; @@ -2421,8 +2439,7 @@ static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, return PTR_ERR(frame); entries = frame->entries; at = frame->at; - lblk = dx_get_block(frame->at); - bh = ext4_read_dirblock(dir, lblk, DIRENT_HTREE); + bh = ext4_read_dirblock(dir, dx_get_block(frame->at), DIRENT_HTREE); if (IS_ERR(bh)) { err = PTR_ERR(bh); bh = NULL; @@ -2434,7 +2451,7 @@ static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, if (err) goto journal_error; - err = add_dirent_to_buf(handle, fname, dir, inode, NULL, lblk, bh); + err = add_dirent_to_buf(handle, fname, dir, inode, NULL, bh); if (err != -ENOSPC) goto cleanup; @@ -2553,12 +2570,12 @@ static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, goto journal_error; } } - de = do_split(handle, dir, &bh, frame, &fname->hinfo, &lblk); + de = do_split(handle, dir, &bh, frame, &fname->hinfo); if (IS_ERR(de)) { err = PTR_ERR(de); goto cleanup; } - err = add_dirent_to_buf(handle, fname, dir, inode, de, lblk, bh); + err = add_dirent_to_buf(handle, fname, dir, inode, de, bh); goto cleanup; journal_error: @@ -2581,7 +2598,6 @@ static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, int ext4_generic_delete_entry(handle_t *handle, struct inode *dir, struct ext4_dir_entry_2 *de_del, - ext4_lblk_t lblk, struct buffer_head *bh, void *entry_buf, int buf_size, @@ -2596,7 +2612,7 @@ int ext4_generic_delete_entry(handle_t *handle, de = (struct ext4_dir_entry_2 *)entry_buf; while (i < buf_size - csum_size) { if (ext4_check_dir_entry(dir, NULL, de, bh, - entry_buf, buf_size, lblk, i)) + entry_buf, buf_size, i)) return -EFSCORRUPTED; if (de == de_del) { if (pde) @@ -2621,7 +2637,6 @@ int ext4_generic_delete_entry(handle_t *handle, static int ext4_delete_entry(handle_t *handle, struct inode *dir, struct ext4_dir_entry_2 *de_del, - ext4_lblk_t lblk, struct buffer_head *bh) { int err, csum_size = 0; @@ -2642,7 +2657,7 @@ static int ext4_delete_entry(handle_t *handle, if (unlikely(err)) goto out; - err = ext4_generic_delete_entry(handle, dir, de_del, lblk, + err = ext4_generic_delete_entry(handle, dir, de_del, bh, bh->b_data, dir->i_sb->s_blocksize, csum_size); if (err) @@ -2981,7 +2996,7 @@ bool ext4_empty_dir(struct inode *inode) return true; de = (struct ext4_dir_entry_2 *) bh->b_data; - if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size, 0, + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size, 0) || le32_to_cpu(de->inode) != inode->i_ino || strcmp(".", de->name)) { ext4_warning_inode(inode, "directory missing '.'"); @@ -2990,7 +3005,7 @@ bool ext4_empty_dir(struct inode *inode) } offset = ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); de = ext4_next_entry(de, sb->s_blocksize); - if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size, 0, + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size, offset) || le32_to_cpu(de->inode) == 0 || strcmp("..", de->name)) { ext4_warning_inode(inode, "directory missing '..'"); @@ -3014,7 +3029,7 @@ bool ext4_empty_dir(struct inode *inode) de = (struct ext4_dir_entry_2 *) (bh->b_data + (offset & (sb->s_blocksize - 1))); if (ext4_check_dir_entry(inode, NULL, de, bh, - bh->b_data, bh->b_size, 0, offset)) { + bh->b_data, bh->b_size, offset)) { offset = (offset | (sb->s_blocksize - 1)) + 1; continue; } @@ -3209,8 +3224,6 @@ static int ext4_rmdir(struct inode *dir, struct dentry *dentry) struct buffer_head *bh; struct ext4_dir_entry_2 *de; handle_t *handle = NULL; - ext4_lblk_t lblk; - if (unlikely(ext4_forced_shutdown(EXT4_SB(dir->i_sb)))) return -EIO; @@ -3225,7 +3238,7 @@ static int ext4_rmdir(struct inode *dir, struct dentry *dentry) return retval; retval = -ENOENT; - bh = ext4_find_entry(dir, &dentry->d_name, &de, NULL, &lblk); + bh = ext4_find_entry(dir, &dentry->d_name, &de, NULL); if (IS_ERR(bh)) return PTR_ERR(bh); if (!bh) @@ -3252,7 +3265,7 @@ static int ext4_rmdir(struct inode *dir, struct dentry *dentry) if (IS_DIRSYNC(dir)) ext4_handle_sync(handle); - retval = ext4_delete_entry(handle, dir, de, lblk, bh); + retval = ext4_delete_entry(handle, dir, de, bh); if (retval) goto end_rmdir; if (!EXT4_DIR_LINK_EMPTY(inode)) @@ -3298,7 +3311,6 @@ static int ext4_unlink(struct inode *dir, struct dentry *dentry) struct buffer_head *bh; struct ext4_dir_entry_2 *de; handle_t *handle = NULL; - ext4_lblk_t lblk; if (unlikely(ext4_forced_shutdown(EXT4_SB(dir->i_sb)))) return -EIO; @@ -3314,7 +3326,7 @@ static int ext4_unlink(struct inode *dir, struct dentry *dentry) return retval; retval = -ENOENT; - bh = ext4_find_entry(dir, &dentry->d_name, &de, NULL, &lblk); + bh = ext4_find_entry(dir, &dentry->d_name, &de, NULL); if (IS_ERR(bh)) return PTR_ERR(bh); if (!bh) @@ -3337,7 +3349,7 @@ static int ext4_unlink(struct inode *dir, struct dentry *dentry) if (IS_DIRSYNC(dir)) ext4_handle_sync(handle); - retval = ext4_delete_entry(handle, dir, de, lblk, bh); + retval = ext4_delete_entry(handle, dir, de, bh); if (retval) goto end_unlink; dir->i_ctime = dir->i_mtime = current_time(dir); @@ -3573,6 +3585,9 @@ static struct buffer_head *ext4_get_first_dir_block(handle_t *handle, struct buffer_head *bh; if (!ext4_has_inline_data(inode)) { + struct ext4_dir_entry_2 *de; + unsigned int offset; + /* The first directory block must not be a hole, so * treat it as DIRENT_HTREE */ @@ -3581,9 +3596,30 @@ static struct buffer_head *ext4_get_first_dir_block(handle_t *handle, *retval = PTR_ERR(bh); return NULL; } - *parent_de = ext4_next_entry( - (struct ext4_dir_entry_2 *)bh->b_data, - inode->i_sb->s_blocksize); + + de = (struct ext4_dir_entry_2 *) bh->b_data; + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, + bh->b_size, 0) || + le32_to_cpu(de->inode) != inode->i_ino || + strcmp(".", de->name)) { + EXT4_ERROR_INODE(inode, "directory missing '.'"); + brelse(bh); + *retval = -EFSCORRUPTED; + return NULL; + } + offset = ext4_rec_len_from_disk(de->rec_len, + inode->i_sb->s_blocksize); + de = ext4_next_entry(de, inode->i_sb->s_blocksize); + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, + bh->b_size, offset) || + le32_to_cpu(de->inode) == 0 || strcmp("..", de->name)) { + EXT4_ERROR_INODE(inode, "directory missing '..'"); + brelse(bh); + *retval = -EFSCORRUPTED; + return NULL; + } + *parent_de = de; + return bh; } @@ -3599,7 +3635,6 @@ struct ext4_renament { int dir_nlink_delta; /* entry for "dentry" */ - ext4_lblk_t lblk; struct buffer_head *bh; struct ext4_dir_entry_2 *de; int inlined; @@ -3692,8 +3727,7 @@ static void ext4_resetent(handle_t *handle, struct ext4_renament *ent, * so the old->de may no longer valid and need to find it again * before reset old inode info. */ - old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de, NULL, - NULL); + old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de, NULL); if (IS_ERR(old.bh)) retval = PTR_ERR(old.bh); if (!old.bh) @@ -3713,13 +3747,12 @@ static int ext4_find_delete_entry(handle_t *handle, struct inode *dir, int retval = -ENOENT; struct buffer_head *bh; struct ext4_dir_entry_2 *de; - ext4_lblk_t lblk; - bh = ext4_find_entry(dir, d_name, &de, NULL, &lblk); + bh = ext4_find_entry(dir, d_name, &de, NULL); if (IS_ERR(bh)) return PTR_ERR(bh); if (bh) { - retval = ext4_delete_entry(handle, dir, de, lblk, bh); + retval = ext4_delete_entry(handle, dir, de, bh); brelse(bh); } return retval; @@ -3743,8 +3776,7 @@ static void ext4_rename_delete(handle_t *handle, struct ext4_renament *ent, retval = ext4_find_delete_entry(handle, ent->dir, &ent->dentry->d_name); } else { - retval = ext4_delete_entry(handle, ent->dir, ent->de, - ent->lblk, ent->bh); + retval = ext4_delete_entry(handle, ent->dir, ent->de, ent->bh); if (retval == -ENOENT) { retval = ext4_find_delete_entry(handle, ent->dir, &ent->dentry->d_name); @@ -3860,8 +3892,7 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry, return retval; } - old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de, NULL, - &old.lblk); + old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de, NULL); if (IS_ERR(old.bh)) return PTR_ERR(old.bh); /* @@ -3875,7 +3906,7 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry, goto release_bh; new.bh = ext4_find_entry(new.dir, &new.dentry->d_name, - &new.de, &new.inlined, NULL); + &new.de, &new.inlined); if (IS_ERR(new.bh)) { retval = PTR_ERR(new.bh); new.bh = NULL; @@ -4066,7 +4097,7 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry, return retval; old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, - &old.de, &old.inlined, NULL); + &old.de, &old.inlined); if (IS_ERR(old.bh)) return PTR_ERR(old.bh); /* @@ -4080,7 +4111,7 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry, goto end_rename; new.bh = ext4_find_entry(new.dir, &new.dentry->d_name, - &new.de, &new.inlined, NULL); + &new.de, &new.inlined); if (IS_ERR(new.bh)) { retval = PTR_ERR(new.bh); new.bh = NULL; diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 3232cb981419..3a28db6cbdcb 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1724,6 +1724,7 @@ static const struct mount_opts { MOPT_EXT4_ONLY | MOPT_CLEAR}, {Opt_warn_on_error, EXT4_MOUNT_WARN_ON_ERROR, MOPT_SET}, {Opt_nowarn_on_error, EXT4_MOUNT_WARN_ON_ERROR, MOPT_CLEAR}, + {Opt_commit, 0, MOPT_NO_EXT2}, {Opt_nojournal_checksum, EXT4_MOUNT_JOURNAL_CHECKSUM, MOPT_EXT4_ONLY | MOPT_CLEAR}, {Opt_journal_checksum, EXT4_MOUNT_JOURNAL_CHECKSUM, diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 653b67062be6..093d7690a97b 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -235,6 +235,9 @@ EXT4_ATTR_FEATURE(casefold); EXT4_ATTR_FEATURE(verity); #endif EXT4_ATTR_FEATURE(metadata_csum_seed); +#if defined(CONFIG_UNICODE) && defined(CONFIG_FS_ENCRYPTION) +EXT4_ATTR_FEATURE(encrypted_casefold); +#endif static struct attribute *ext4_feat_attrs[] = { ATTR_LIST(lazy_itable_init), @@ -251,6 +254,9 @@ static struct attribute *ext4_feat_attrs[] = { ATTR_LIST(verity), #endif ATTR_LIST(metadata_csum_seed), +#if defined(CONFIG_UNICODE) && defined(CONFIG_FS_ENCRYPTION) + ATTR_LIST(encrypted_casefold), +#endif NULL, }; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 10af9ce2e350..899a65232b31 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -352,16 +352,19 @@ void f2fs_drop_inmem_page(struct inode *inode, struct page *page) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct list_head *head = &fi->inmem_pages; struct inmem_pages *cur = NULL; + struct inmem_pages *tmp; f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); mutex_lock(&fi->inmem_lock); - list_for_each_entry(cur, head, list) { - if (cur->page == page) + list_for_each_entry(tmp, head, list) { + if (tmp->page == page) { + cur = tmp; break; + } } - f2fs_bug_on(sbi, list_empty(head) || cur->page != page); + f2fs_bug_on(sbi, !cur); list_del(&cur->list); mutex_unlock(&fi->inmem_lock); diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 218487dd9f02..3764165ceff5 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -531,11 +531,10 @@ static inline int reserved_sections(struct f2fs_sb_info *sbi) return GET_SEC_FROM_SEG(sbi, (unsigned int)reserved_segments(sbi)); } -static inline bool has_curseg_enough_space(struct f2fs_sb_info *sbi) +static inline bool has_curseg_enough_space(struct f2fs_sb_info *sbi, + unsigned int node_blocks, unsigned int dent_blocks) { - unsigned int node_blocks = get_pages(sbi, F2FS_DIRTY_NODES) + - get_pages(sbi, F2FS_DIRTY_DENTS); - unsigned int dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS); + unsigned int segno, left_blocks; int i; @@ -561,19 +560,28 @@ static inline bool has_curseg_enough_space(struct f2fs_sb_info *sbi) static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi, int freed, int needed) { - int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); - int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); - int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); + unsigned int total_node_blocks = get_pages(sbi, F2FS_DIRTY_NODES) + + get_pages(sbi, F2FS_DIRTY_DENTS) + + get_pages(sbi, F2FS_DIRTY_IMETA); + unsigned int total_dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS); + unsigned int node_secs = total_node_blocks / BLKS_PER_SEC(sbi); + unsigned int dent_secs = total_dent_blocks / BLKS_PER_SEC(sbi); + unsigned int node_blocks = total_node_blocks % BLKS_PER_SEC(sbi); + unsigned int dent_blocks = total_dent_blocks % BLKS_PER_SEC(sbi); + unsigned int free, need_lower, need_upper; if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) return false; - if (free_sections(sbi) + freed == reserved_sections(sbi) + needed && - has_curseg_enough_space(sbi)) + free = free_sections(sbi) + freed; + need_lower = node_secs + dent_secs + reserved_sections(sbi) + needed; + need_upper = need_lower + (node_blocks ? 1 : 0) + (dent_blocks ? 1 : 0); + + if (free > need_upper) return false; - return (free_sections(sbi) + freed) <= - (node_secs + 2 * dent_secs + imeta_secs + - reserved_sections(sbi) + needed); + else if (free <= need_lower) + return true; + return !has_curseg_enough_space(sbi, node_blocks, dent_blocks); } static inline bool f2fs_is_checkpoint_ready(struct f2fs_sb_info *sbi) diff --git a/fs/fat/fatent.c b/fs/fat/fatent.c index 4c6c635bc8aa..5e35307a3d6b 100644 --- a/fs/fat/fatent.c +++ b/fs/fat/fatent.c @@ -93,7 +93,8 @@ static int fat12_ent_bread(struct super_block *sb, struct fat_entry *fatent, err_brelse: brelse(bhs[0]); err: - fat_msg(sb, KERN_ERR, "FAT read failed (blocknr %llu)", (llu)blocknr); + fat_msg_ratelimit(sb, KERN_ERR, "FAT read failed (blocknr %llu)", + (llu)blocknr); return -EIO; } @@ -106,8 +107,8 @@ static int fat_ent_bread(struct super_block *sb, struct fat_entry *fatent, fatent->fat_inode = MSDOS_SB(sb)->fat_inode; fatent->bhs[0] = sb_bread(sb, blocknr); if (!fatent->bhs[0]) { - fat_msg(sb, KERN_ERR, "FAT read failed (blocknr %llu)", - (llu)blocknr); + fat_msg_ratelimit(sb, KERN_ERR, "FAT read failed (blocknr %llu)", + (llu)blocknr); return -EIO; } fatent->nr_bhs = 1; diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 869a34a48958..4d687e2e2373 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -1568,11 +1568,12 @@ static long writeback_sb_inodes(struct super_block *sb, }; unsigned long start_time = jiffies; long write_chunk; - long wrote = 0; /* count both pages and inodes */ + long total_wrote = 0; /* count both pages and inodes */ while (!list_empty(&wb->b_io)) { struct inode *inode = wb_inode(wb->b_io.prev); struct bdi_writeback *tmp_wb; + long wrote; if (inode->i_sb != sb) { if (work->sb) { @@ -1648,7 +1649,9 @@ static long writeback_sb_inodes(struct super_block *sb, wbc_detach_inode(&wbc); work->nr_pages -= write_chunk - wbc.nr_to_write; - wrote += write_chunk - wbc.nr_to_write; + wrote = write_chunk - wbc.nr_to_write - wbc.pages_skipped; + wrote = wrote < 0 ? 0 : wrote; + total_wrote += wrote; if (need_resched()) { /* @@ -1670,7 +1673,7 @@ static long writeback_sb_inodes(struct super_block *sb, tmp_wb = inode_to_wb_and_lock_list(inode); spin_lock(&inode->i_lock); if (!(inode->i_state & I_DIRTY_ALL)) - wrote++; + total_wrote++; requeue_inode(inode, tmp_wb, &wbc); inode_sync_complete(inode); spin_unlock(&inode->i_lock); @@ -1684,14 +1687,14 @@ static long writeback_sb_inodes(struct super_block *sb, * bail out to wb_writeback() often enough to check * background threshold and other termination conditions. */ - if (wrote) { + if (total_wrote) { if (time_is_before_jiffies(start_time + HZ / 10UL)) break; if (work->nr_pages <= 0) break; } } - return wrote; + return total_wrote; } static long __writeback_inodes_wb(struct bdi_writeback *wb, diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c index d1a6a1a0a4ca..0a125cef2af8 100644 --- a/fs/jffs2/fs.c +++ b/fs/jffs2/fs.c @@ -598,6 +598,7 @@ int jffs2_do_fill_super(struct super_block *sb, void *data, int silent) jffs2_free_raw_node_refs(c); kvfree(c->blocks); jffs2_clear_xattr_subsystem(c); + jffs2_sum_exit(c); out_inohash: kfree(c->inocache_list); out_wbuf: diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c index f05805a10a50..1014f2a24697 100644 --- a/fs/jfs/jfs_dmap.c +++ b/fs/jfs/jfs_dmap.c @@ -398,7 +398,8 @@ int dbFree(struct inode *ip, s64 blkno, s64 nblocks) } /* write the last buffer. */ - write_metapage(mp); + if (mp) + write_metapage(mp); IREAD_UNLOCK(ipbmap); diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index a4a538abcaf9..99627d3438e5 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -20,7 +20,15 @@ DEFINE_MUTEX(kernfs_mutex); static DEFINE_SPINLOCK(kernfs_rename_lock); /* kn->parent and ->name */ -static char kernfs_pr_cont_buf[PATH_MAX]; /* protected by rename_lock */ +/* + * Don't use rename_lock to piggy back on pr_cont_buf. We don't want to + * call pr_cont() while holding rename_lock. Because sometimes pr_cont() + * will perform wakeups when releasing console_sem. Holding rename_lock + * will introduce deadlock if the scheduler reads the kernfs_name in the + * wakeup path. + */ +static DEFINE_SPINLOCK(kernfs_pr_cont_lock); +static char kernfs_pr_cont_buf[PATH_MAX]; /* protected by pr_cont_lock */ static DEFINE_SPINLOCK(kernfs_idr_lock); /* root->ino_idr */ #define rb_to_kn(X) rb_entry((X), struct kernfs_node, rb) @@ -229,12 +237,12 @@ void pr_cont_kernfs_name(struct kernfs_node *kn) { unsigned long flags; - spin_lock_irqsave(&kernfs_rename_lock, flags); + spin_lock_irqsave(&kernfs_pr_cont_lock, flags); - kernfs_name_locked(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf)); + kernfs_name(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf)); pr_cont("%s", kernfs_pr_cont_buf); - spin_unlock_irqrestore(&kernfs_rename_lock, flags); + spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags); } /** @@ -248,10 +256,10 @@ void pr_cont_kernfs_path(struct kernfs_node *kn) unsigned long flags; int sz; - spin_lock_irqsave(&kernfs_rename_lock, flags); + spin_lock_irqsave(&kernfs_pr_cont_lock, flags); - sz = kernfs_path_from_node_locked(kn, NULL, kernfs_pr_cont_buf, - sizeof(kernfs_pr_cont_buf)); + sz = kernfs_path_from_node(kn, NULL, kernfs_pr_cont_buf, + sizeof(kernfs_pr_cont_buf)); if (sz < 0) { pr_cont("(error)"); goto out; @@ -265,7 +273,7 @@ void pr_cont_kernfs_path(struct kernfs_node *kn) pr_cont("%s", kernfs_pr_cont_buf); out: - spin_unlock_irqrestore(&kernfs_rename_lock, flags); + spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags); } /** @@ -867,13 +875,12 @@ static struct kernfs_node *kernfs_walk_ns(struct kernfs_node *parent, lockdep_assert_held(&kernfs_mutex); - /* grab kernfs_rename_lock to piggy back on kernfs_pr_cont_buf */ - spin_lock_irq(&kernfs_rename_lock); + spin_lock_irq(&kernfs_pr_cont_lock); len = strlcpy(kernfs_pr_cont_buf, path, sizeof(kernfs_pr_cont_buf)); if (len >= sizeof(kernfs_pr_cont_buf)) { - spin_unlock_irq(&kernfs_rename_lock); + spin_unlock_irq(&kernfs_pr_cont_lock); return NULL; } @@ -885,7 +892,7 @@ static struct kernfs_node *kernfs_walk_ns(struct kernfs_node *parent, parent = kernfs_find_ns(parent, name, ns); } - spin_unlock_irq(&kernfs_rename_lock); + spin_unlock_irq(&kernfs_pr_cont_lock); return parent; } diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 759c834b60fd..f48a11fa78bb 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -2920,6 +2920,10 @@ static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata, } out: + if (opendata->lgp) { + nfs4_lgopen_release(opendata->lgp); + opendata->lgp = NULL; + } if (!opendata->cancelled) nfs4_sequence_free_slot(&opendata->o_res.seq_res); return ret; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 0f1c15859418..a7d638bfb46b 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1897,6 +1897,7 @@ pnfs_update_layout(struct inode *ino, lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); if (lo == NULL) { spin_unlock(&ino->i_lock); + lseg = ERR_PTR(-ENOMEM); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_NOMEM); goto out; @@ -2024,6 +2025,7 @@ pnfs_update_layout(struct inode *ino, lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); if (!lgp) { + lseg = ERR_PTR(-ENOMEM); trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, PNFS_UPDATE_LAYOUT_NOMEM); nfs_layoutget_end(lo); diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index ed5429d18595..78191320f8e2 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6401,16 +6401,12 @@ nfsd4_release_lockowner(struct svc_rqst *rqstp, if (sop->so_is_open_owner || !same_owner_str(sop, owner)) continue; - /* see if there are still any locks associated with it */ - lo = lockowner(sop); - list_for_each_entry(stp, &sop->so_stateids, st_perstateowner) { - if (check_for_locks(stp->st_stid.sc_file, lo)) { - status = nfserr_locks_held; - spin_unlock(&clp->cl_lock); - return status; - } + if (atomic_read(&sop->so_count) != 1) { + spin_unlock(&clp->cl_lock); + return nfserr_locks_held; } + lo = lockowner(sop); nfs4_get_stateowner(sop); break; } diff --git a/fs/notify/fdinfo.c b/fs/notify/fdinfo.c index 86fcf5814279..74aeabbf0ea4 100644 --- a/fs/notify/fdinfo.c +++ b/fs/notify/fdinfo.c @@ -83,16 +83,9 @@ static void inotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark) inode_mark = container_of(mark, struct inotify_inode_mark, fsn_mark); inode = igrab(fsnotify_conn_inode(mark->connector)); if (inode) { - /* - * IN_ALL_EVENTS represents all of the mask bits - * that we expose to userspace. There is at - * least one bit (FS_EVENT_ON_CHILD) which is - * used only internally to the kernel. - */ - u32 mask = mark->mask & IN_ALL_EVENTS; - seq_printf(m, "inotify wd:%x ino:%lx sdev:%x mask:%x ignored_mask:%x ", + seq_printf(m, "inotify wd:%x ino:%lx sdev:%x mask:%x ignored_mask:0 ", inode_mark->wd, inode->i_ino, inode->i_sb->s_dev, - mask, mark->ignored_mask); + inotify_mark_user_mask(mark)); show_mark_fhandle(m, inode); seq_putc(m, '\n'); iput(inode); diff --git a/fs/notify/inotify/inotify.h b/fs/notify/inotify/inotify.h index 7e4578d35b61..5d94c00b1233 100644 --- a/fs/notify/inotify/inotify.h +++ b/fs/notify/inotify/inotify.h @@ -21,6 +21,18 @@ static inline struct inotify_event_info *INOTIFY_E(struct fsnotify_event *fse) return container_of(fse, struct inotify_event_info, fse); } +/* + * INOTIFY_USER_FLAGS represents all of the mask bits that we expose to + * userspace. There is at least one bit (FS_EVENT_ON_CHILD) which is + * used only internally to the kernel. + */ +#define INOTIFY_USER_MASK (IN_ALL_EVENTS | IN_ONESHOT | IN_EXCL_UNLINK) + +static inline __u32 inotify_mark_user_mask(struct fsnotify_mark *fsn_mark) +{ + return fsn_mark->mask & INOTIFY_USER_MASK; +} + extern void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark, struct fsnotify_group *group); extern int inotify_handle_event(struct fsnotify_group *group, diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 9530a2dfabc3..4037dd1b7d1e 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -96,7 +96,7 @@ static inline __u32 inotify_arg_to_mask(u32 arg) mask = (FS_IN_IGNORED | FS_EVENT_ON_CHILD | FS_UNMOUNT); /* mask off the flags used to open the fd */ - mask |= (arg & (IN_ALL_EVENTS | IN_ONESHOT | IN_EXCL_UNLINK)); + mask |= (arg & INOTIFY_USER_MASK); return mask; } diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 09535f6423fc..3afd58170984 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -434,7 +434,7 @@ void fsnotify_free_mark(struct fsnotify_mark *mark) void fsnotify_destroy_mark(struct fsnotify_mark *mark, struct fsnotify_group *group) { - mutex_lock_nested(&group->mark_mutex, SINGLE_DEPTH_NESTING); + mutex_lock(&group->mark_mutex); fsnotify_detach_mark(mark); mutex_unlock(&group->mark_mutex); fsnotify_free_mark(mark); @@ -703,7 +703,7 @@ void fsnotify_clear_marks_by_group(struct fsnotify_group *group, * move marks to free to to_free list in one go and then free marks in * to_free list one by one. */ - mutex_lock_nested(&group->mark_mutex, SINGLE_DEPTH_NESTING); + mutex_lock(&group->mark_mutex); list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) { if ((1U << mark->connector->type) & type_mask) list_move(&mark->g_list, &to_free); @@ -712,7 +712,7 @@ void fsnotify_clear_marks_by_group(struct fsnotify_group *group, clear: while (1) { - mutex_lock_nested(&group->mark_mutex, SINGLE_DEPTH_NESTING); + mutex_lock(&group->mark_mutex); if (list_empty(head)) { mutex_unlock(&group->mark_mutex); break; diff --git a/fs/ocfs2/dlmfs/userdlm.c b/fs/ocfs2/dlmfs/userdlm.c index 9cecf4857195..24dbbbf13827 100644 --- a/fs/ocfs2/dlmfs/userdlm.c +++ b/fs/ocfs2/dlmfs/userdlm.c @@ -449,6 +449,11 @@ int user_dlm_cluster_lock(struct user_lock_res *lockres, } spin_lock(&lockres->l_lock); + if (lockres->l_flags & USER_LOCK_IN_TEARDOWN) { + spin_unlock(&lockres->l_lock); + status = -EAGAIN; + goto bail; + } /* We only compare against the currently granted level * here. If the lock is blocked waiting on a downconvert, @@ -615,7 +620,7 @@ int user_dlm_destroy_lock(struct user_lock_res *lockres) spin_lock(&lockres->l_lock); if (lockres->l_flags & USER_LOCK_IN_TEARDOWN) { spin_unlock(&lockres->l_lock); - return 0; + goto bail; } lockres->l_flags |= USER_LOCK_IN_TEARDOWN; @@ -629,12 +634,17 @@ int user_dlm_destroy_lock(struct user_lock_res *lockres) } if (lockres->l_ro_holders || lockres->l_ex_holders) { + lockres->l_flags &= ~USER_LOCK_IN_TEARDOWN; spin_unlock(&lockres->l_lock); goto bail; } status = 0; if (!(lockres->l_flags & USER_LOCK_ATTACHED)) { + /* + * lock is never requested, leave USER_LOCK_IN_TEARDOWN set + * to avoid new lock request coming in. + */ spin_unlock(&lockres->l_lock); goto bail; } @@ -645,6 +655,10 @@ int user_dlm_destroy_lock(struct user_lock_res *lockres) status = ocfs2_dlm_unlock(conn, &lockres->l_lksb, DLM_LKF_VALBLK); if (status) { + spin_lock(&lockres->l_lock); + lockres->l_flags &= ~USER_LOCK_IN_TEARDOWN; + lockres->l_flags &= ~USER_LOCK_BUSY; + spin_unlock(&lockres->l_lock); user_log_dlm_error("ocfs2_dlm_unlock", status, lockres); goto bail; } diff --git a/fs/proc/generic.c b/fs/proc/generic.c index bab10368a04d..d8b3c6a7173f 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -445,6 +445,9 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, proc_set_user(ent, (*parent)->uid, (*parent)->gid); ent->proc_dops = &proc_misc_dentry_ops; + /* Revalidate everything under /proc/${pid}/net */ + if ((*parent)->proc_dops == &proc_net_dentry_ops) + pde_force_lookup(ent); out: return ent; diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c index 096bcc1e7a8a..68ce173d5590 100644 --- a/fs/proc/proc_net.c +++ b/fs/proc/proc_net.c @@ -342,6 +342,9 @@ static __net_init int proc_net_ns_init(struct net *net) proc_set_user(netd, uid, gid); + /* Seed dentry revalidation for /proc/${pid}/net */ + pde_force_lookup(netd); + err = -EEXIST; net_statd = proc_net_mkdir(net, "stat", netd); if (!net_statd) diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index d64f022e508a..b15c95b3b6e7 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -46,13 +46,18 @@ static inline void hchacha_block(const u32 *state, u32 *out, int nrounds) hchacha_block_generic(state, out, nrounds); } -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv); -static inline void chacha_init_generic(u32 *state, const u32 *key, const u8 *iv) +static inline void chacha_init_consts(u32 *state) { state[0] = 0x61707865; /* "expa" */ state[1] = 0x3320646e; /* "nd 3" */ state[2] = 0x79622d32; /* "2-by" */ state[3] = 0x6b206574; /* "te k" */ +} + +void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv); +static inline void chacha_init_generic(u32 *state, const u32 *key, const u8 *iv) +{ + chacha_init_consts(state); state[4] = key[0]; state[5] = key[1]; state[6] = key[2]; diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index 3fb581bf3b87..334d38f01a1e 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -280,4 +280,11 @@ enum drbg_prefixes { DRBG_PREFIX3 }; +extern int drbg_alloc_state(struct drbg_state *drbg); +extern void drbg_dealloc_state(struct drbg_state *drbg); +extern void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, + bool *pr); +extern const struct drbg_core drbg_cores[]; +extern unsigned short drbg_sec_strength(drbg_flag_t flags); + #endif /* _DRBG_H */ diff --git a/include/crypto/internal/jitterentropy.h b/include/crypto/internal/jitterentropy.h new file mode 100644 index 000000000000..c83fff32d130 --- /dev/null +++ b/include/crypto/internal/jitterentropy.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +extern void *jent_zalloc(unsigned int len); +extern void jent_zfree(void *ptr); +extern int jent_fips_enabled(void); +extern void jent_panic(char *s); +extern void jent_memcpy(void *dest, const void *src, unsigned int n); +extern void jent_get_nstime(__u64 *out); + +struct rand_data; +extern int jent_entropy_init(void); +extern int jent_read_entropy(struct rand_data *ec, unsigned char *data, + unsigned int len); + +extern struct rand_data *jent_entropy_collector_alloc(unsigned int osr, + unsigned int flags); +extern void jent_entropy_collector_free(struct rand_data *entropy_collector); diff --git a/include/crypto/sha.h b/include/crypto/sha.h index 8a46202b1857..5a686d211420 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -112,4 +112,15 @@ extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash); + +/* + * An implementation of SHA-1's compression function. Don't use in new code! + * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't + * the correct way to hash something with SHA-1 (use crypto_shash instead). + */ +#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) +#define SHA1_WORKSPACE_WORDS 16 +void sha_init(__u32 *buf); +void sha_transform(__u32 *digest, const char *data, __u32 *W); + #endif diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2c304fd60a5f..c2f537dd4052 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -689,6 +689,8 @@ void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, struct net_device *netdev); bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); +void unpriv_ebpf_notify(int new_state); + #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); diff --git a/include/linux/diagchar.h b/include/linux/diagchar.h index f379dc7a11f5..79d522d5139a 100644 --- a/include/linux/diagchar.h +++ b/include/linux/diagchar.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2008-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef DIAGCHAR_SHARED @@ -143,7 +144,7 @@ * a new RANGE of SSIDs to the msg_mask_tbl. */ #define MSG_MASK_TBL_CNT 27 -#define APPS_EVENT_LAST_ID 0xD02 +#define APPS_EVENT_LAST_ID 0xD07 #define MSG_SSID_0 0 #define MSG_SSID_0_LAST 134 @@ -950,7 +951,7 @@ static const uint32_t msg_bld_masks_26[] = { /* LOG CODES */ static const uint32_t log_code_last_tbl[] = { 0x0, /* EQUIP ID 0 */ - 0x1D92, /* EQUIP ID 1 */ + 0x1DB2, /* EQUIP ID 1 */ 0x0, /* EQUIP ID 2 */ 0x0, /* EQUIP ID 3 */ 0x4910, /* EQUIP ID 4 */ diff --git a/include/linux/efi.h b/include/linux/efi.h index 9a5d4b499271..ec89e8bcc92f 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -150,6 +150,8 @@ struct capsule_info { size_t page_bytes_remain; }; +int efi_capsule_setup_info(struct capsule_info *cap_info, void *kbuff, + size_t hdr_bytes); int __efi_capsule_setup_info(struct capsule_info *cap_info); /* diff --git a/include/linux/linkage.h b/include/linux/linkage.h index d7618c41f74c..f3ae8f3dea2c 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -75,25 +75,58 @@ #ifdef __ASSEMBLY__ +/* SYM_T_FUNC -- type used by assembler to mark functions */ +#ifndef SYM_T_FUNC +#define SYM_T_FUNC STT_FUNC +#endif + +/* SYM_T_OBJECT -- type used by assembler to mark data */ +#ifndef SYM_T_OBJECT +#define SYM_T_OBJECT STT_OBJECT +#endif + +/* SYM_T_NONE -- type used by assembler to mark entries of unknown type */ +#ifndef SYM_T_NONE +#define SYM_T_NONE STT_NOTYPE +#endif + +/* SYM_A_* -- align the symbol? */ +#define SYM_A_ALIGN ALIGN +#define SYM_A_NONE /* nothing */ + +/* SYM_L_* -- linkage of symbols */ +#define SYM_L_GLOBAL(name) .globl name +#define SYM_L_WEAK(name) .weak name +#define SYM_L_LOCAL(name) /* nothing */ + #ifndef LINKER_SCRIPT #define ALIGN __ALIGN #define ALIGN_STR __ALIGN_STR -#ifndef ENTRY -#define ENTRY(name) \ +/* === DEPRECATED annotations === */ + +#ifndef GLOBAL +/* deprecated, use SYM_DATA*, SYM_ENTRY, or similar */ +#define GLOBAL(name) \ .globl name ASM_NL \ - ALIGN ASM_NL \ name: #endif + +#ifndef ENTRY +/* deprecated, use SYM_FUNC_START */ +#define ENTRY(name) \ + SYM_FUNC_START(name) +#endif #endif /* LINKER_SCRIPT */ #ifndef WEAK +/* deprecated, use SYM_FUNC_START_WEAK* */ #define WEAK(name) \ - .weak name ASM_NL \ - name: + SYM_FUNC_START_WEAK(name) #endif #ifndef END +/* deprecated, use SYM_FUNC_END, SYM_DATA_END, or SYM_END */ #define END(name) \ .size name, .-name #endif @@ -103,11 +136,214 @@ * static analysis tools such as stack depth analyzer. */ #ifndef ENDPROC +/* deprecated, use SYM_FUNC_END */ #define ENDPROC(name) \ - .type name, @function ASM_NL \ - END(name) + SYM_FUNC_END(name) +#endif + +/* === generic annotations === */ + +/* SYM_ENTRY -- use only if you have to for non-paired symbols */ +#ifndef SYM_ENTRY +#define SYM_ENTRY(name, linkage, align...) \ + linkage(name) ASM_NL \ + align ASM_NL \ + name: +#endif + +/* SYM_START -- use only if you have to */ +#ifndef SYM_START +#define SYM_START(name, linkage, align...) \ + SYM_ENTRY(name, linkage, align) +#endif + +/* SYM_END -- use only if you have to */ +#ifndef SYM_END +#define SYM_END(name, sym_type) \ + .type name sym_type ASM_NL \ + .size name, .-name +#endif + +/* === code annotations === */ + +/* + * FUNC -- C-like functions (proper stack frame etc.) + * CODE -- non-C code (e.g. irq handlers with different, special stack etc.) + * + * Objtool validates stack for FUNC, but not for CODE. + * Objtool generates debug info for both FUNC & CODE, but needs special + * annotations for each CODE's start (to describe the actual stack frame). + * + * ALIAS -- does not generate debug info -- the aliased function will + */ + +/* SYM_INNER_LABEL_ALIGN -- only for labels in the middle of code */ +#ifndef SYM_INNER_LABEL_ALIGN +#define SYM_INNER_LABEL_ALIGN(name, linkage) \ + .type name SYM_T_NONE ASM_NL \ + SYM_ENTRY(name, linkage, SYM_A_ALIGN) +#endif + +/* SYM_INNER_LABEL -- only for labels in the middle of code */ +#ifndef SYM_INNER_LABEL +#define SYM_INNER_LABEL(name, linkage) \ + .type name SYM_T_NONE ASM_NL \ + SYM_ENTRY(name, linkage, SYM_A_NONE) +#endif + +/* + * SYM_FUNC_START_LOCAL_ALIAS -- use where there are two local names for one + * function + */ +#ifndef SYM_FUNC_START_LOCAL_ALIAS +#define SYM_FUNC_START_LOCAL_ALIAS(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) +#endif + +/* + * SYM_FUNC_START_ALIAS -- use where there are two global names for one + * function + */ +#ifndef SYM_FUNC_START_ALIAS +#define SYM_FUNC_START_ALIAS(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) #endif +/* SYM_FUNC_START -- use for global functions */ +#ifndef SYM_FUNC_START +/* + * The same as SYM_FUNC_START_ALIAS, but we will need to distinguish these two + * later. + */ +#define SYM_FUNC_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) +#endif + +/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */ +#ifndef SYM_FUNC_START_NOALIGN +#define SYM_FUNC_START_NOALIGN(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) +#endif + +/* SYM_FUNC_START_LOCAL -- use for local functions */ +#ifndef SYM_FUNC_START_LOCAL +/* the same as SYM_FUNC_START_LOCAL_ALIAS, see comment near SYM_FUNC_START */ +#define SYM_FUNC_START_LOCAL(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) +#endif + +/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */ +#ifndef SYM_FUNC_START_LOCAL_NOALIGN +#define SYM_FUNC_START_LOCAL_NOALIGN(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) +#endif + +/* SYM_FUNC_START_WEAK -- use for weak functions */ +#ifndef SYM_FUNC_START_WEAK +#define SYM_FUNC_START_WEAK(name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) +#endif + +/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */ +#ifndef SYM_FUNC_START_WEAK_NOALIGN +#define SYM_FUNC_START_WEAK_NOALIGN(name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_NONE) +#endif + +/* SYM_FUNC_END_ALIAS -- the end of LOCAL_ALIASed or ALIASed function */ +#ifndef SYM_FUNC_END_ALIAS +#define SYM_FUNC_END_ALIAS(name) \ + SYM_END(name, SYM_T_FUNC) +#endif + +/* + * SYM_FUNC_END -- the end of SYM_FUNC_START_LOCAL, SYM_FUNC_START, + * SYM_FUNC_START_WEAK, ... + */ +#ifndef SYM_FUNC_END +/* the same as SYM_FUNC_END_ALIAS, see comment near SYM_FUNC_START */ +#define SYM_FUNC_END(name) \ + SYM_END(name, SYM_T_FUNC) +#endif + +/* SYM_CODE_START -- use for non-C (special) functions */ +#ifndef SYM_CODE_START +#define SYM_CODE_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) +#endif + +/* SYM_CODE_START_NOALIGN -- use for non-C (special) functions, w/o alignment */ +#ifndef SYM_CODE_START_NOALIGN +#define SYM_CODE_START_NOALIGN(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) #endif +/* SYM_CODE_START_LOCAL -- use for local non-C (special) functions */ +#ifndef SYM_CODE_START_LOCAL +#define SYM_CODE_START_LOCAL(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) #endif + +/* + * SYM_CODE_START_LOCAL_NOALIGN -- use for local non-C (special) functions, + * w/o alignment + */ +#ifndef SYM_CODE_START_LOCAL_NOALIGN +#define SYM_CODE_START_LOCAL_NOALIGN(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) +#endif + +/* SYM_CODE_END -- the end of SYM_CODE_START_LOCAL, SYM_CODE_START, ... */ +#ifndef SYM_CODE_END +#define SYM_CODE_END(name) \ + SYM_END(name, SYM_T_NONE) +#endif + +/* === data annotations === */ + +/* SYM_DATA_START -- global data symbol */ +#ifndef SYM_DATA_START +#define SYM_DATA_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) +#endif + +/* SYM_DATA_START -- local data symbol */ +#ifndef SYM_DATA_START_LOCAL +#define SYM_DATA_START_LOCAL(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) +#endif + +/* SYM_DATA_END -- the end of SYM_DATA_START symbol */ +#ifndef SYM_DATA_END +#define SYM_DATA_END(name) \ + SYM_END(name, SYM_T_OBJECT) +#endif + +/* SYM_DATA_END_LABEL -- the labeled end of SYM_DATA_START symbol */ +#ifndef SYM_DATA_END_LABEL +#define SYM_DATA_END_LABEL(name, linkage, label) \ + linkage(label) ASM_NL \ + .type label SYM_T_OBJECT ASM_NL \ + label: \ + SYM_END(name, SYM_T_OBJECT) +#endif + +/* SYM_DATA -- start+end wrapper around simple global data */ +#ifndef SYM_DATA +#define SYM_DATA(name, data...) \ + SYM_DATA_START(name) ASM_NL \ + data ASM_NL \ + SYM_DATA_END(name) +#endif + +/* SYM_DATA_LOCAL -- start+end wrapper around simple local data */ +#ifndef SYM_DATA_LOCAL +#define SYM_DATA_LOCAL(name, data...) \ + SYM_DATA_START_LOCAL(name) ASM_NL \ + data ASM_NL \ + SYM_DATA_END(name) +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* _LINUX_LINKAGE_H */ diff --git a/include/linux/lrng.h b/include/linux/lrng.h new file mode 100644 index 000000000000..62fb1e40c21b --- /dev/null +++ b/include/linux/lrng.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_H +#define _LRNG_H + +#include +#include +#include +#include + +#define kfree_sensitive kzfree + +/* + * struct lrng_drng_cb - cryptographic callback functions defining a DRNG + * @drng_name Name of DRNG + * @drng_alloc: Allocate DRNG -- the provided integer should be used for + * sanity checks. + * return: allocated data structure or PTR_ERR on error + * @drng_dealloc: Deallocate DRNG + * @drng_seed: Seed the DRNG with data of arbitrary length drng: is + * pointer to data structure allocated with drng_alloc + * return: >= 0 on success, < 0 on error + * @drng_generate: Generate random numbers from the DRNG with arbitrary + * length + */ +struct lrng_drng_cb { + const char *(*drng_name)(void); + void *(*drng_alloc)(u32 sec_strength); + void (*drng_dealloc)(void *drng); + int (*drng_seed)(void *drng, const u8 *inbuf, u32 inbuflen); + int (*drng_generate)(void *drng, u8 *outbuf, u32 outbuflen); +}; + +/* + * struct lrng_hash_cb - cryptographic callback functions defining a hash + * @hash_name Name of Hash used for reading entropy pool arbitrary + * length + * @hash_alloc: Allocate the hash for reading the entropy pool + * return: allocated data structure (NULL is success too) + * or ERR_PTR on error + * @hash_dealloc: Deallocate Hash + * @hash_digestsize: Return the digestsize for the used hash to read out + * entropy pool + * hash: is pointer to data structure allocated with + * hash_alloc + * return: size of digest of hash in bytes + * @hash_init: Initialize hash + * hash: is pointer to data structure allocated with + * hash_alloc + * return: 0 on success, < 0 on error + * @hash_update: Update hash operation + * hash: is pointer to data structure allocated with + * hash_alloc + * return: 0 on success, < 0 on error + * @hash_final Final hash operation + * hash: is pointer to data structure allocated with + * hash_alloc + * return: 0 on success, < 0 on error + * @hash_desc_zero Zeroization of hash state buffer + * + * Assumptions: + * + * 1. Hash operation will not sleep + * 2. The hash' volatile state information is provided with *shash by caller. + */ +struct lrng_hash_cb { + const char *(*hash_name)(void); + void *(*hash_alloc)(void); + void (*hash_dealloc)(void *hash); + u32 (*hash_digestsize)(void *hash); + int (*hash_init)(struct shash_desc *shash, void *hash); + int (*hash_update)(struct shash_desc *shash, const u8 *inbuf, + u32 inbuflen); + int (*hash_final)(struct shash_desc *shash, u8 *digest); + void (*hash_desc_zero)(struct shash_desc *shash); +}; + +/* Register cryptographic backend */ +#ifdef CONFIG_LRNG_SWITCH +int lrng_set_drng_cb(const struct lrng_drng_cb *cb); +int lrng_set_hash_cb(const struct lrng_hash_cb *cb); +#else /* CONFIG_LRNG_SWITCH */ +static inline int +lrng_set_drng_cb(const struct lrng_drng_cb *cb) { return -EOPNOTSUPP; } +static inline int +lrng_set_hash_cb(const struct lrng_hash_cb *cb) { return -EOPNOTSUPP; } +#endif /* CONFIG_LRNG_SWITCH */ + +/* Callback to feed events to the scheduler entropy source */ +#ifdef CONFIG_LRNG_SCHED +extern void add_sched_randomness(const struct task_struct *p, int cpu); +#else +static inline void +add_sched_randomness(const struct task_struct *p, int cpu) { } +#endif + +/* + * lrng_get_random_bytes() - Provider of cryptographic strong random numbers + * for kernel-internal usage. + * + * This function is appropriate for in-kernel use cases operating in atomic + * contexts. It will always use the ChaCha20 DRNG and it may be the case that + * it is not fully seeded when being used. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +#ifdef CONFIG_LRNG_DRNG_ATOMIC +void lrng_get_random_bytes(void *buf, int nbytes); +#endif + +/* + * lrng_get_random_bytes_full() - Provider of cryptographic strong + * random numbers for kernel-internal usage from a fully initialized LRNG. + * + * This function will always return random numbers from a fully seeded and + * fully initialized LRNG. + * + * This function is appropriate only for non-atomic use cases as this + * function may sleep. It provides access to the full functionality of LRNG + * including the switchable DRNG support, that may support other DRNGs such + * as the SP800-90A DRBG. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +#ifdef CONFIG_LRNG +void lrng_get_random_bytes_full(void *buf, int nbytes); +#endif + +/* + * lrng_get_random_bytes_min() - Provider of cryptographic strong + * random numbers for kernel-internal usage from at least a minimally seeded + * LRNG, which is not necessarily fully initialized yet (e.g. SP800-90C + * oversampling applied in FIPS mode is not applied yet). + * + * This function is appropriate only for non-atomic use cases as this + * function may sleep. It provides access to the full functionality of LRNG + * including the switchable DRNG support, that may support other DRNGs such + * as the SP800-90A DRBG. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +#ifdef CONFIG_LRNG +void lrng_get_random_bytes_min(void *buf, int nbytes); +#endif + +#endif /* _LRNG_H */ diff --git a/include/linux/lz4.h b/include/linux/lz4.h index 60687cdf3170..c0b6be1f98a2 100644 --- a/include/linux/lz4.h +++ b/include/linux/lz4.h @@ -296,7 +296,7 @@ int LZ4_decompress_fast(const char *source, char *dest, int originalSize); int LZ4_decompress_safe(const char *source, char *dest, int compressedSize, int maxDecompressedSize); -#if 0 +#if IS_ENABLED(CONFIG_EROFS_FS) /** * LZ4_decompress_safe_partial() - Decompress a block of size 'compressedSize' * at position 'source' into buffer 'dest' @@ -322,7 +322,7 @@ int LZ4_decompress_safe(const char *source, char *dest, int compressedSize, * or a negative result in case of error * */ -static int LZ4_decompress_safe_partial(const char *source, char *dest, +int LZ4_decompress_safe_partial(const char *source, char *dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); #endif diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index e8d5e6446fba..4ecad0554338 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -70,7 +70,11 @@ extern char * const migratetype_names[MIGRATE_TYPES]; #ifdef CONFIG_CMA # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) -# define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) +# define is_migrate_cma_page(_page) ({ \ + int mt = get_pageblock_migratetype(_page); \ + bool ret = (mt == MIGRATE_ISOLATE || mt == MIGRATE_CMA) ? true : false; \ + ret; \ +}) # define get_cma_migrate_type() MIGRATE_CMA #else # define is_migrate_cma(migratetype) false diff --git a/include/linux/mtd/cfi.h b/include/linux/mtd/cfi.h index 9b57a9b1b081..4ead3d1559f5 100644 --- a/include/linux/mtd/cfi.h +++ b/include/linux/mtd/cfi.h @@ -293,6 +293,7 @@ struct cfi_private { map_word sector_erase_cmd; unsigned long chipshift; /* Because they're of the same type */ const char *im_name; /* inter_module name for cmdset_setup */ + unsigned long quirks; struct flchip chips[0]; /* per-chip data structure for each chip */ }; diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index a46bfb99b015..eb8de9903c3c 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -42,11 +42,11 @@ * void nodes_shift_right(dst, src, n) Shift right * void nodes_shift_left(dst, src, n) Shift left * - * int first_node(mask) Number lowest set bit, or MAX_NUMNODES - * int next_node(node, mask) Next node past 'node', or MAX_NUMNODES - * int next_node_in(node, mask) Next node past 'node', or wrap to first, + * unsigned int first_node(mask) Number lowest set bit, or MAX_NUMNODES + * unsigend int next_node(node, mask) Next node past 'node', or MAX_NUMNODES + * unsigned int next_node_in(node, mask) Next node past 'node', or wrap to first, * or MAX_NUMNODES - * int first_unset_node(mask) First node not set in mask, or + * unsigned int first_unset_node(mask) First node not set in mask, or * MAX_NUMNODES * * nodemask_t nodemask_of_node(node) Return nodemask with bit 'node' set @@ -153,7 +153,7 @@ static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits) #define node_test_and_set(node, nodemask) \ __node_test_and_set((node), &(nodemask)) -static inline int __node_test_and_set(int node, nodemask_t *addr) +static inline bool __node_test_and_set(int node, nodemask_t *addr) { return test_and_set_bit(node, addr->bits); } @@ -200,7 +200,7 @@ static inline void __nodes_complement(nodemask_t *dstp, #define nodes_equal(src1, src2) \ __nodes_equal(&(src1), &(src2), MAX_NUMNODES) -static inline int __nodes_equal(const nodemask_t *src1p, +static inline bool __nodes_equal(const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { return bitmap_equal(src1p->bits, src2p->bits, nbits); @@ -208,7 +208,7 @@ static inline int __nodes_equal(const nodemask_t *src1p, #define nodes_intersects(src1, src2) \ __nodes_intersects(&(src1), &(src2), MAX_NUMNODES) -static inline int __nodes_intersects(const nodemask_t *src1p, +static inline bool __nodes_intersects(const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { return bitmap_intersects(src1p->bits, src2p->bits, nbits); @@ -216,20 +216,20 @@ static inline int __nodes_intersects(const nodemask_t *src1p, #define nodes_subset(src1, src2) \ __nodes_subset(&(src1), &(src2), MAX_NUMNODES) -static inline int __nodes_subset(const nodemask_t *src1p, +static inline bool __nodes_subset(const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { return bitmap_subset(src1p->bits, src2p->bits, nbits); } #define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES) -static inline int __nodes_empty(const nodemask_t *srcp, unsigned int nbits) +static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits) { return bitmap_empty(srcp->bits, nbits); } #define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES) -static inline int __nodes_full(const nodemask_t *srcp, unsigned int nbits) +static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits) { return bitmap_full(srcp->bits, nbits); } @@ -260,15 +260,15 @@ static inline void __nodes_shift_left(nodemask_t *dstp, > MAX_NUMNODES, then the silly min_ts could be dropped. */ #define first_node(src) __first_node(&(src)) -static inline int __first_node(const nodemask_t *srcp) +static inline unsigned int __first_node(const nodemask_t *srcp) { - return min_t(int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES)); + return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES)); } #define next_node(n, src) __next_node((n), &(src)) -static inline int __next_node(int n, const nodemask_t *srcp) +static inline unsigned int __next_node(int n, const nodemask_t *srcp) { - return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1)); + return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1)); } /* @@ -276,7 +276,7 @@ static inline int __next_node(int n, const nodemask_t *srcp) * the first node in src if needed. Returns MAX_NUMNODES if src is empty. */ #define next_node_in(n, src) __next_node_in((n), &(src)) -int __next_node_in(int node, const nodemask_t *srcp); +unsigned int __next_node_in(int node, const nodemask_t *srcp); static inline void init_nodemask_of_node(nodemask_t *mask, int node) { @@ -296,9 +296,9 @@ static inline void init_nodemask_of_node(nodemask_t *mask, int node) }) #define first_unset_node(mask) __first_unset_node(&(mask)) -static inline int __first_unset_node(const nodemask_t *maskp) +static inline unsigned int __first_unset_node(const nodemask_t *maskp) { - return min_t(int,MAX_NUMNODES, + return min_t(unsigned int, MAX_NUMNODES, find_first_zero_bit(maskp->bits, MAX_NUMNODES)); } @@ -375,14 +375,13 @@ static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp, } #if MAX_NUMNODES > 1 -#define for_each_node_mask(node, mask) \ - for ((node) = first_node(mask); \ - (node) < MAX_NUMNODES; \ - (node) = next_node((node), (mask))) +#define for_each_node_mask(node, mask) \ + for ((node) = first_node(mask); \ + (node >= 0) && (node) < MAX_NUMNODES; \ + (node) = next_node((node), (mask))) #else /* MAX_NUMNODES == 1 */ -#define for_each_node_mask(node, mask) \ - if (!nodes_empty(mask)) \ - for ((node) = 0; (node) < 1; (node)++) +#define for_each_node_mask(node, mask) \ + for ((node) = 0; (node) < 1 && !nodes_empty(mask); (node)++) #endif /* MAX_NUMNODES */ /* @@ -435,11 +434,11 @@ static inline int num_node_state(enum node_states state) #define first_online_node first_node(node_states[N_ONLINE]) #define first_memory_node first_node(node_states[N_MEMORY]) -static inline int next_online_node(int nid) +static inline unsigned int next_online_node(int nid) { return next_node(nid, node_states[N_ONLINE]); } -static inline int next_memory_node(int nid) +static inline unsigned int next_memory_node(int nid) { return next_node(nid, node_states[N_MEMORY]); } diff --git a/include/linux/psi.h b/include/linux/psi.h index af892c290116..aca9f636e870 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -30,7 +30,7 @@ void cgroup_move_task(struct task_struct *p, struct css_set *to); struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, size_t nbytes, enum psi_res res); -void psi_trigger_replace(void **trigger_ptr, struct psi_trigger *t); +void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, poll_table *wait); diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index 1249c9fad5a0..290cecfe4c2f 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -121,9 +121,6 @@ struct psi_trigger { */ u64 last_event_time; - /* Refcounting to prevent premature destruction */ - struct kref refcount; - /* * Stall time growth for the last event in ns. */ diff --git a/include/linux/ptrace.h b/include/linux/ptrace.h index 561feb560619..d41de55cdfb9 100644 --- a/include/linux/ptrace.h +++ b/include/linux/ptrace.h @@ -40,12 +40,6 @@ extern int ptrace_access_vm(struct task_struct *tsk, unsigned long addr, #define PT_EXITKILL (PTRACE_O_EXITKILL << PT_OPT_FLAG_SHIFT) #define PT_SUSPEND_SECCOMP (PTRACE_O_SUSPEND_SECCOMP << PT_OPT_FLAG_SHIFT) -/* single stepping state bits (used on ARM and PA-RISC) */ -#define PT_SINGLESTEP_BIT 31 -#define PT_SINGLESTEP (1< +#include #include #include @@ -136,4 +138,24 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) } #endif +/* + * Called from the boot CPU during startup; not valid to call once + * secondary CPUs are up and preemption is possible. + */ +#ifndef arch_get_random_seed_long_early +static inline bool __init arch_get_random_seed_long_early(unsigned long *v) +{ + WARN_ON(system_state != SYSTEM_BOOTING); + return arch_get_random_seed_long(v); +} +#endif + +#ifndef arch_get_random_long_early +static inline bool __init arch_get_random_long_early(unsigned long *v) +{ + WARN_ON(system_state != SYSTEM_BOOTING); + return arch_get_random_long(v); +} +#endif + #endif /* _LINUX_RANDOM_H */ diff --git a/include/net/if_inet6.h b/include/net/if_inet6.h index d7578cf49c3a..3953003b0490 100644 --- a/include/net/if_inet6.h +++ b/include/net/if_inet6.h @@ -69,6 +69,14 @@ struct inet6_ifaddr { struct hlist_node addr_lst; struct list_head if_list; + /* + * Used to safely traverse idev->addr_list in process context + * if the idev->lock needed to protect idev->addr_list cannot be held. + * In that case, add the items to this list temporarily and iterate + * without holding idev->lock. + * See addrconf_ifdown and dev_forward_change. + */ + struct list_head if_list_aux; struct list_head tmp_list; struct inet6_ifaddr *ifpub; diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index fa5fe23ca6aa..2d04f3e06de1 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -407,7 +407,7 @@ static inline void sk_rcv_saddr_set(struct sock *sk, __be32 addr) } int __inet_hash_connect(struct inet_timewait_death_row *death_row, - struct sock *sk, u32 port_offset, + struct sock *sk, u64 port_offset, int (*check_established)(struct inet_timewait_death_row *, struct sock *, __u16, struct inet_timewait_sock **)); diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index 2a3e0974a6af..4e3fff9f929b 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -58,8 +58,13 @@ static inline int nf_conntrack_confirm(struct sk_buff *skb) int ret = NF_ACCEPT; if (ct) { - if (!nf_ct_is_confirmed(ct)) + if (!nf_ct_is_confirmed(ct)) { ret = __nf_conntrack_confirm(skb); + + if (ret == NF_ACCEPT) + ct = (struct nf_conn *)skb_nfct(skb); + } + if (likely(ret == NF_ACCEPT)) nf_ct_deliver_cached_events(ct); } diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h index d7d2495f83c2..dac91aa38c5a 100644 --- a/include/net/secure_seq.h +++ b/include/net/secure_seq.h @@ -4,8 +4,8 @@ #include -u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); -u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, +u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); +u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport); u32 secure_tcp_seq(__be32 saddr, __be32 daddr, __be16 sport, __be16 dport); diff --git a/include/scsi/libfcoe.h b/include/scsi/libfcoe.h index eafccb2d4d6f..71a00fb2279f 100644 --- a/include/scsi/libfcoe.h +++ b/include/scsi/libfcoe.h @@ -261,7 +261,8 @@ int fcoe_ctlr_recv_flogi(struct fcoe_ctlr *, struct fc_lport *, struct fc_frame *); /* libfcoe funcs */ -u64 fcoe_wwn_from_mac(unsigned char mac[MAX_ADDR_LEN], unsigned int, unsigned int); +u64 fcoe_wwn_from_mac(unsigned char mac[ETH_ALEN], unsigned int scheme, + unsigned int port); int fcoe_libfc_config(struct fc_lport *, struct fcoe_ctlr *, const struct libfc_function_template *, int init_fcp); u32 fcoe_fc_crc(struct fc_frame *fp); diff --git a/include/soc/qcom/scm.h b/include/soc/qcom/scm.h index ae4ec932d07f..8f1a419bf8a0 100644 --- a/include/soc/qcom/scm.h +++ b/include/soc/qcom/scm.h @@ -108,7 +108,6 @@ extern int scm_get_feat_version(u32 feat); extern bool is_scm_armv8(void); extern struct mutex scm_lmh_lock; -extern bool under_scm_call(void); #else @@ -167,9 +166,5 @@ static inline bool scm_is_secure_device(void) return false; } -extern bool under_scm_call(void) -{ - return false; -} #endif #endif diff --git a/drivers/staging/erofs/include/trace/events/erofs.h b/include/trace/events/erofs.h similarity index 85% rename from drivers/staging/erofs/include/trace/events/erofs.h rename to include/trace/events/erofs.h index 852c94176d1b..ecfad7641096 100644 --- a/drivers/staging/erofs/include/trace/events/erofs.h +++ b/include/trace/events/erofs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ #undef TRACE_SYSTEM #define TRACE_SYSTEM erofs @@ -6,6 +6,9 @@ #define _TRACE_EROFS_H #include +#include + +struct erofs_map_blocks; #define show_dev(dev) MAJOR(dev), MINOR(dev) #define show_dev_nid(entry) show_dev(entry->dev), entry->nid @@ -38,7 +41,7 @@ TRACE_EVENT(erofs_lookup, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->nid = EROFS_V(dir)->nid; + __entry->nid = EROFS_I(dir)->nid; __assign_str(name, dentry->d_name.name); __entry->flags = flags; ), @@ -63,7 +66,7 @@ TRACE_EVENT(erofs_fill_inode, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->nid = EROFS_V(inode)->nid; + __entry->nid = EROFS_I(inode)->nid; __entry->blkaddr = erofs_blknr(iloc(EROFS_I_SB(inode), __entry->nid)); __entry->ofs = erofs_blkoff(iloc(EROFS_I_SB(inode), __entry->nid)); __entry->isdir = isdir; @@ -92,7 +95,7 @@ TRACE_EVENT(erofs_readpage, TP_fast_assign( __entry->dev = page->mapping->host->i_sb->s_dev; - __entry->nid = EROFS_V(page->mapping->host)->nid; + __entry->nid = EROFS_I(page->mapping->host)->nid; __entry->dir = S_ISDIR(page->mapping->host->i_mode); __entry->index = page->index; __entry->uptodate = PageUptodate(page); @@ -125,7 +128,7 @@ TRACE_EVENT(erofs_readpages, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->nid = EROFS_V(inode)->nid; + __entry->nid = EROFS_I(inode)->nid; __entry->start = page->index; __entry->nrpage = nrpage; __entry->raw = raw; @@ -154,7 +157,7 @@ DECLARE_EVENT_CLASS(erofs__map_blocks_enter, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->nid = EROFS_V(inode)->nid; + __entry->nid = EROFS_I(inode)->nid; __entry->la = map->m_la; __entry->llen = map->m_llen; __entry->flags = flags; @@ -162,7 +165,8 @@ DECLARE_EVENT_CLASS(erofs__map_blocks_enter, TP_printk("dev = (%d,%d), nid = %llu, la %llu llen %llu flags %s", show_dev_nid(__entry), - __entry->la, __entry->llen, show_map_flags(__entry->flags)) + __entry->la, __entry->llen, + __entry->flags ? show_map_flags(__entry->flags) : "NULL") ); DEFINE_EVENT(erofs__map_blocks_enter, erofs_map_blocks_flatmode_enter, @@ -172,6 +176,13 @@ DEFINE_EVENT(erofs__map_blocks_enter, erofs_map_blocks_flatmode_enter, TP_ARGS(inode, map, flags) ); +DEFINE_EVENT(erofs__map_blocks_enter, z_erofs_map_blocks_iter_enter, + TP_PROTO(struct inode *inode, struct erofs_map_blocks *map, + unsigned int flags), + + TP_ARGS(inode, map, flags) +); + DECLARE_EVENT_CLASS(erofs__map_blocks_exit, TP_PROTO(struct inode *inode, struct erofs_map_blocks *map, unsigned int flags, int ret), @@ -192,7 +203,7 @@ DECLARE_EVENT_CLASS(erofs__map_blocks_exit, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->nid = EROFS_V(inode)->nid; + __entry->nid = EROFS_I(inode)->nid; __entry->flags = flags; __entry->la = map->m_la; __entry->pa = map->m_pa; @@ -204,7 +215,8 @@ DECLARE_EVENT_CLASS(erofs__map_blocks_exit, TP_printk("dev = (%d,%d), nid = %llu, flags %s " "la %llu pa %llu llen %llu plen %llu mflags %s ret %d", - show_dev_nid(__entry), show_map_flags(__entry->flags), + show_dev_nid(__entry), + __entry->flags ? show_map_flags(__entry->flags) : "NULL", __entry->la, __entry->pa, __entry->llen, __entry->plen, show_mflags(__entry->mflags), __entry->ret) ); @@ -216,6 +228,13 @@ DEFINE_EVENT(erofs__map_blocks_exit, erofs_map_blocks_flatmode_exit, TP_ARGS(inode, map, flags, ret) ); +DEFINE_EVENT(erofs__map_blocks_exit, z_erofs_map_blocks_iter_exit, + TP_PROTO(struct inode *inode, struct erofs_map_blocks *map, + unsigned int flags, int ret), + + TP_ARGS(inode, map, flags, ret) +); + TRACE_EVENT(erofs_destroy_inode, TP_PROTO(struct inode *inode), @@ -228,7 +247,7 @@ TRACE_EVENT(erofs_destroy_inode, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->nid = EROFS_V(inode)->nid; + __entry->nid = EROFS_I(inode)->nid; ), TP_printk("dev = (%d,%d), nid = %llu", show_dev_nid(__entry)) diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h index a1cb91342231..7add8c87fe22 100644 --- a/include/trace/events/vmscan.h +++ b/include/trace/events/vmscan.h @@ -294,7 +294,7 @@ TRACE_EVENT(mm_vmscan_lru_isolate, __field(unsigned long, nr_scanned) __field(unsigned long, nr_skipped) __field(unsigned long, nr_taken) - __field(isolate_mode_t, isolate_mode) + __field(unsigned int, isolate_mode) __field(int, lru) ), @@ -305,7 +305,7 @@ TRACE_EVENT(mm_vmscan_lru_isolate, __entry->nr_scanned = nr_scanned; __entry->nr_skipped = nr_skipped; __entry->nr_taken = nr_taken; - __entry->isolate_mode = isolate_mode; + __entry->isolate_mode = (__force unsigned int)isolate_mode; __entry->lru = lru; ), diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 94f60eb438b3..0903d08538dc 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3555,14 +3555,19 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, cgroup_get(cgrp); cgroup_kn_unlock(of->kn); + /* Allow only one trigger per file descriptor */ + if (of->priv) { + cgroup_put(cgrp); + return -EBUSY; + } + new = psi_trigger_create(&cgrp->psi, buf, nbytes, res); if (IS_ERR(new)) { cgroup_put(cgrp); return PTR_ERR(new); } - psi_trigger_replace(&of->priv, new); - + smp_store_release(&of->priv, new); cgroup_put(cgrp); return nbytes; @@ -3597,7 +3602,7 @@ static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of, static void cgroup_pressure_release(struct kernfs_open_file *of) { - psi_trigger_replace(&of->priv, NULL); + psi_trigger_destroy(of->priv); } bool cgroup_psi_enabled(void) diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 9c9a5b12f92f..7c6cd00d0fca 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -469,7 +469,7 @@ void debug_dma_dump_mappings(struct device *dev) * At any time debug_dma_assert_idle() can be called to trigger a * warning if any cachelines in the given page are in the active set. */ -static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT); +static RADIX_TREE(dma_active_cacheline, GFP_ATOMIC); static DEFINE_SPINLOCK(radix_lock); #define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1) #define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT) diff --git a/kernel/power/process.c b/kernel/power/process.c index d76e61606f51..b8ead5043086 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -146,6 +146,7 @@ int freeze_processes(void) pr_cont("\n"); BUG_ON(in_atomic()); +#ifndef CONFIG_ANDROID /* * Now that the whole userspace is frozen we need to disbale * the OOM killer to disallow any further interference with @@ -154,6 +155,7 @@ int freeze_processes(void) */ if (!error && !oom_killer_disable(msecs_to_jiffies(freeze_timeout_msecs))) error = -EBUSY; +#endif if (error) thaw_processes(); @@ -198,7 +200,9 @@ void thaw_processes(void) pm_freezing = false; pm_nosig_freezing = false; +#ifndef CONFIG_ANDROID oom_killer_enable(); +#endif pr_info("Restarting tasks ... "); diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 8a98e3e78c70..bf8360e86f62 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -1121,9 +1121,8 @@ int ptrace_request(struct task_struct *child, long request, return ptrace_resume(child, request, data); case PTRACE_KILL: - if (child->exit_state) /* already dead */ - return 0; - return ptrace_resume(child, request, SIGKILL); + send_sig_info(SIGKILL, SEND_SIG_NOINFO, child); + return 0; #ifdef CONFIG_HAVE_ARCH_TRACEHOOK case PTRACE_GETREGSET: diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f3f006bb8713..0d97e3ca5a3b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5,6 +5,8 @@ * * Copyright (C) 1991-2002 Linus Torvalds */ +#include + #include "sched.h" #include @@ -2262,6 +2264,8 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) { struct rq *rq; + add_sched_randomness(p, cpu); + if (!schedstat_enabled()) return; diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 6cd27dcba775..984661d47c90 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1059,7 +1059,6 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, t->event = 0; t->last_event_time = 0; init_waitqueue_head(&t->event_wait); - kref_init(&t->refcount); mutex_lock(&group->trigger_lock); @@ -1092,15 +1091,19 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, return t; } -static void psi_trigger_destroy(struct kref *ref) +void psi_trigger_destroy(struct psi_trigger *t) { - struct psi_trigger *t = container_of(ref, struct psi_trigger, refcount); - struct psi_group *group = t->group; + struct psi_group *group; struct kthread_worker *kworker_to_destroy = NULL; - if (static_branch_likely(&psi_disabled)) + /* + * We do not check psi_disabled since it might have been disabled after + * the trigger got created. + */ + if (!t) return; + group = t->group; /* * Wakeup waiters to stop polling. Can happen if cgroup is deleted * from under a polling process. @@ -1135,9 +1138,9 @@ static void psi_trigger_destroy(struct kref *ref) mutex_unlock(&group->trigger_lock); /* - * Wait for both *trigger_ptr from psi_trigger_replace and - * poll_kworker RCUs to complete their read-side critical sections - * before destroying the trigger and optionally the poll_kworker + * Wait for psi_schedule_poll_work RCU to complete its read-side + * critical section before destroying the trigger and optionally the + * poll_task. */ synchronize_rcu(); /* @@ -1159,18 +1162,6 @@ static void psi_trigger_destroy(struct kref *ref) kfree(t); } -void psi_trigger_replace(void **trigger_ptr, struct psi_trigger *new) -{ - struct psi_trigger *old = *trigger_ptr; - - if (static_branch_likely(&psi_disabled)) - return; - - rcu_assign_pointer(*trigger_ptr, new); - if (old) - kref_put(&old->refcount, psi_trigger_destroy); -} - __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, poll_table *wait) { @@ -1180,24 +1171,15 @@ __poll_t psi_trigger_poll(void **trigger_ptr, if (static_branch_likely(&psi_disabled)) return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI; - rcu_read_lock(); - - t = rcu_dereference(*(void __rcu __force **)trigger_ptr); - if (!t) { - rcu_read_unlock(); + t = smp_load_acquire(trigger_ptr); + if (!t) return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI; - } - kref_get(&t->refcount); - - rcu_read_unlock(); poll_wait(file, &t->event_wait, wait); if (cmpxchg(&t->event, 1, 0) == 1) ret |= EPOLLPRI; - kref_put(&t->refcount, psi_trigger_destroy); - return ret; } @@ -1221,14 +1203,24 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, buf[buf_size - 1] = '\0'; - new = psi_trigger_create(&psi_system, buf, nbytes, res); - if (IS_ERR(new)) - return PTR_ERR(new); - seq = file->private_data; + /* Take seq->lock to protect seq->private from concurrent writes */ mutex_lock(&seq->lock); - psi_trigger_replace(&seq->private, new); + + /* Allow only one trigger per file descriptor */ + if (seq->private) { + mutex_unlock(&seq->lock); + return -EBUSY; + } + + new = psi_trigger_create(&psi_system, buf, nbytes, res); + if (IS_ERR(new)) { + mutex_unlock(&seq->lock); + return PTR_ERR(new); + } + + smp_store_release(&seq->private, new); mutex_unlock(&seq->lock); return nbytes; @@ -1300,7 +1292,7 @@ static int psi_fop_release(struct inode *inode, struct file *file) { struct seq_file *seq = file->private_data; - psi_trigger_replace(&seq->private, NULL); + psi_trigger_destroy(seq->private); return single_release(inode, file); } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index ee1873cc47db..519063f02aad 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -2332,7 +2332,7 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, } EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); -static DEFINE_SPINLOCK(tracepoint_iter_lock); +static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock); static DEFINE_MUTEX(tracepoint_printk_mutex); static void output_printk(struct trace_event_buffer *fbuffer) @@ -2353,14 +2353,14 @@ static void output_printk(struct trace_event_buffer *fbuffer) event = &fbuffer->trace_file->event_call->event; - spin_lock_irqsave(&tracepoint_iter_lock, flags); + raw_spin_lock_irqsave(&tracepoint_iter_lock, flags); trace_seq_init(&iter->seq); iter->ent = fbuffer->entry; event_call->event.funcs->trace(iter, 0, event); trace_seq_putc(&iter->seq, 0); printk("%s", iter->seq.buffer); - spin_unlock_irqrestore(&tracepoint_iter_lock, flags); + raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags); } int tracepoint_printk_sysctl(struct ctl_table *table, int write, @@ -5374,12 +5374,18 @@ static void tracing_set_nop(struct trace_array *tr) tr->current_trace = &nop_trace; } +static bool tracer_options_updated; + static void add_tracer_options(struct trace_array *tr, struct tracer *t) { /* Only enable if the directory has been created already. */ if (!tr->dir) return; + /* Only create trace option files after update_tracer_options finish */ + if (!tracer_options_updated) + return; + create_trace_option_files(tr, t); } @@ -7831,6 +7837,7 @@ static void __update_tracer_options(struct trace_array *tr) static void update_tracer_options(struct trace_array *tr) { mutex_lock(&trace_types_lock); + tracer_options_updated = true; __update_tracer_options(tr); mutex_unlock(&trace_types_lock); } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index d754b1c45842..3821ab54738f 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -2456,8 +2456,11 @@ static int init_var_ref(struct hist_field *ref_field, return err; free: kfree(ref_field->system); + ref_field->system = NULL; kfree(ref_field->event_name); + ref_field->event_name = NULL; kfree(ref_field->name); + ref_field->name = NULL; goto out; } diff --git a/lib/Kconfig b/lib/Kconfig index 29c2fb59feb5..7a6e8a33fd60 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -634,3 +634,6 @@ config GENERIC_LIB_CMPDI2 config GENERIC_LIB_UCMPDI2 bool + +config CRYPTO_LIB_SHA256 + tristate diff --git a/lib/Makefile b/lib/Makefile index 424494d6160d..22f122565d9a 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -291,3 +291,6 @@ obj-$(CONFIG_GENERIC_LIB_LSHRDI3) += lshrdi3.o obj-$(CONFIG_GENERIC_LIB_MULDI3) += muldi3.o obj-$(CONFIG_GENERIC_LIB_CMPDI2) += cmpdi2.o obj-$(CONFIG_GENERIC_LIB_UCMPDI2) += ucmpdi2.o + +obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o +libsha256-y := sha256.o diff --git a/lib/assoc_array.c b/lib/assoc_array.c index 59875eb278ea..3b1ff063ceca 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -1465,6 +1465,7 @@ int assoc_array_gc(struct assoc_array *array, struct assoc_array_ptr *cursor, *ptr; struct assoc_array_ptr *new_root, *new_parent, **new_ptr_pp; unsigned long nr_leaves_on_tree; + bool retained; int keylen, slot, nr_free, next_slot, i; pr_devel("-->%s()\n", __func__); @@ -1541,6 +1542,7 @@ int assoc_array_gc(struct assoc_array *array, goto descend; } +retry_compress: pr_devel("-- compress node %p --\n", new_n); /* Count up the number of empty slots in this node and work out the @@ -1558,6 +1560,7 @@ int assoc_array_gc(struct assoc_array *array, pr_devel("free=%d, leaves=%lu\n", nr_free, new_n->nr_leaves_on_branch); /* See what we can fold in */ + retained = false; next_slot = 0; for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) { struct assoc_array_shortcut *s; @@ -1607,9 +1610,14 @@ int assoc_array_gc(struct assoc_array *array, pr_devel("[%d] retain node %lu/%d [nx %d]\n", slot, child->nr_leaves_on_branch, nr_free + 1, next_slot); + retained = true; } } + if (retained && new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) { + pr_devel("internal nodes remain despite enough space, retrying\n"); + goto retry_compress; + } pr_devel("after: %lu\n", new_n->nr_leaves_on_branch); nr_leaves_on_tree = new_n->nr_leaves_on_branch; diff --git a/lib/crc32.c b/lib/crc32.c index 1a5d08470044..13d865cb2154 100644 --- a/lib/crc32.c +++ b/lib/crc32.c @@ -183,29 +183,31 @@ static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p, } #if CRC_LE_BITS == 1 -u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_le(u32 crc, unsigned char const *p, size_t len) { return crc32_le_generic(crc, p, len, NULL, CRC32_POLY_LE); } -u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len) { return crc32_le_generic(crc, p, len, NULL, CRC32C_POLY_LE); } #else -u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_le(u32 crc, unsigned char const *p, size_t len) { - return crc32_le_generic(crc, p, len, - (const u32 (*)[256])crc32table_le, CRC32_POLY_LE); + return crc32_le_generic(crc, p, len, crc32table_le, CRC32_POLY_LE); } -u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len) { - return crc32_le_generic(crc, p, len, - (const u32 (*)[256])crc32ctable_le, CRC32C_POLY_LE); + return crc32_le_generic(crc, p, len, crc32ctable_le, CRC32C_POLY_LE); } #endif EXPORT_SYMBOL(crc32_le); EXPORT_SYMBOL(__crc32c_le); +u32 __pure crc32_le_base(u32, unsigned char const *, size_t) __alias(crc32_le); +u32 __pure __crc32c_le_base(u32, unsigned char const *, size_t) __alias(__crc32c_le); +u32 __pure crc32_be_base(u32, unsigned char const *, size_t) __alias(crc32_be); + /* * This multiplies the polynomials x and y modulo the given modulus. * This follows the "little-endian" CRC convention that the lsbit @@ -329,15 +331,14 @@ static inline u32 __pure crc32_be_generic(u32 crc, unsigned char const *p, } #if CRC_BE_BITS == 1 -u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_be(u32 crc, unsigned char const *p, size_t len) { return crc32_be_generic(crc, p, len, NULL, CRC32_POLY_BE); } #else -u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_be(u32 crc, unsigned char const *p, size_t len) { - return crc32_be_generic(crc, p, len, - (const u32 (*)[256])crc32table_be, CRC32_POLY_BE); + return crc32_be_generic(crc, p, len, crc32table_be, CRC32_POLY_BE); } #endif EXPORT_SYMBOL(crc32_be); diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c index df2d86bc29a8..63a22bac6044 100644 --- a/lib/lz4/lz4_decompress.c +++ b/lib/lz4/lz4_decompress.c @@ -270,8 +270,12 @@ static FORCE_INLINE int LZ4_decompress_generic( ip += length; op += length; - /* Necessarily EOF, due to parsing restrictions */ - if (!partialDecoding || (cpy == oend)) + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend - 2))) break; } else { /* may overwrite up to WILDCOPYLENGTH beyond cpy */ @@ -463,8 +467,8 @@ int LZ4_decompress_safe(const char *source, char *dest, noDict, (BYTE *)dest, NULL, 0); } -#if 0 -static int LZ4_decompress_safe_partial(const char *src, char *dst, +#if IS_ENABLED(CONFIG_EROFS_FS) +int LZ4_decompress_safe_partial(const char *src, char *dst, int compressedSize, int targetOutputSize, int dstCapacity) { dstCapacity = min(targetOutputSize, dstCapacity); diff --git a/lib/nodemask.c b/lib/nodemask.c index 3aa454c54c0d..e22647f5181b 100644 --- a/lib/nodemask.c +++ b/lib/nodemask.c @@ -3,9 +3,9 @@ #include #include -int __next_node_in(int node, const nodemask_t *srcp) +unsigned int __next_node_in(int node, const nodemask_t *srcp) { - int ret = __next_node(node, srcp); + unsigned int ret = __next_node(node, srcp); if (ret == MAX_NUMNODES) ret = __first_node(srcp); diff --git a/lib/string.c b/lib/string.c index f7f7770444bf..fc626e5957eb 100644 --- a/lib/string.c +++ b/lib/string.c @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -569,21 +570,13 @@ EXPORT_SYMBOL(strnlen); size_t strspn(const char *s, const char *accept) { const char *p; - const char *a; - size_t count = 0; for (p = s; *p != '\0'; ++p) { - for (a = accept; *a != '\0'; ++a) { - if (*p == *a) - break; - } - if (*a == '\0') - return count; - ++count; + if (!strchr(accept, *p)) + break; } - return count; + return p - s; } - EXPORT_SYMBOL(strspn); #endif @@ -596,17 +589,12 @@ EXPORT_SYMBOL(strspn); size_t strcspn(const char *s, const char *reject) { const char *p; - const char *r; - size_t count = 0; for (p = s; *p != '\0'; ++p) { - for (r = reject; *r != '\0'; ++r) { - if (*p == *r) - return count; - } - ++count; + if (strchr(reject, *p)) + break; } - return count; + return p - s; } EXPORT_SYMBOL(strcspn); #endif @@ -914,6 +902,21 @@ __visible int memcmp(const void *cs, const void *ct, size_t count) const unsigned char *su1, *su2; int res = 0; +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (count >= sizeof(unsigned long)) { + const unsigned long *u1 = cs; + const unsigned long *u2 = ct; + do { + if (get_unaligned(u1) != get_unaligned(u2)) + break; + u1++; + u2++; + count -= sizeof(unsigned long); + } while (count >= sizeof(unsigned long)); + cs = u1; + ct = u2; + } +#endif for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--) if ((res = *su1 - *su2) != 0) break; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 7a1fcf75971a..d0cad088d7dd 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4837,7 +4837,14 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) pud_clear(pud); put_page(virt_to_page(ptep)); mm_dec_nr_pmds(mm); - *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; + /* + * This update of passed address optimizes loops sequentially + * processing addresses in increments of huge page size (PMD_SIZE + * in this case). By clearing the pud, a PUD_SIZE area is unmapped. + * Update address to the 'last page' in the cleared area so that + * calling loop can move to first page past this area. + */ + *addr |= PUD_SIZE - PMD_SIZE; return 1; } #define want_pmd_share() (1) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ddc4f02d561f..ecf912ea871f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -487,8 +487,12 @@ static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page bitidx = pfn_to_bitidx(page, pfn); word_bitidx = bitidx / BITS_PER_LONG; bitidx &= (BITS_PER_LONG-1); - - word = bitmap[word_bitidx]; + /* + * This races, without locks, with set_pfnblock_flags_mask(). Ensure + * a consistent read of the memory array, so that results, even though + * racy, are not corrupted. + */ + word = READ_ONCE(bitmap[word_bitidx]); bitidx += end_bitidx; return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; } diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index c3cfc7a1b746..526594ce508c 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1812,11 +1812,40 @@ static enum fullness_group putback_zspage(struct size_class *class, */ static void lock_zspage(struct zspage *zspage) { - struct page *page = get_first_page(zspage); + struct page *curr_page, *page; - do { - lock_page(page); - } while ((page = get_next_page(page)) != NULL); + /* + * Pages we haven't locked yet can be migrated off the list while we're + * trying to lock them, so we need to be careful and only attempt to + * lock each page under migrate_read_lock(). Otherwise, the page we lock + * may no longer belong to the zspage. This means that we may wait for + * the wrong page to unlock, so we must take a reference to the page + * prior to waiting for it to unlock outside migrate_read_lock(). + */ + while (1) { + migrate_read_lock(zspage); + page = get_first_page(zspage); + if (trylock_page(page)) + break; + get_page(page); + migrate_read_unlock(zspage); + wait_on_page_locked(page); + put_page(page); + } + + curr_page = page; + while ((page = get_next_page(curr_page))) { + if (trylock_page(page)) { + curr_page = page; + } else { + get_page(page); + migrate_read_unlock(zspage); + wait_on_page_locked(page); + put_page(page); + migrate_read_lock(zspage); + } + } + migrate_read_unlock(zspage); } static struct dentry *zs_mount(struct file_system_type *fs_type, diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 1e0a1c0a56b5..14b5288d1432 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -563,19 +563,24 @@ static int sco_sock_connect(struct socket *sock, struct sockaddr *addr, int alen addr->sa_family != AF_BLUETOOTH) return -EINVAL; - if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) - return -EBADFD; + lock_sock(sk); + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { + err = -EBADFD; + goto done; + } - if (sk->sk_type != SOCK_SEQPACKET) - return -EINVAL; + if (sk->sk_type != SOCK_SEQPACKET) { + err = -EINVAL; + goto done; + } hdev = hci_get_route(&sa->sco_bdaddr, &sco_pi(sk)->src, BDADDR_BREDR); - if (!hdev) - return -EHOSTUNREACH; + if (!hdev) { + err = -EHOSTUNREACH; + goto done; + } hci_dev_lock(hdev); - lock_sock(sk); - /* Set destination address and psm */ bacpy(&sco_pi(sk)->dst, &sa->sco_bdaddr); diff --git a/net/core/filter.c b/net/core/filter.c index b87fb20d3c71..64204a5c44a7 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1666,7 +1666,7 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH))) return -EINVAL; - if (unlikely(offset > 0xffff)) + if (unlikely(offset > INT_MAX)) return -EFAULT; if (unlikely(bpf_try_make_writable(skb, offset + len))) return -EFAULT; @@ -1701,7 +1701,7 @@ BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset, { void *ptr; - if (unlikely(offset > 0xffff)) + if (unlikely(offset > INT_MAX)) goto err_clear; ptr = skb_header_pointer(skb, offset, len, to); diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index 3a8128341e6a..6fd25279bee9 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -96,7 +96,7 @@ u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, } EXPORT_SYMBOL(secure_tcpv6_seq); -u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, +u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport) { const struct { @@ -146,7 +146,7 @@ u32 secure_tcp_seq(__be32 saddr, __be32 daddr, } EXPORT_SYMBOL_GPL(secure_tcp_seq); -u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport) +u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport) { net_secret_init(); return siphash_4u32((__force u32)saddr, (__force u32)daddr, diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index c96a5871b49d..0a8aec3f37cc 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -507,7 +507,7 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row, return -EADDRNOTAVAIL; } -static u32 inet_sk_port_offset(const struct sock *sk) +static u64 inet_sk_port_offset(const struct sock *sk) { const struct inet_sock *inet = inet_sk(sk); @@ -714,8 +714,19 @@ void inet_unhash(struct sock *sk) } EXPORT_SYMBOL_GPL(inet_unhash); +/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm + * Note that we use 32bit integers (vs RFC 'short integers') + * because 2^16 is not a multiple of num_ephemeral and this + * property might be used by clever attacker. + * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, + * we use 256 instead to really give more isolation and + * privacy, this only consumes 1 KB of kernel memory. + */ +#define INET_TABLE_PERTURB_SHIFT 8 +static u32 table_perturb[1 << INET_TABLE_PERTURB_SHIFT]; + int __inet_hash_connect(struct inet_timewait_death_row *death_row, - struct sock *sk, u32 port_offset, + struct sock *sk, u64 port_offset, int (*check_established)(struct inet_timewait_death_row *, struct sock *, __u16, struct inet_timewait_sock **)) { @@ -727,7 +738,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, struct inet_bind_bucket *tb; u32 remaining, offset; int ret, i, low, high; - static u32 hint; + u32 index; if (port) { head = &hinfo->bhash[inet_bhashfn(net, port, @@ -752,7 +763,12 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, if (likely(remaining > 1)) remaining &= ~1U; - offset = (hint + port_offset) % remaining; + net_get_random_once(table_perturb, sizeof(table_perturb)); + index = hash_32(port_offset, INET_TABLE_PERTURB_SHIFT); + + offset = READ_ONCE(table_perturb[index]) + port_offset; + offset %= remaining; + /* In first pass we try ports of @low parity. * inet_csk_get_port() does the opposite choice. */ @@ -805,7 +821,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, return -EADDRNOTAVAIL; ok: - hint += i + 2; + WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2); /* Head lock still held and bh's disabled */ inet_bind_hash(sk, tb, port); @@ -828,7 +844,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, int inet_hash_connect(struct inet_timewait_death_row *death_row, struct sock *sk) { - u32 port_offset = 0; + u64 port_offset = 0; if (!inet_sk(sk)->inet_num) port_offset = inet_sk_port_offset(sk); diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 41d0f9bb5191..cf60d0e07965 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -678,21 +678,20 @@ static netdev_tx_t ipgre_xmit(struct sk_buff *skb, } if (dev->header_ops) { - const int pull_len = tunnel->hlen + sizeof(struct iphdr); - if (skb_cow_head(skb, 0)) goto free_skb; tnl_params = (const struct iphdr *)skb->data; - if (pull_len > skb_transport_offset(skb)) - goto free_skb; - /* Pull skb since ip_tunnel_xmit() needs skb->data pointing * to gre header. */ - skb_pull(skb, pull_len); + skb_pull(skb, tunnel->hlen + sizeof(struct iphdr)); skb_reset_mac_header(skb); + + if (skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_start(skb) < skb->data) + goto free_skb; } else { if (skb_cow_head(skb, dev->needed_headroom)) goto free_skb; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index e2d90919764e..2e0b1acb9c87 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -2633,12 +2633,15 @@ static void tcp_mtup_probe_success(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); struct inet_connection_sock *icsk = inet_csk(sk); + u64 val; - /* FIXME: breaks with very large cwnd */ tp->prior_ssthresh = tcp_current_ssthresh(sk); - tp->snd_cwnd = tp->snd_cwnd * - tcp_mss_to_mtu(sk, tp->mss_cache) / - icsk->icsk_mtup.probe_size; + + val = (u64)tp->snd_cwnd * tcp_mss_to_mtu(sk, tp->mss_cache); + do_div(val, icsk->icsk_mtup.probe_size); + WARN_ON_ONCE((u32)val != val); + tp->snd_cwnd = max_t(u32, 1U, val); + tp->snd_cwnd_cnt = 0; tp->snd_cwnd_stamp = tcp_jiffies32; tp->snd_ssthresh = tcp_current_ssthresh(sk); diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 40626a8a48e7..231bf40ef8b7 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -4074,8 +4074,8 @@ int tcp_rtx_synack(const struct sock *sk, struct request_sock *req) tcp_rsk(req)->txhash = net_tx_rndhash(); res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL); if (!res) { - __TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS); - __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); + TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS); + NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); if (unlikely(tcp_passive_fastopen(sk))) tcp_sk(sk)->total_retrans++; trace_tcp_retransmit_synack(sk, req); diff --git a/net/ipv4/xfrm4_protocol.c b/net/ipv4/xfrm4_protocol.c index 8dd0e6ab8606..0e1f5dc2766b 100644 --- a/net/ipv4/xfrm4_protocol.c +++ b/net/ipv4/xfrm4_protocol.c @@ -297,4 +297,3 @@ void __init xfrm4_protocol_init(void) { xfrm_input_register_afinfo(&xfrm4_input_afinfo); } -EXPORT_SYMBOL(xfrm4_protocol_init); diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index a6ed81400096..f4b00a150351 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -741,6 +741,7 @@ static void dev_forward_change(struct inet6_dev *idev) { struct net_device *dev; struct inet6_ifaddr *ifa; + LIST_HEAD(tmp_addr_list); if (!idev) return; @@ -759,14 +760,24 @@ static void dev_forward_change(struct inet6_dev *idev) } } + read_lock_bh(&idev->lock); list_for_each_entry(ifa, &idev->addr_list, if_list) { if (ifa->flags&IFA_F_TENTATIVE) continue; + list_add_tail(&ifa->if_list_aux, &tmp_addr_list); + } + read_unlock_bh(&idev->lock); + + while (!list_empty(&tmp_addr_list)) { + ifa = list_first_entry(&tmp_addr_list, + struct inet6_ifaddr, if_list_aux); + list_del(&ifa->if_list_aux); if (idev->cnf.forwarding) addrconf_join_anycast(ifa); else addrconf_leave_anycast(ifa); } + inet6_netconf_notify_devconf(dev_net(dev), RTM_NEWNETCONF, NETCONFA_FORWARDING, dev->ifindex, &idev->cnf); @@ -3693,7 +3704,8 @@ static int addrconf_ifdown(struct net_device *dev, int how) unsigned long event = how ? NETDEV_UNREGISTER : NETDEV_DOWN; struct net *net = dev_net(dev); struct inet6_dev *idev; - struct inet6_ifaddr *ifa, *tmp; + struct inet6_ifaddr *ifa; + LIST_HEAD(tmp_addr_list); bool keep_addr = false; int state, i; @@ -3781,16 +3793,23 @@ static int addrconf_ifdown(struct net_device *dev, int how) write_lock_bh(&idev->lock); } - list_for_each_entry_safe(ifa, tmp, &idev->addr_list, if_list) { + list_for_each_entry(ifa, &idev->addr_list, if_list) + list_add_tail(&ifa->if_list_aux, &tmp_addr_list); + write_unlock_bh(&idev->lock); + + while (!list_empty(&tmp_addr_list)) { struct fib6_info *rt = NULL; bool keep; + ifa = list_first_entry(&tmp_addr_list, + struct inet6_ifaddr, if_list_aux); + list_del(&ifa->if_list_aux); + addrconf_del_dad_work(ifa); keep = keep_addr && (ifa->flags & IFA_F_PERMANENT) && !addr_is_local(&ifa->addr); - write_unlock_bh(&idev->lock); spin_lock_bh(&ifa->lock); if (keep) { @@ -3821,15 +3840,14 @@ static int addrconf_ifdown(struct net_device *dev, int how) addrconf_leave_solict(ifa->idev, &ifa->addr); } - write_lock_bh(&idev->lock); if (!keep) { + write_lock_bh(&idev->lock); list_del_rcu(&ifa->if_list); + write_unlock_bh(&idev->lock); in6_ifa_put(ifa); } } - write_unlock_bh(&idev->lock); - /* Step 5: Discard anycast and multicast list */ if (how) { ipv6_ac_destroy_dev(idev); @@ -4160,7 +4178,8 @@ static void addrconf_dad_completed(struct inet6_ifaddr *ifp, bool bump_id, send_rs = send_mld && ipv6_accept_ra(ifp->idev) && ifp->idev->cnf.rtr_solicits != 0 && - (dev->flags&IFF_LOOPBACK) == 0; + (dev->flags & IFF_LOOPBACK) == 0 && + (dev->type != ARPHRD_TUNNEL); read_unlock_bh(&ifp->idev->lock); /* While dad is in progress mld report's source address is in6_addrany. diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index d9e2575dad94..d8391921363f 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -311,7 +311,7 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row, return -EADDRNOTAVAIL; } -static u32 inet6_sk_port_offset(const struct sock *sk) +static u64 inet6_sk_port_offset(const struct sock *sk) { const struct inet_sock *inet = inet_sk(sk); @@ -323,7 +323,7 @@ static u32 inet6_sk_port_offset(const struct sock *sk) int inet6_hash_connect(struct inet_timewait_death_row *death_row, struct sock *sk) { - u32 port_offset = 0; + u64 port_offset = 0; if (!inet_sk(sk)->inet_num) port_offset = inet6_sk_port_offset(sk); diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index 8546f94f30d4..a886a8f4c0cb 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -406,7 +406,6 @@ int __init seg6_hmac_init(void) { return seg6_hmac_init_algo(); } -EXPORT_SYMBOL(seg6_hmac_init); int __net_init seg6_hmac_net_init(struct net *net) { diff --git a/net/key/af_key.c b/net/key/af_key.c index 170960ef7e36..af67e0d265c0 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -2836,10 +2836,12 @@ static int pfkey_process(struct sock *sk, struct sk_buff *skb, const struct sadb void *ext_hdrs[SADB_EXT_MAX]; int err; - err = pfkey_broadcast(skb_clone(skb, GFP_KERNEL), GFP_KERNEL, - BROADCAST_PROMISC_ONLY, NULL, sock_net(sk)); - if (err) - return err; + /* Non-zero return value of pfkey_broadcast() does not always signal + * an error and even on an actual error we may still want to process + * the message so rather ignore the return value. + */ + pfkey_broadcast(skb_clone(skb, GFP_KERNEL), GFP_KERNEL, + BROADCAST_PROMISC_ONLY, NULL, sock_net(sk)); memset(ext_hdrs, 0, sizeof(ext_hdrs)); err = parse_exthdrs(skb, hdr, ext_hdrs); @@ -2910,7 +2912,7 @@ static int count_ah_combs(const struct xfrm_tmpl *t) break; if (!aalg->pfkey_supported) continue; - if (aalg_tmpl_set(t, aalg)) + if (aalg_tmpl_set(t, aalg) && aalg->available) sz += sizeof(struct sadb_comb); } return sz + sizeof(struct sadb_prop); @@ -2928,7 +2930,7 @@ static int count_esp_combs(const struct xfrm_tmpl *t) if (!ealg->pfkey_supported) continue; - if (!(ealg_tmpl_set(t, ealg))) + if (!(ealg_tmpl_set(t, ealg) && ealg->available)) continue; for (k = 1; ; k++) { @@ -2939,7 +2941,7 @@ static int count_esp_combs(const struct xfrm_tmpl *t) if (!aalg->pfkey_supported) continue; - if (aalg_tmpl_set(t, aalg)) + if (aalg_tmpl_set(t, aalg) && aalg->available) sz += sizeof(struct sadb_comb); } } diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index d9558ffb8acf..6d95ce1c4a27 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -1638,12 +1638,9 @@ int ieee80211_vif_use_reserved_context(struct ieee80211_sub_if_data *sdata) if (new_ctx->replace_state == IEEE80211_CHANCTX_REPLACE_NONE) { if (old_ctx) - err = ieee80211_vif_use_reserved_reassign(sdata); - else - err = ieee80211_vif_use_reserved_assign(sdata); + return ieee80211_vif_use_reserved_reassign(sdata); - if (err) - return err; + return ieee80211_vif_use_reserved_assign(sdata); } /* diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index dea48696f994..2c2532196ef6 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -1072,6 +1072,9 @@ struct tpt_led_trigger { * a scan complete for an aborted scan. * @SCAN_HW_CANCELLED: Set for our scan work function when the scan is being * cancelled. + * @SCAN_BEACON_WAIT: Set whenever we're passive scanning because of radar/no-IR + * and could send a probe request after receiving a beacon. + * @SCAN_BEACON_DONE: Beacon received, we can now send a probe request */ enum { SCAN_SW_SCANNING, @@ -1080,6 +1083,8 @@ enum { SCAN_COMPLETED, SCAN_ABORTED, SCAN_HW_CANCELLED, + SCAN_BEACON_WAIT, + SCAN_BEACON_DONE, }; /** diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c index 5d2a11777718..de42bcfeda9c 100644 --- a/net/mac80211/scan.c +++ b/net/mac80211/scan.c @@ -222,6 +222,16 @@ void ieee80211_scan_rx(struct ieee80211_local *local, struct sk_buff *skb) if (likely(!sdata1 && !sdata2)) return; + if (test_and_clear_bit(SCAN_BEACON_WAIT, &local->scanning)) { + /* + * we were passive scanning because of radar/no-IR, but + * the beacon/proberesp rx gives us an opportunity to upgrade + * to active scan + */ + set_bit(SCAN_BEACON_DONE, &local->scanning); + ieee80211_queue_delayed_work(&local->hw, &local->scan_work, 0); + } + if (ieee80211_is_probe_resp(mgmt->frame_control)) { struct cfg80211_scan_request *scan_req; struct cfg80211_sched_scan_request *sched_scan_req; @@ -706,6 +716,8 @@ static int __ieee80211_start_scan(struct ieee80211_sub_if_data *sdata, IEEE80211_CHAN_RADAR)) || !req->n_ssids) { next_delay = IEEE80211_PASSIVE_CHANNEL_TIME; + if (req->n_ssids) + set_bit(SCAN_BEACON_WAIT, &local->scanning); } else { ieee80211_scan_state_send_probe(local, &next_delay); next_delay = IEEE80211_CHANNEL_TIME; @@ -886,6 +898,8 @@ static void ieee80211_scan_state_set_channel(struct ieee80211_local *local, !scan_req->n_ssids) { *next_delay = IEEE80211_PASSIVE_CHANNEL_TIME; local->next_scan_state = SCAN_DECISION; + if (scan_req->n_ssids) + set_bit(SCAN_BEACON_WAIT, &local->scanning); return; } @@ -978,6 +992,8 @@ void ieee80211_scan_work(struct work_struct *work) goto out; } + clear_bit(SCAN_BEACON_WAIT, &local->scanning); + /* * as long as no delay is required advance immediately * without scheduling a new work @@ -988,6 +1004,10 @@ void ieee80211_scan_work(struct work_struct *work) goto out_complete; } + if (test_and_clear_bit(SCAN_BEACON_DONE, &local->scanning) && + local->next_scan_state == SCAN_DECISION) + local->next_scan_state = SCAN_SEND_PROBE; + switch (local->next_scan_state) { case SCAN_DECISION: /* if no more bands/channels left, complete scan */ diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 9cc8e92f4b00..ab68076d2cba 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -2167,27 +2167,31 @@ struct nft_expr *nft_expr_init(const struct nft_ctx *ctx, err = nf_tables_expr_parse(ctx, nla, &info); if (err < 0) - goto err1; + goto err_expr_parse; + + err = -EOPNOTSUPP; + if (!(info.ops->type->flags & NFT_EXPR_STATEFUL)) + goto err_expr_stateful; err = -ENOMEM; expr = kzalloc(info.ops->size, GFP_KERNEL); if (expr == NULL) - goto err2; + goto err_expr_stateful; err = nf_tables_newexpr(ctx, &info, expr); if (err < 0) - goto err3; + goto err_expr_new; return expr; -err3: +err_expr_new: kfree(expr); -err2: +err_expr_stateful: owner = info.ops->type->owner; if (info.ops->type->release_ops) info.ops->type->release_ops(info.ops); module_put(owner); -err1: +err_expr_parse: return ERR_PTR(err); } diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index 4e544044fc2d..cc076d535e14 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -193,9 +193,6 @@ static int nft_dynset_init(const struct nft_ctx *ctx, return PTR_ERR(priv->expr); err = -EOPNOTSUPP; - if (!(priv->expr->ops->type->flags & NFT_EXPR_STATEFUL)) - goto err1; - if (priv->expr->ops->type->flags & NFT_EXPR_GC) { if (set->flags & NFT_SET_TIMEOUT) goto err1; diff --git a/net/nfc/core.c b/net/nfc/core.c index 54168bbc07cb..a84f824da051 100644 --- a/net/nfc/core.c +++ b/net/nfc/core.c @@ -1171,6 +1171,7 @@ void nfc_unregister_device(struct nfc_dev *dev) if (dev->rfkill) { rfkill_unregister(dev->rfkill); rfkill_destroy(dev->rfkill); + dev->rfkill = NULL; } dev->shutting_down = true; device_unlock(&dev->dev); diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c index 468efc3660c0..12f5c1870103 100644 --- a/net/rxrpc/call_event.c +++ b/net/rxrpc/call_event.c @@ -429,7 +429,8 @@ void rxrpc_process_call(struct work_struct *work) goto recheck_state; } - if (test_and_clear_bit(RXRPC_CALL_EV_RESEND, &call->events)) { + if (test_and_clear_bit(RXRPC_CALL_EV_RESEND, &call->events) && + call->state != RXRPC_CALL_CLIENT_RECV_REPLY) { rxrpc_resend(call, now); goto recheck_state; } diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index edd76c41765f..0220a2935002 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -440,6 +440,12 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, success: ret = copied; + if (READ_ONCE(call->state) == RXRPC_CALL_COMPLETE) { + read_lock_bh(&call->state_lock); + if (call->error < 0) + ret = call->error; + read_unlock_bh(&call->state_lock); + } out: call->tx_pending = skb; _leave(" = %d", ret); diff --git a/net/rxrpc/sysctl.c b/net/rxrpc/sysctl.c index d75bd15151e6..50f825f55c21 100644 --- a/net/rxrpc/sysctl.c +++ b/net/rxrpc/sysctl.c @@ -17,7 +17,7 @@ static struct ctl_table_header *rxrpc_sysctl_reg_table; static const unsigned int one = 1; static const unsigned int four = 4; -static const unsigned int thirtytwo = 32; +static const unsigned int max_backlog = RXRPC_BACKLOG_MAX - 1; static const unsigned int n_65535 = 65535; static const unsigned int n_max_acks = RXRPC_RXTX_BUFF_SIZE - 1; static const unsigned long one_jiffy = 1; @@ -111,7 +111,7 @@ static struct ctl_table rxrpc_sysctl_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = (void *)&four, - .extra2 = (void *)&thirtytwo, + .extra2 = (void *)&max_backlog, }, { .procname = "rx_window_size", diff --git a/net/sctp/input.c b/net/sctp/input.c index 6a97be0fabb0..bd3af0c9ea8f 100644 --- a/net/sctp/input.c +++ b/net/sctp/input.c @@ -104,6 +104,7 @@ int sctp_rcv(struct sk_buff *skb) struct sctp_chunk *chunk; union sctp_addr src; union sctp_addr dest; + int bound_dev_if; int family; struct sctp_af *af; struct net *net = dev_net(skb->dev); @@ -181,7 +182,8 @@ int sctp_rcv(struct sk_buff *skb) * If a frame arrives on an interface and the receiving socket is * bound to another interface, via SO_BINDTODEVICE, treat it as OOTB */ - if (sk->sk_bound_dev_if && (sk->sk_bound_dev_if != af->skb_iif(skb))) { + bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); + if (bound_dev_if && (bound_dev_if != af->skb_iif(skb))) { if (transport) { sctp_transport_put(transport); asoc = NULL; diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index 540e340e2565..34596d0e4bde 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -544,7 +544,11 @@ static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, */ xdr->p = (void *)p + frag2bytes; space_left = xdr->buf->buflen - xdr->buf->len; - xdr->end = (void *)p + min_t(int, space_left, PAGE_SIZE); + if (space_left - nbytes >= PAGE_SIZE) + xdr->end = (void *)p + PAGE_SIZE; + else + xdr->end = (void *)p + space_left - frag1bytes; + xdr->buf->page_len += frag2bytes; xdr->buf->len += nbytes; return p; diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index f2eaf264726b..3d65a2bccfc7 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -980,6 +980,7 @@ static bool rpcrdma_is_bcall(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) #if defined(CONFIG_SUNRPC_BACKCHANNEL) { + struct rpc_xprt *xprt = &r_xprt->rx_xprt; struct xdr_stream *xdr = &rep->rr_stream; __be32 *p; @@ -1003,6 +1004,10 @@ rpcrdma_is_bcall(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) if (*p != cpu_to_be32(RPC_CALL)) return false; + /* No bc service. */ + if (xprt->bc_serv == NULL) + return false; + /* Now that we are sure this is a backchannel call, * advance to the RPC header. */ diff --git a/net/tipc/bearer.c b/net/tipc/bearer.c index e1006ed4d90a..0f970259d0d5 100644 --- a/net/tipc/bearer.c +++ b/net/tipc/bearer.c @@ -246,9 +246,8 @@ static int tipc_enable_bearer(struct net *net, const char *name, u32 i; if (!bearer_name_validate(name, &b_names)) { - errstr = "illegal name"; NL_SET_ERR_MSG(extack, "Illegal name"); - goto rejected; + return res; } if (prio > TIPC_MAX_LINK_PRI && prio != TIPC_MEDIA_LINK_PRI) { diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 82279dbd2f62..e79c32942796 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -445,7 +445,7 @@ static int unix_dgram_peer_wake_me(struct sock *sk, struct sock *other) * -ECONNREFUSED. Otherwise, if we haven't queued any skbs * to other and its full, we will hang waiting for POLLOUT. */ - if (unix_recvq_full(other) && !sock_flag(other, SOCK_DEAD)) + if (unix_recvq_full_lockless(other) && !sock_flag(other, SOCK_DEAD)) return 1; if (connected) diff --git a/net/wireless/core.c b/net/wireless/core.c index 0f6b44662439..78dceeb4cdfa 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -4,7 +4,7 @@ * Copyright 2006-2010 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -851,9 +851,6 @@ int wiphy_register(struct wiphy *wiphy) return res; } - /* set up regulatory info */ - wiphy_regulatory_register(wiphy); - list_add_rcu(&rdev->list, &cfg80211_rdev_list); cfg80211_rdev_list_generation++; @@ -867,6 +864,9 @@ int wiphy_register(struct wiphy *wiphy) cfg80211_debugfs_rdev_add(rdev); nl80211_notify_wiphy(rdev, NL80211_CMD_NEW_WIPHY); + /* set up regulatory info */ + wiphy_regulatory_register(wiphy); + if (wiphy->regulatory_flags & REGULATORY_CUSTOM_REG) { struct regulatory_request request; diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 4d676412602f..8f01cc67bfbe 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -2993,6 +2993,7 @@ static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flag wdev_lock(wdev); switch (wdev->iftype) { case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: if (wdev->ssid_len && nla_put(msg, NL80211_ATTR_SSID, wdev->ssid_len, wdev->ssid)) goto nla_put_failure_locked; diff --git a/net/wireless/reg.c b/net/wireless/reg.c index e279c49ddd5e..1f4b8ba1e20e 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -3806,6 +3806,7 @@ void wiphy_regulatory_register(struct wiphy *wiphy) wiphy_update_regulatory(wiphy, lr->initiator); wiphy_all_share_dfs_chan_state(wiphy); + reg_process_self_managed_hints(); } void wiphy_regulatory_deregister(struct wiphy *wiphy) diff --git a/scripts/build_kernel b/scripts/build_kernel index 02a1fa187ca3..495a8a8c5402 100755 --- a/scripts/build_kernel +++ b/scripts/build_kernel @@ -16,11 +16,32 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Start cleanup of previous build +if [[ -d anykernel ]]; then + rm -fr $(pwd)/anykernel/ +fi + +if [[ -f ${OUT_DIR}/arch/arm64/boot/dtb ]]; then + rm -fr ${OUT_DIR}/arch/arm64/boot/dtb +fi + +if [[ -f ${OUT_DIR}/arch/arm64/boot/Image.gz ]]; then + rm -fr ${OUT_DIR}/arch/arm64/boot/Image.gz +elif [[ -f ${OUT_DIR}/arch/arm64/boot/Image ]]; then + rm -fr ${OUT_DIR}/arch/arm64/boot/Image +fi + +if [[ -f ${OUT_DIR}/arch/arm64/boot/dtbo.img ]]; then + rm -fr ${OUT_DIR}/arch/arm64/boot/dtbo.img +fi + KBUILD_COMPILER_STRING=$(${HOME}/clang/bin/clang --version | head -n 1 | perl -pe 's/\(http.*?\)//gs' | sed -e 's/ */ /g' -e 's/[[:space:]]*$//') KBUILD_LINKER_STRING=$(${HOME}/clang/bin/ld.lld --version | head -n 1 | perl -pe 's/\(http.*?\)//gs' | sed -e 's/ */ /g' -e 's/[[:space:]]*$//' | sed 's/(compatible with [^)]*)//') export KBUILD_COMPILER_STRING export KBUILD_LINKER_STRING +cp -r $HOME/anykernel ./ + # # Enviromental Variables # @@ -197,9 +218,10 @@ scripts/config --file ${OUT_DIR}/.config \ -e MIGT \ -e MIHW \ -e MILLET \ - -e MIUI_DRM_WAKE_OPT \ + -e MI_DRM_OPT \ -e MIUI_ZRAM_MEMORY_TRACKING \ -e MI_RECLAIM \ + -d OSSFOD \ -e PERF_HUMANTASK \ -e TASK_DELAY_ACCT @@ -285,22 +307,21 @@ DIFF=$((END - START)) cd anykernel || exit zip -r9 "${ZIPNAME}" ./* -x .git .gitignore ./*.zip -#RESPONSE=$(curl -# -F "name=${ZIPNAME}" -F "file=@${ZIPNAME}" -u :"${PD_API_KEY}" https://pixeldrain.com/api/file) -#FILEID=$(echo $RESPONSE | grep -Po '(?<="id":")[^"]*') +FILEID=$(pd upload $ZIPNAME) + +echo $FILEID CHECKER=$(find ./ -maxdepth 1 -type f -name ${ZIPNAME} -printf "%s\n") if (($((CHECKER / 1048576)) > 5)); then curl -s -X POST https://api.telegram.org/bot"${BOT_API_KEY}"/sendMessage -d text="✅ Kernel compiled successfully in $((DIFF / 60)) minute(s) and $((DIFF % 60)) seconds for ${DEVICE}" -d chat_id="${CI_CHANNEL_ID}" -d parse_mode=HTML -# curl -s -X POST https://api.telegram.org/bot"${BOT_API_KEY}"/sendMessage -d text="Kernel build link: https://pixeldrain.com/u/$FILEID" -d chat_id="${CI_CHANNEL_ID}" -d parse_mode=HTML + curl -s -X POST https://api.telegram.org/bot"${BOT_API_KEY}"/sendMessage -d text="Kernel build link: $FILEID" -d chat_id="${CI_CHANNEL_ID}" -d parse_mode=HTML curl -F chat_id="${CI_CHANNEL_ID}" -F document=@"$(pwd)/${ZIPNAME}" https://api.telegram.org/bot"${BOT_API_KEY}"/sendDocument else tg_post_error fi -cd "$(pwd)" || exit - -# Cleanup -rm -fr anykernel/ -rm -fr ${OUT_DIR}/arch/arm64/boot/dtb -rm -fr ${OUT_DIR}/arch/arm64/boot/dtbo.img -rm -fr ${OUT_DIR}/arch/arm64/boot/Image.gz -rm -fr ${OUT_DIR}/arch/arm64/boot/Image +cd $(pwd) || exit + +echo "" +echo -e $ZIPNAME " is ready!" +echo "" + diff --git a/scripts/extract-cert.c b/scripts/extract-cert.c index b071bf476fea..dd1a4bd706a2 100644 --- a/scripts/extract-cert.c +++ b/scripts/extract-cert.c @@ -23,6 +23,13 @@ #include #include +/* + * OpenSSL 3.0 deprecates the OpenSSL's ENGINE API. + * + * Remove this if/when that API is no longer used + */ +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + #define PKEY_ID_PKCS7 2 static __attribute__((noreturn)) diff --git a/scripts/faddr2line b/scripts/faddr2line index a0149db00be7..226c3f559dc5 100755 --- a/scripts/faddr2line +++ b/scripts/faddr2line @@ -44,17 +44,6 @@ set -o errexit set -o nounset -READELF="${CROSS_COMPILE:-}readelf" -ADDR2LINE="${CROSS_COMPILE:-}addr2line" -SIZE="${CROSS_COMPILE:-}size" -NM="${CROSS_COMPILE:-}nm" - -command -v awk >/dev/null 2>&1 || die "awk isn't installed" -command -v ${READELF} >/dev/null 2>&1 || die "readelf isn't installed" -command -v ${ADDR2LINE} >/dev/null 2>&1 || die "addr2line isn't installed" -command -v ${SIZE} >/dev/null 2>&1 || die "size isn't installed" -command -v ${NM} >/dev/null 2>&1 || die "nm isn't installed" - usage() { echo "usage: faddr2line [--list] ..." >&2 exit 1 @@ -69,6 +58,14 @@ die() { exit 1 } +READELF="${CROSS_COMPILE:-}readelf" +ADDR2LINE="${CROSS_COMPILE:-}addr2line" +AWK="awk" + +command -v ${AWK} >/dev/null 2>&1 || die "${AWK} isn't installed" +command -v ${READELF} >/dev/null 2>&1 || die "${READELF} isn't installed" +command -v ${ADDR2LINE} >/dev/null 2>&1 || die "${ADDR2LINE} isn't installed" + # Try to figure out the source directory prefix so we can remove it from the # addr2line output. HACK ALERT: This assumes that start_kernel() is in # kernel/init.c! This only works for vmlinux. Otherwise it falls back to @@ -76,7 +73,7 @@ die() { find_dir_prefix() { local objfile=$1 - local start_kernel_addr=$(${READELF} -sW $objfile | awk '$8 == "start_kernel" {printf "0x%s", $2}') + local start_kernel_addr=$(${READELF} --symbols --wide $objfile | ${AWK} '$8 == "start_kernel" {printf "0x%s", $2}') [[ -z $start_kernel_addr ]] && return local file_line=$(${ADDR2LINE} -e $objfile $start_kernel_addr) @@ -97,86 +94,133 @@ __faddr2line() { local dir_prefix=$3 local print_warnings=$4 - local func=${func_addr%+*} + local sym_name=${func_addr%+*} local offset=${func_addr#*+} offset=${offset%/*} - local size= - [[ $func_addr =~ "/" ]] && size=${func_addr#*/} + local user_size= + [[ $func_addr =~ "/" ]] && user_size=${func_addr#*/} - if [[ -z $func ]] || [[ -z $offset ]] || [[ $func = $func_addr ]]; then + if [[ -z $sym_name ]] || [[ -z $offset ]] || [[ $sym_name = $func_addr ]]; then warn "bad func+offset $func_addr" DONE=1 return fi # Go through each of the object's symbols which match the func name. - # In rare cases there might be duplicates. - file_end=$(${SIZE} -Ax $objfile | awk '$1 == ".text" {print $2}') - while read symbol; do - local fields=($symbol) - local sym_base=0x${fields[0]} - local sym_type=${fields[1]} - local sym_end=${fields[3]} - - # calculate the size - local sym_size=$(($sym_end - $sym_base)) + # In rare cases there might be duplicates, in which case we print all + # matches. + while read line; do + local fields=($line) + local sym_addr=0x${fields[1]} + local sym_elf_size=${fields[2]} + local sym_sec=${fields[6]} + + # Get the section size: + local sec_size=$(${READELF} --section-headers --wide $objfile | + sed 's/\[ /\[/' | + ${AWK} -v sec=$sym_sec '$1 == "[" sec "]" { print "0x" $6; exit }') + + if [[ -z $sec_size ]]; then + warn "bad section size: section: $sym_sec" + DONE=1 + return + fi + + # Calculate the symbol size. + # + # Unfortunately we can't use the ELF size, because kallsyms + # also includes the padding bytes in its size calculation. For + # kallsyms, the size calculation is the distance between the + # symbol and the next symbol in a sorted list. + local sym_size + local cur_sym_addr + local found=0 + while read line; do + local fields=($line) + cur_sym_addr=0x${fields[1]} + local cur_sym_elf_size=${fields[2]} + local cur_sym_name=${fields[7]:-} + + if [[ $cur_sym_addr = $sym_addr ]] && + [[ $cur_sym_elf_size = $sym_elf_size ]] && + [[ $cur_sym_name = $sym_name ]]; then + found=1 + continue + fi + + if [[ $found = 1 ]]; then + sym_size=$(($cur_sym_addr - $sym_addr)) + [[ $sym_size -lt $sym_elf_size ]] && continue; + found=2 + break + fi + done < <(${READELF} --symbols --wide $objfile | ${AWK} -v sec=$sym_sec '$7 == sec' | sort --key=2) + + if [[ $found = 0 ]]; then + warn "can't find symbol: sym_name: $sym_name sym_sec: $sym_sec sym_addr: $sym_addr sym_elf_size: $sym_elf_size" + DONE=1 + return + fi + + # If nothing was found after the symbol, assume it's the last + # symbol in the section. + [[ $found = 1 ]] && sym_size=$(($sec_size - $sym_addr)) + if [[ -z $sym_size ]] || [[ $sym_size -le 0 ]]; then - warn "bad symbol size: base: $sym_base end: $sym_end" + warn "bad symbol size: sym_addr: $sym_addr cur_sym_addr: $cur_sym_addr" DONE=1 return fi + sym_size=0x$(printf %x $sym_size) - # calculate the address - local addr=$(($sym_base + $offset)) + # Calculate the section address from user-supplied offset: + local addr=$(($sym_addr + $offset)) if [[ -z $addr ]] || [[ $addr = 0 ]]; then - warn "bad address: $sym_base + $offset" + warn "bad address: $sym_addr + $offset" DONE=1 return fi addr=0x$(printf %x $addr) - # weed out non-function symbols - if [[ $sym_type != t ]] && [[ $sym_type != T ]]; then - [[ $print_warnings = 1 ]] && - echo "skipping $func address at $addr due to non-function symbol of type '$sym_type'" - continue - fi - - # if the user provided a size, make sure it matches the symbol's size - if [[ -n $size ]] && [[ $size -ne $sym_size ]]; then + # If the user provided a size, make sure it matches the symbol's size: + if [[ -n $user_size ]] && [[ $user_size -ne $sym_size ]]; then [[ $print_warnings = 1 ]] && - echo "skipping $func address at $addr due to size mismatch ($size != $sym_size)" + echo "skipping $sym_name address at $addr due to size mismatch ($user_size != $sym_size)" continue; fi - # make sure the provided offset is within the symbol's range + # Make sure the provided offset is within the symbol's range: if [[ $offset -gt $sym_size ]]; then [[ $print_warnings = 1 ]] && - echo "skipping $func address at $addr due to size mismatch ($offset > $sym_size)" + echo "skipping $sym_name address at $addr due to size mismatch ($offset > $sym_size)" continue fi - # separate multiple entries with a blank line + # In case of duplicates or multiple addresses specified on the + # cmdline, separate multiple entries with a blank line: [[ $FIRST = 0 ]] && echo FIRST=0 - # pass real address to addr2line - echo "$func+$offset/$sym_size:" - local file_lines=$(${ADDR2LINE} -fpie $objfile $addr | sed "s; $dir_prefix\(\./\)*; ;") - [[ -z $file_lines ]] && return + echo "$sym_name+$offset/$sym_size:" + # Pass section address to addr2line and strip absolute paths + # from the output: + local output=$(${ADDR2LINE} -fpie $objfile $addr | sed "s; $dir_prefix\(\./\)*; ;") + [[ -z $output ]] && continue + + # Default output (non --list): if [[ $LIST = 0 ]]; then - echo "$file_lines" | while read -r line + echo "$output" | while read -r line do echo $line done DONE=1; - return + continue fi - # show each line with context - echo "$file_lines" | while read -r line + # For --list, show each line with its corresponding source code: + echo "$output" | while read -r line do echo echo $line @@ -184,12 +228,12 @@ __faddr2line() { n1=$[$n-5] n2=$[$n+5] f=$(echo $line | sed 's/.*at \(.\+\):.*/\1/g') - awk 'NR>=strtonum("'$n1'") && NR<=strtonum("'$n2'") { if (NR=='$n') printf(">%d<", NR); else printf(" %d ", NR); printf("\t%s\n", $0)}' $f + ${AWK} 'NR>=strtonum("'$n1'") && NR<=strtonum("'$n2'") { if (NR=='$n') printf(">%d<", NR); else printf(" %d ", NR); printf("\t%s\n", $0)}' $f done DONE=1 - done < <(${NM} -n $objfile | awk -v fn=$func -v end=$file_end '$3 == fn { found=1; line=$0; start=$1; next } found == 1 { found=0; print line, "0x"$1 } END {if (found == 1) print line, end; }') + done < <(${READELF} --symbols --wide $objfile | ${AWK} -v fn=$sym_name '$4 == "FUNC" && $8 == fn') } [[ $# -lt 2 ]] && usage diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 6129898ee029..1857edcfe844 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1239,7 +1239,8 @@ static int secref_whitelist(const struct sectioncheck *mismatch, static inline int is_arm_mapping_symbol(const char *str) { - return str[0] == '$' && strchr("axtd", str[1]) + return str[0] == '$' && + (str[1] == 'a' || str[1] == 'd' || str[1] == 't' || str[1] == 'x') && (str[2] == '\0' || str[2] == '.'); } @@ -1958,7 +1959,7 @@ static char *remove_dot(char *s) if (n && s[n]) { size_t m = strspn(s + n + 1, "0123456789"); - if (m && (s[n + m] == '.' || s[n + m] == 0)) + if (m && (s[n + m + 1] == '.' || s[n + m + 1] == 0)) s[n] = 0; /* strip trailing .lto */ diff --git a/scripts/sign-file.c b/scripts/sign-file.c index fbd34b8e8f57..7434e9ea926e 100644 --- a/scripts/sign-file.c +++ b/scripts/sign-file.c @@ -29,6 +29,13 @@ #include #include +/* + * OpenSSL 3.0 deprecates the OpenSSL's ENGINE API. + * + * Remove this if/when that API is no longer used + */ +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + /* * Use CMS if we have openssl-1.0.0 or newer available - otherwise we have to * assume that it's not available and its header file is missing and that we diff --git a/sound/pci/hda/patch_conexant.c b/sound/pci/hda/patch_conexant.c index 8851cd11dc9c..629857f5539c 100644 --- a/sound/pci/hda/patch_conexant.c +++ b/sound/pci/hda/patch_conexant.c @@ -1025,6 +1025,13 @@ static int patch_conexant_auto(struct hda_codec *codec) snd_hda_pick_fixup(codec, cxt5051_fixup_models, cxt5051_fixups, cxt_fixups); break; + case 0x14f15098: + codec->pin_amp_workaround = 1; + spec->gen.mixer_nid = 0x22; + spec->gen.add_stereo_mix_input = HDA_HINT_STEREO_MIX_AUTO; + snd_hda_pick_fixup(codec, cxt5066_fixup_models, + cxt5066_fixups, cxt_fixups); + break; case 0x14f150f2: codec->power_save_node = 1; /* Fall through */ diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index a5263453ea64..ed0b8ec50d71 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -1916,6 +1916,7 @@ enum { ALC1220_FIXUP_CLEVO_PB51ED_PINS, ALC887_FIXUP_ASUS_AUDIO, ALC887_FIXUP_ASUS_HMIC, + ALCS1200A_FIXUP_MIC_VREF, }; static void alc889_fixup_coef(struct hda_codec *codec, @@ -2461,6 +2462,14 @@ static const struct hda_fixup alc882_fixups[] = { .chained = true, .chain_id = ALC887_FIXUP_ASUS_AUDIO, }, + [ALCS1200A_FIXUP_MIC_VREF] = { + .type = HDA_FIXUP_PINCTLS, + .v.pins = (const struct hda_pintbl[]) { + { 0x18, PIN_VREF50 }, /* rear mic */ + { 0x19, PIN_VREF50 }, /* front mic */ + {} + } + }, }; static const struct snd_pci_quirk alc882_fixup_tbl[] = { @@ -2498,6 +2507,7 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x835f, "Asus Eee 1601", ALC888_FIXUP_EEE1601), SND_PCI_QUIRK(0x1043, 0x84bc, "ASUS ET2700", ALC887_FIXUP_ASUS_BASS), SND_PCI_QUIRK(0x1043, 0x8691, "ASUS ROG Ranger VIII", ALC882_FIXUP_GPIO3), + SND_PCI_QUIRK(0x1043, 0x8797, "ASUS TUF B550M-PLUS", ALCS1200A_FIXUP_MIC_VREF), SND_PCI_QUIRK(0x104d, 0x9043, "Sony Vaio VGC-LN51JGB", ALC882_FIXUP_NO_PRIMARY_HP), SND_PCI_QUIRK(0x104d, 0x9044, "Sony VAIO AiO", ALC882_FIXUP_NO_PRIMARY_HP), SND_PCI_QUIRK(0x104d, 0x9047, "Sony Vaio TT", ALC889_FIXUP_VAIO_TT), diff --git a/sound/soc/codecs/rt5514.c b/sound/soc/codecs/rt5514.c index 32fe76c3134a..0ecff512013e 100644 --- a/sound/soc/codecs/rt5514.c +++ b/sound/soc/codecs/rt5514.c @@ -422,7 +422,7 @@ static int rt5514_dsp_voice_wake_up_put(struct snd_kcontrol *kcontrol, } } - return 0; + return 1; } static const struct snd_kcontrol_new rt5514_snd_controls[] = { diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c index 9185bd7c5a6d..d34000182f67 100644 --- a/sound/soc/codecs/rt5645.c +++ b/sound/soc/codecs/rt5645.c @@ -4105,9 +4105,14 @@ static int rt5645_i2c_remove(struct i2c_client *i2c) if (i2c->irq) free_irq(i2c->irq, rt5645); + /* + * Since the rt5645_btn_check_callback() can queue jack_detect_work, + * the timer need to be delted first + */ + del_timer_sync(&rt5645->btn_check_timer); + cancel_delayed_work_sync(&rt5645->jack_detect_work); cancel_delayed_work_sync(&rt5645->rcclock_work); - del_timer_sync(&rt5645->btn_check_timer); regulator_bulk_disable(ARRAY_SIZE(rt5645->supplies), rt5645->supplies); diff --git a/sound/soc/codecs/tscs454.c b/sound/soc/codecs/tscs454.c index ff85a0bf6170..00a90ccd6566 100644 --- a/sound/soc/codecs/tscs454.c +++ b/sound/soc/codecs/tscs454.c @@ -3129,18 +3129,17 @@ static int set_aif_sample_format(struct snd_soc_component *component, unsigned int width; int ret; - switch (format) { - case SNDRV_PCM_FORMAT_S16_LE: + switch (snd_pcm_format_width(format)) { + case 16: width = FV_WL_16; break; - case SNDRV_PCM_FORMAT_S20_3LE: + case 20: width = FV_WL_20; break; - case SNDRV_PCM_FORMAT_S24_3LE: + case 24: width = FV_WL_24; break; - case SNDRV_PCM_FORMAT_S24_LE: - case SNDRV_PCM_FORMAT_S32_LE: + case 32: width = FV_WL_32; break; default: @@ -3338,6 +3337,7 @@ static const struct snd_soc_component_driver soc_component_dev_tscs454 = { .num_dapm_routes = ARRAY_SIZE(tscs454_intercon), .controls = tscs454_snd_controls, .num_controls = ARRAY_SIZE(tscs454_snd_controls), + .endianness = 1, }; #define TSCS454_RATES SNDRV_PCM_RATE_8000_96000 diff --git a/sound/soc/codecs/wm2000.c b/sound/soc/codecs/wm2000.c index c5ae07234a00..cad39f63b763 100644 --- a/sound/soc/codecs/wm2000.c +++ b/sound/soc/codecs/wm2000.c @@ -545,7 +545,7 @@ static int wm2000_anc_transition(struct wm2000_priv *wm2000, { struct i2c_client *i2c = wm2000->i2c; int i, j; - int ret; + int ret = 0; if (wm2000->anc_mode == mode) return 0; @@ -575,13 +575,13 @@ static int wm2000_anc_transition(struct wm2000_priv *wm2000, ret = anc_transitions[i].step[j](i2c, anc_transitions[i].analogue); if (ret != 0) - return ret; + break; } if (anc_transitions[i].dest == ANC_OFF) clk_disable_unprepare(wm2000->mclk); - return 0; + return ret; } static int wm2000_anc_set_mode(struct wm2000_priv *wm2000) diff --git a/sound/soc/mediatek/mt2701/mt2701-wm8960.c b/sound/soc/mediatek/mt2701/mt2701-wm8960.c index 89f34efd9747..a5ede216b795 100644 --- a/sound/soc/mediatek/mt2701/mt2701-wm8960.c +++ b/sound/soc/mediatek/mt2701/mt2701-wm8960.c @@ -118,7 +118,8 @@ static int mt2701_wm8960_machine_probe(struct platform_device *pdev) if (!codec_node) { dev_err(&pdev->dev, "Property 'audio-codec' missing or invalid\n"); - return -EINVAL; + ret = -EINVAL; + goto put_platform_node; } for (i = 0; i < card->num_links; i++) { if (mt2701_wm8960_dai_links[i].codec_name) @@ -129,7 +130,7 @@ static int mt2701_wm8960_machine_probe(struct platform_device *pdev) ret = snd_soc_of_parse_audio_routing(card, "audio-routing"); if (ret) { dev_err(&pdev->dev, "failed to parse audio-routing: %d\n", ret); - return ret; + goto put_codec_node; } ret = devm_snd_soc_register_card(&pdev->dev, card); @@ -137,6 +138,10 @@ static int mt2701_wm8960_machine_probe(struct platform_device *pdev) dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n", __func__, ret); +put_codec_node: + of_node_put(codec_node); +put_platform_node: + of_node_put(platform_node); return ret; } diff --git a/sound/soc/mediatek/mt8173/mt8173-max98090.c b/sound/soc/mediatek/mt8173/mt8173-max98090.c index 431ba3db1759..c9fc719c2af9 100644 --- a/sound/soc/mediatek/mt8173/mt8173-max98090.c +++ b/sound/soc/mediatek/mt8173/mt8173-max98090.c @@ -156,7 +156,8 @@ static int mt8173_max98090_dev_probe(struct platform_device *pdev) if (!codec_node) { dev_err(&pdev->dev, "Property 'audio-codec' missing or invalid\n"); - return -EINVAL; + ret = -EINVAL; + goto put_platform_node; } for (i = 0; i < card->num_links; i++) { if (mt8173_max98090_dais[i].codec_name) @@ -171,6 +172,8 @@ static int mt8173_max98090_dev_probe(struct platform_device *pdev) __func__, ret); of_node_put(codec_node); + +put_platform_node: of_node_put(platform_node); return ret; } diff --git a/sound/soc/mxs/mxs-saif.c b/sound/soc/mxs/mxs-saif.c index 93c019670199..6d0ab4e75518 100644 --- a/sound/soc/mxs/mxs-saif.c +++ b/sound/soc/mxs/mxs-saif.c @@ -780,6 +780,7 @@ static int mxs_saif_probe(struct platform_device *pdev) saif->master_id = saif->id; } else { ret = of_alias_get_id(master, "saif"); + of_node_put(master); if (ret < 0) return ret; else diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 8b52a2fa540c..e93d5dce10f4 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -3312,7 +3312,6 @@ int snd_soc_dapm_put_volsw(struct snd_kcontrol *kcontrol, update.val = val; card->update = &update; } - change |= reg_change; ret = soc_dapm_mixer_update_power(card, kcontrol, connect, rconnect); @@ -3418,7 +3417,6 @@ int snd_soc_dapm_put_enum_double(struct snd_kcontrol *kcontrol, update.val = val; card->update = &update; } - change |= reg_change; ret = soc_dapm_mux_update_power(card, kcontrol, item[0], e); diff --git a/techpack/camera/drivers/cam_fd/fd_hw_mgr/cam_fd_hw_mgr.c b/techpack/camera/drivers/cam_fd/fd_hw_mgr/cam_fd_hw_mgr.c index 84c2160a9d05..e99ddb2567a9 100644 --- a/techpack/camera/drivers/cam_fd/fd_hw_mgr/cam_fd_hw_mgr.c +++ b/techpack/camera/drivers/cam_fd/fd_hw_mgr/cam_fd_hw_mgr.c @@ -878,6 +878,7 @@ static int cam_fd_mgr_util_submit_frame(void *priv, void *data) list_del_init(&frame_req->list); hw_mgr->num_pending_frames--; frame_req->submit_timestamp = ktime_get(); + list_add_tail(&frame_req->list, &hw_mgr->frame_processing_list); if (hw_device->hw_intf->hw_ops.start) { start_args.hw_ctx = hw_ctx; @@ -909,7 +910,6 @@ static int cam_fd_mgr_util_submit_frame(void *priv, void *data) hw_device->ready_to_process = false; hw_device->cur_hw_ctx = hw_ctx; hw_device->req_id = frame_req->request_id; - list_add_tail(&frame_req->list, &hw_mgr->frame_processing_list); mutex_unlock(&hw_device->lock); mutex_unlock(&hw_mgr->frame_req_mutex); diff --git a/techpack/display/msm/dsi/dsi_display.c b/techpack/display/msm/dsi/dsi_display.c index d448fd04c4f5..31bb05d75c29 100644 --- a/techpack/display/msm/dsi/dsi_display.c +++ b/techpack/display/msm/dsi/dsi_display.c @@ -5783,7 +5783,9 @@ int dsi_display_dev_probe(struct platform_device *pdev) display->panel_node = panel_node; display->pdev = pdev; display->boot_disp = boot_disp; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) display->is_prim_display = true; +#endif dsi_display_parse_cmdline_topology(display, index); diff --git a/techpack/display/msm/dsi/dsi_display.h b/techpack/display/msm/dsi/dsi_display.h index 50bd45d1e998..2da0505ac062 100644 --- a/techpack/display/msm/dsi/dsi_display.h +++ b/techpack/display/msm/dsi/dsi_display.h @@ -199,7 +199,9 @@ struct dsi_display { const char *display_type; struct list_head list; bool is_cont_splash_enabled; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) bool is_prim_display; +#endif bool sw_te_using_wd; struct mutex display_lock; int disp_te_gpio; diff --git a/techpack/display/msm/dsi/dsi_drm.c b/techpack/display/msm/dsi/dsi_drm.c index 1fed013fcfe1..4bc8ddb7027c 100644 --- a/techpack/display/msm/dsi/dsi_drm.c +++ b/techpack/display/msm/dsi/dsi_drm.c @@ -12,8 +12,10 @@ #include "sde_connector.h" #include "dsi_drm.h" #include "sde_trace.h" +#if IS_ENABLED(CONFIG_MI_DRM_OPT) #include #include +#endif #include "msm_drv.h" #include "sde_dbg.h" #include "dsi_defs.h" @@ -35,12 +37,14 @@ static struct dsi_display_mode_priv_info default_priv_info = { .dsc_enabled = false, }; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) #define WAIT_RESUME_TIMEOUT 200 struct dsi_bridge *gbridge; static struct delayed_work prim_panel_work; static atomic_t prim_panel_is_on; static struct wakeup_source *prim_panel_wakelock; +#endif static void convert_to_dsi_mode(const struct drm_display_mode *drm_mode, struct dsi_display_mode *dsi_mode) @@ -204,6 +208,7 @@ static void dsi_bridge_pre_enable(struct drm_bridge *bridge) return; } +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (c_bridge->display->is_prim_display && atomic_read(&prim_panel_is_on) && !mi_cfg->fod_dimlayer_enabled) { cancel_delayed_work_sync(&prim_panel_work); __pm_relax(prim_panel_wakelock); @@ -216,6 +221,7 @@ static void dsi_bridge_pre_enable(struct drm_bridge *bridge) return; } } +#endif if (mi_cfg->fod_dimlayer_enabled) { power_mode = sde_connector_get_lp(c_bridge->display->drm_conn); @@ -261,11 +267,14 @@ static void dsi_bridge_pre_enable(struct drm_bridge *bridge) DSI_ERR("Continuous splash pipeline cleanup failed, rc=%d\n", rc); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (c_bridge->display->is_prim_display) atomic_set(&prim_panel_is_on, true); +#endif } +#if IS_ENABLED(CONFIG_MI_DRM_OPT) /** * dsi_bridge_interface_enable - Panel light on interface for fingerprint * In order to improve panel light on performance when unlock device by @@ -309,6 +318,7 @@ int dsi_bridge_interface_enable(int timeout) return ret; } EXPORT_SYMBOL(dsi_bridge_interface_enable); +#endif static void dsi_bridge_enable(struct drm_bridge *bridge) { @@ -443,11 +453,13 @@ static void dsi_bridge_post_disable(struct drm_bridge *bridge) mi_drm_notifier_call_chain(MI_DRM_EVENT_BLANK, ¬ify_data); SDE_ATRACE_END("dsi_bridge_post_disable"); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (c_bridge->display->is_prim_display) atomic_set(&prim_panel_is_on, false); - +#endif } +#if IS_ENABLED(CONFIG_MI_DRM_OPT) static void prim_panel_off_delayed_work(struct work_struct *work) { mutex_lock(&gbridge->base.lock); @@ -459,6 +471,7 @@ static void prim_panel_off_delayed_work(struct work_struct *work) } mutex_unlock(&gbridge->base.lock); } +#endif static void dsi_bridge_mode_set(struct drm_bridge *bridge, struct drm_display_mode *mode, @@ -1210,6 +1223,7 @@ struct dsi_bridge *dsi_drm_bridge_init(struct dsi_display *display, encoder->bridge->is_dsi_drm_bridge = true; mutex_init(&encoder->bridge->lock); +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (display->is_prim_display) { gbridge = bridge; atomic_set(&resume_pending, 0); @@ -1219,6 +1233,7 @@ struct dsi_bridge *dsi_drm_bridge_init(struct dsi_display *display, init_waitqueue_head(&resume_wait_q); INIT_DELAYED_WORK(&prim_panel_work, prim_panel_off_delayed_work); } +#endif return bridge; error_free_bridge: @@ -1232,12 +1247,14 @@ void dsi_drm_bridge_cleanup(struct dsi_bridge *bridge) if (bridge && bridge->base.encoder) bridge->base.encoder->bridge = NULL; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) if (bridge == gbridge) { atomic_set(&prim_panel_is_on, false); cancel_delayed_work_sync(&prim_panel_work); wakeup_source_remove(prim_panel_wakelock); wakeup_source_destroy(prim_panel_wakelock); } +#endif kfree(bridge); } diff --git a/techpack/display/msm/dsi/dsi_drm.h b/techpack/display/msm/dsi/dsi_drm.h index 051dd6d97bf0..537d936a2935 100644 --- a/techpack/display/msm/dsi/dsi_drm.h +++ b/techpack/display/msm/dsi/dsi_drm.h @@ -23,7 +23,9 @@ struct dsi_bridge { struct dsi_display *display; struct dsi_display_mode dsi_mode; +#if IS_ENABLED(CONFIG_MI_DRM_OPT) struct mutex lock; +#endif }; /** diff --git a/techpack/display/msm/dsi/dsi_phy.c b/techpack/display/msm/dsi/dsi_phy.c index a21966e86617..095a8dbdeda4 100644 --- a/techpack/display/msm/dsi/dsi_phy.c +++ b/techpack/display/msm/dsi/dsi_phy.c @@ -325,7 +325,11 @@ static int dsi_phy_settings_init(struct platform_device *pdev, /* Actual timing values are dependent on panel */ timing->count_per_lane = phy->ver_info->timing_cfg_count; +#ifndef CONFIG_BOARD_UMI phy->allow_phy_power_off = true; +#else + phy->allow_phy_power_off = false; +#endif of_property_read_u32(pdev->dev.of_node, "qcom,dsi-phy-regulator-min-datarate-bps", diff --git a/techpack/display/msm/msm_drv.c b/techpack/display/msm/msm_drv.c index 1c8cb6773889..d11e201a1300 100644 --- a/techpack/display/msm/msm_drv.c +++ b/techpack/display/msm/msm_drv.c @@ -64,8 +64,11 @@ #define MSM_VERSION_PATCHLEVEL 0 static DEFINE_MUTEX(msm_release_lock); + +#if IS_ENABLED(CONFIG_MI_DRM_OPT) atomic_t resume_pending; wait_queue_head_t resume_wait_q; +#endif #define IDLE_ENCODER_MASK_DEFAULT 1 #define IDLE_TIMEOUT_MS_DEFAULT 100 @@ -1882,6 +1885,7 @@ static struct drm_driver msm_driver = { }; #ifdef CONFIG_PM_SLEEP +#if IS_ENABLED(CONFIG_MI_DRM_OPT) static int msm_pm_prepare(struct device *dev) { atomic_inc(&resume_pending); @@ -1894,6 +1898,7 @@ static void msm_pm_complete(struct device *dev) wake_up_all(&resume_wait_q); return; } +#endif static int msm_pm_suspend(struct device *dev) { @@ -1980,8 +1985,10 @@ static int msm_runtime_resume(struct device *dev) #endif static const struct dev_pm_ops msm_pm_ops = { +#if IS_ENABLED(CONFIG_MI_DRM_OPT) .prepare = msm_pm_prepare, .complete = msm_pm_complete, +#endif SET_SYSTEM_SLEEP_PM_OPS(msm_pm_suspend, msm_pm_resume) SET_RUNTIME_PM_OPS(msm_runtime_suspend, msm_runtime_resume, NULL) }; diff --git a/techpack/display/msm/msm_drv.h b/techpack/display/msm/msm_drv.h index b34382f030da..70712d272ded 100644 --- a/techpack/display/msm/msm_drv.h +++ b/techpack/display/msm/msm_drv.h @@ -73,8 +73,10 @@ struct msm_gem_vma; #define TEARDOWN_DEADLOCK_RETRY_MAX 5 +#if IS_ENABLED(CONFIG_MI_DRM_OPT) extern atomic_t resume_pending; extern wait_queue_head_t resume_wait_q; +#endif struct msm_file_private { rwlock_t queuelock; diff --git a/techpack/display/msm/sde/sde_encoder.c b/techpack/display/msm/sde/sde_encoder.c index b595dc966f27..e1bf703a9d59 100644 --- a/techpack/display/msm/sde/sde_encoder.c +++ b/techpack/display/msm/sde/sde_encoder.c @@ -4709,7 +4709,11 @@ static void sde_encoder_touch_notify_work_handler(struct kthread_work *work) if (c_bridge) dsi_display = c_bridge->display; - if (dsi_display && dsi_display->is_prim_display && dsi_display->panel + if (dsi_display && +#if IS_ENABLED(CONFIG_MI_DRM_OPT) + dsi_display->is_prim_display && +#endif + dsi_display->panel && dsi_display->panel->mi_cfg.smart_fps_restore) { if (dsi_display->panel->mi_cfg.smart_fps_support && fm_stat.enabled) { calc_fps(0, (int)true); diff --git a/tools/lib/traceevent/Makefile b/tools/lib/traceevent/Makefile index 05f8a0f27121..b7f7e4e541d7 100644 --- a/tools/lib/traceevent/Makefile +++ b/tools/lib/traceevent/Makefile @@ -263,7 +263,7 @@ define do_generate_dynamic_list_file xargs echo "U w W" | tr 'w ' 'W\n' | sort -u | xargs echo`;\ if [ "$$symbol_type" = "U W" ];then \ (echo '{'; \ - $(NM) -u -D $1 | awk 'NF>1 {print "\t"$$2";"}' | sort -u;\ + $(NM) -u -D $1 | awk 'NF>1 {sub("@.*", "", $$2); print "\t"$$2";"}' | sort -u;\ echo '};'; \ ) > $2; \ else \ diff --git a/tools/perf/bench/bench.h b/tools/perf/bench/bench.h index 6c9fcd757f31..b3e418afc21a 100644 --- a/tools/perf/bench/bench.h +++ b/tools/perf/bench/bench.h @@ -2,6 +2,10 @@ #ifndef BENCH_H #define BENCH_H +#include + +extern struct timeval bench__start, bench__end, bench__runtime; + /* * The madvise transparent hugepage constants were added in glibc * 2.13. For compatibility with older versions of glibc, define these diff --git a/tools/perf/bench/futex-hash.c b/tools/perf/bench/futex-hash.c index 9aa3a674829b..ee9b28065109 100644 --- a/tools/perf/bench/futex-hash.c +++ b/tools/perf/bench/futex-hash.c @@ -35,7 +35,7 @@ static unsigned int nfutexes = 1024; static bool fshared = false, done = false, silent = false; static int futex_flag = 0; -struct timeval start, end, runtime; +struct timeval bench__start, bench__end, bench__runtime; static pthread_mutex_t thread_lock; static unsigned int threads_starting; static struct stats throughput_stats; @@ -101,8 +101,8 @@ static void toggle_done(int sig __maybe_unused, { /* inform all threads that we're done for the day */ done = true; - gettimeofday(&end, NULL); - timersub(&end, &start, &runtime); + gettimeofday(&bench__end, NULL); + timersub(&bench__end, &bench__start, &bench__runtime); } static void print_summary(void) @@ -112,7 +112,7 @@ static void print_summary(void) printf("%sAveraged %ld operations/sec (+- %.2f%%), total secs = %d\n", !silent ? "\n" : "", avg, rel_stddev_stats(stddev, avg), - (int) runtime.tv_sec); + (int)bench__runtime.tv_sec); } int bench_futex_hash(int argc, const char **argv) @@ -159,7 +159,7 @@ int bench_futex_hash(int argc, const char **argv) threads_starting = nthreads; pthread_attr_init(&thread_attr); - gettimeofday(&start, NULL); + gettimeofday(&bench__start, NULL); for (i = 0; i < nthreads; i++) { worker[i].tid = i; worker[i].futex = calloc(nfutexes, sizeof(*worker[i].futex)); @@ -202,7 +202,7 @@ int bench_futex_hash(int argc, const char **argv) pthread_mutex_destroy(&thread_lock); for (i = 0; i < nthreads; i++) { - unsigned long t = worker[i].ops/runtime.tv_sec; + unsigned long t = worker[i].ops / bench__runtime.tv_sec; update_stats(&throughput_stats, t); if (!silent) { if (nfutexes == 1) diff --git a/tools/perf/bench/futex-lock-pi.c b/tools/perf/bench/futex-lock-pi.c index 8e9c4753e304..017609ae3590 100644 --- a/tools/perf/bench/futex-lock-pi.c +++ b/tools/perf/bench/futex-lock-pi.c @@ -35,7 +35,6 @@ static bool silent = false, multi = false; static bool done = false, fshared = false; static unsigned int nthreads = 0; static int futex_flag = 0; -struct timeval start, end, runtime; static pthread_mutex_t thread_lock; static unsigned int threads_starting; static struct stats throughput_stats; @@ -62,7 +61,7 @@ static void print_summary(void) printf("%sAveraged %ld operations/sec (+- %.2f%%), total secs = %d\n", !silent ? "\n" : "", avg, rel_stddev_stats(stddev, avg), - (int) runtime.tv_sec); + (int)bench__runtime.tv_sec); } static void toggle_done(int sig __maybe_unused, @@ -71,8 +70,8 @@ static void toggle_done(int sig __maybe_unused, { /* inform all threads that we're done for the day */ done = true; - gettimeofday(&end, NULL); - timersub(&end, &start, &runtime); + gettimeofday(&bench__end, NULL); + timersub(&bench__end, &bench__start, &bench__runtime); } static void *workerfn(void *arg) @@ -183,7 +182,7 @@ int bench_futex_lock_pi(int argc, const char **argv) threads_starting = nthreads; pthread_attr_init(&thread_attr); - gettimeofday(&start, NULL); + gettimeofday(&bench__start, NULL); create_threads(worker, thread_attr, cpu); pthread_attr_destroy(&thread_attr); @@ -209,7 +208,7 @@ int bench_futex_lock_pi(int argc, const char **argv) pthread_mutex_destroy(&thread_lock); for (i = 0; i < nthreads; i++) { - unsigned long t = worker[i].ops/runtime.tv_sec; + unsigned long t = worker[i].ops / bench__runtime.tv_sec; update_stats(&throughput_stats, t); if (!silent) diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index 2bd39fdc8ab0..f3c142bd1a11 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -944,8 +944,8 @@ percent_rmt_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused, double per_left; double per_right; - per_left = PERCENT(left, lcl_hitm); - per_right = PERCENT(right, lcl_hitm); + per_left = PERCENT(left, rmt_hitm); + per_right = PERCENT(right, rmt_hitm); return per_left - per_right; } @@ -2724,9 +2724,7 @@ static int perf_c2c__report(int argc, const char **argv) "the input file to process"), OPT_INCR('N', "node-info", &c2c.node_info, "show extra node info in report (repeat for more info)"), -#ifdef HAVE_SLANG_SUPPORT OPT_BOOLEAN(0, "stdio", &c2c.use_stdio, "Use the stdio interface"), -#endif OPT_BOOLEAN(0, "stats", &c2c.stats_only, "Display only statistic tables (implies --stdio)"), OPT_BOOLEAN(0, "full-symbols", &c2c.symbol_full, @@ -2753,6 +2751,10 @@ static int perf_c2c__report(int argc, const char **argv) if (argc) usage_with_options(report_c2c_usage, options); +#ifndef HAVE_SLANG_SUPPORT + c2c.use_stdio = true; +#endif + if (c2c.stats_only) c2c.use_stdio = true; diff --git a/tools/perf/pmu-events/jevents.c b/tools/perf/pmu-events/jevents.c index 31331c42b0e3..0f5a63026d21 100644 --- a/tools/perf/pmu-events/jevents.c +++ b/tools/perf/pmu-events/jevents.c @@ -563,7 +563,7 @@ int json_events(const char *fn, } else if (json_streq(map, field, "ExtSel")) { char *code = NULL; addfield(map, &code, "", "", val); - eventcode |= strtoul(code, NULL, 0) << 21; + eventcode |= strtoul(code, NULL, 0) << 8; free(code); } else if (json_streq(map, field, "EventName")) { addfield(map, &name, "", "", val); diff --git a/tools/perf/tests/bp_account.c b/tools/perf/tests/bp_account.c index a20cbc445426..624e4ef73d1c 100644 --- a/tools/perf/tests/bp_account.c +++ b/tools/perf/tests/bp_account.c @@ -22,7 +22,7 @@ #include "perf.h" #include "cloexec.h" -volatile long the_var; +static volatile long the_var; static noinline int test_function(void) {